@browserbasehq/orca 3.0.9-alpha-4 → 3.1.0-patch.1

package/dist/esm/lib/v3/agent/OpenAICUAClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OpenAICUAClient.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/OpenAICUAClient.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAe5B,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EACL,4BAA4B,EAC5B,oBAAoB,GACrB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,wBAAwB,GACzB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAEpC;;;GAGG;AACH,MAAM,OAAO,eAAgB,SAAQ,WAAW;IACtC,MAAM,CAAS;IACf,YAAY,CAAU;IACtB,OAAO,CAAS;IAChB,MAAM,CAAS;IAChB,cAAc,CAAU;IACvB,eAAe,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC/C,UAAU,CAAU;IACpB,kBAAkB,CAAyB;IAC3C,aAAa,CAA0C;IACvD,cAAc,GAA8B,IAAI,GAAG,EAAE,CAAC;IACtD,WAAW,GAAW,SAAS,CAAC,CAAC,2CAA2C;IAC5E,KAAK,CAAW;IAChB,yBAAyB,CAA6B;IAE9D,YACE,IAAe,EACf,SAAiB,EACjB,wBAAiC,EACjC,aAA6B,EAC7B,KAAe;QAEf,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,wBAAwB,CAAC,CAAC;QAEjD,yBAAyB;QACzB,IAAI,CAAC,MAAM;YACR,aAAa,EAAE,MAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,OAAO,GAAI,aAAa,EAAE,OAAkB,IAAI,SAAS,CAAC;QAC/D,IAAI,CAAC,YAAY;YACd,aAAa,EAAE,YAAuB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QAEpE,+BAA+B;QAC/B,IACE,aAAa,EAAE,WAAW;YAC1B,OAAO,aAAa,CAAC,WAAW,KAAK,QAAQ,EAC7C,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,aAAa,CAAC,WAAW,CAAC;QAC/C,CAAC;QAED,qCAAqC;QACrC,IAAI,CAAC,aAAa,GAAG;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,aAAa,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5C,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,MAAc;QACvC,IAAI,CAAC,eAAe,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACxB,CAAC;IAED,qBAAqB,CAAC,QAA+B;QACnD,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAED,gBAAgB,CAAC,OAA+C;QAC9D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IAC/B,CAAC;IAED,QAAQ,CAAC,KAAc;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,4BAA4B,CAAC,OAAmC;QAC9D,IAAI,CAAC,yBAAyB,GAAG,OAAO,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,gBAAuC;QACnD,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC7C,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAExC,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,qCAAqC;QAElE,qCAAqC;QACrC,IAAI,UAAU,GAAG,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,kBAAkB,GAAuB,SAAS,CAAC;QACvD,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,IAAI,CAAC;YACH,sDAAsD;YACtD,OAAO,CAAC,SAAS,IAAI,WAAW,GAAG,QAAQ,EAAE,CAAC;gBAC5C,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,kBAAkB,WAAW,GAAG,CAAC,IAAI,QAAQ,EAAE;oBACxD,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,UAAU,EACV,kBAAkB,EAClB,MAAM,CACP,CAAC;gBACF,gBAAgB,IAAI,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;gBAC9C,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC;gBAChD,kBAAkB,IAAI,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;gBAErD,0BAA0B;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;gBAEhC,2BAA2B;gBAC3B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAE7B,sDAAsD;gBACtD,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC;gBAEvC,+DAA+D;gBAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC;gBACrC,CAAC;gBAED,mCAAmC;gBACnC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBACjC,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC;gBAChC,CAAC;gBAED,yBAAyB;gBACzB,WAAW,EAAE,CAAC;YAChB,CAAC;YAED,0BAA0B;YAC1B,OAAO;gBACL,OAAO,EAAE,SAAS;gBAClB,OAAO;gBACP,OAAO,EAAE,YAAY;gBACrB,SAAS;gBACT,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,+BAA+B,YAAY,EAAE;gBACtD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO;gBACP,OAAO,EAAE,2BAA2B,YAAY,EAAE;gBAClD,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,UAA+B,EAC/B,kBAAsC,EACtC,MAAkC;QAalC,IAAI,CAAC;YACH,8BAA8B;YAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACrC,MAAM,KAAK,GAAG;gBACZ,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;gBACvC,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa;gBACzC,iBAAiB,EAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB;aAClD,CAAC;YAEF,qCAAqC;YACrC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBACvC,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE;wBACnD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAkB,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnE,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,MAAM,CAAC,IAAI,cAAc,IAAI,CAAC,OAAO,EAAE;wBAC7E,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBACtD,IAAI,MAAM,EAAE,CAAC;wBACX,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACzB,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,sCAAsC,MAAM,CAAC,IAAI,EAAE;4BAC5D,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;qBAAM,IACL,IAAI,CAAC,IAAI,KAAK,eAAe;oBAC7B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAC7B,CAAC;oBACD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,cAAc,IAAI,CAAC,OAAO,EAAE;wBACtE,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBACtD,IAAI,MAAM,EAAE,CAAC;wBACX,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACzB,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,sCAAsC,MAAM,CAAC,IAAI,EAAE;4BAC5D,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,uBAAuB;YACvB,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC5B,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,qBAAqB;wBAC9B,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;wBAChD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;4BACnC,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gCACnD,OAAO,IAAI,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;gCAC/B,MAAM,CAAC;oCACL,QAAQ,EAAE,OAAO;oCACjB,OAAO,EAAE,iBAAiB,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE;oCACtD,KAAK,EAAE,CAAC;iCACT,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAE7D,qBAAqB;YACrB,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,CAAC;gBACnB,MAAM,CAAC,KAAK,CACV,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,CAC/D,CAAC;YAEJ,OAAO;gBACL,OAAO,EAAE,WAAW;gBACpB,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;gBACvB,SAAS;gBACT,cAAc;gBACd,UAAU;gBACV,KAAK,EAAE,KAAK;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,yBAAyB,YAAY,EAAE;gBAChD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,kBAAkB,CAAC,IAAkB;QAC3C,OAAO,CACL,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,SAAS,IAAI,IAAI;YACjB,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAChC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,mBAAkC,EAClC,MAAkC;QAElC,IAAI,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnC,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,sCAAsC,mBAAmB,CAAC,MAAM,cAAc,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC1I,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,MAAM,QAAQ,GACZ,MAAM,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;YAE5D,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;gBAC1B,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO,mBAAmB,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,gCAAgC;oBACzC,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;QAED,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,sBAAsB,mBAAmB,CAAC,MAAM,kBAAkB;YAC3E,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IAEO,kBAAkB,CAAC,IAAkB;QAC3C,OAAO,CACL,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,SAAS,IAAI,IAAI;YACjB,MAAM,IAAI,IAAI;YACd,WAAW,IAAI,IAAI,CACpB,CAAC;IACJ,CAAC;IAEO,uBAAuB,CAAC,WAAmB;QACjD,6EAA6E;QAC7E,OAAO;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,IAAI,CAAC,wBAAwB;aACvC;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,UAA+B,EAC/B,kBAA2B;QAM3B,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,aAAa,GAA4B;gBAC7C,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,sBAAsB;wBAC5B,aAAa,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK;wBACzC,cAAc,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;wBAC3C,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B;iBACF;gBACD,KAAK,EAAE,UAAU;gBACjB,UAAU,EAAE,MAAM;aACnB,CAAC;YAEF,gCAAgC;YAChC,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;oBACpE,IAAI,EAAE,UAAmB;oBACzB,IAAI;oBACJ,QAAQ,EAAE;wBACR,IAAI;wBACJ,WAAW,EAAE,IAAI,CAAC,WAAW;wBAC7B,UAAU,EAAE,IAAI,CAAC,WAAW;qBAC7B;iBACF,CAAC,CAAC,CAAC;gBAEJ,aAAa,CAAC,KAAK,GAAG;oBACpB,GAAI,aAAa,CAAC,KAAmC;oBACrD,GAAG,WAAW;iBACf,CAAC;YACJ,CAAC;YAED,wCAAwC;YACxC,IAAI,kBAAkB,EAAE,CAAC;gBACvB,aAAa,CAAC,oBAAoB,GAAG,kBAAkB,CAAC;YAC1D,CAAC;YAED,kBAAkB;YAClB,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC;YAC9B,iBAAiB,CAAC,aAAa,CAAC;gBAC9B,SAAS,EAAE,YAAY;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,SAAS,EAAE,eAAe;gBAC1B,MAAM,EAAE,sBAAsB,CAAC,UAAU,CAAC;aAC3C,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,qDAAqD;YACrD,qEAAqE;YACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;YAEtC,kDAAkD;YAClD,MAAM,KAAK,GAAG;gBACZ,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;gBACzC,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;gBAC3C,iBAAiB,EAAE,SAAS;aAC7B,CAAC;YAEF,mBAAmB;YACnB,iBAAiB,CAAC,cAAc,CAAC;gBAC/B,SAAS,EAAE,YAAY;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,SAAS,EAAE,eAAe;gBAC1B,MAAM,EAAE,wBAAwB,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACjD,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;gBACxC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;aAC3C,CAAC,CAAC;YAEH,uCAAuC;YACvC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,EAAE,CAAC;YAElC,oCAAoC;YACpC,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAmC;gBACpD,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,KAAK;aACN,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CACd,MAAsB,EACtB,MAAkC;QAElC,MAAM,cAAc,GAAwB,EAAE,CAAC;QAE/C,2BAA2B;QAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnE,wBAAwB;gBACxB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBAEtD,IAAI,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,8BAA8B,MAAM,CAAC,IAAI,EAAE;4BACpD,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACnC,CAAC;oBAED,uBAAuB;oBACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAElD,qDAAqD;oBACrD,MAAM,UAAU,GAAG;wBACjB,IAAI,EAAE,sBAA+B;wBACrC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE;4BACN,IAAI,EAAE,aAAsB;4BAC5B,SAAS,EAAE,UAAU;yBACtB;qBACmB,CAAC;oBAEvB,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,2CAA2C,IAAI,CAAC,OAAO,EAAE;wBAClE,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,+BAA+B;oBAC/B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,MAAM,kBAAkB,GAAG,UAS1B,CAAC;wBACF,kBAAkB,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;oBAC1D,CAAC;oBAED,IACE,IAAI,CAAC,qBAAqB;wBAC1B,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EACrC,CAAC;wBACD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC5D,IAAI,CAAC,qBAAqB,EAC1B,MAAM,CACP,CAAC;wBAEF,IAAI,kBAAkB,EAAE,CAAC;4BACvB,MAAM,kBAAkB,GAAG,UAQ1B,CAAC;4BACF,kBAAkB,CAAC,0BAA0B;gCAC3C,kBAAkB,CAAC;wBACvB,CAAC;oBACH,CAAC;oBAED,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;wBAC1C,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAEzD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,kCAAkC,YAAY,EAAE;wBACzD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,IAAI,CAAC;wBACH,qCAAqC;wBACrC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBAElD,MAAM,eAAe,GAAG;4BACtB,IAAI,EAAE,sBAA+B;4BACrC,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE;gCACN,IAAI,EAAE,aAAsB;gCAC5B,SAAS,EAAE,UAAU;gCACrB,KAAK,EAAE,YAAY;6BACpB;yBACmB,CAAC;wBAEvB,+BAA+B;wBAC/B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;4BACpB,MAAM,kBAAkB,GAAG,eAS1B,CAAC;4BACF,kBAAkB,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;wBAC1D,CAAC;wBAED,IACE,IAAI,CAAC,qBAAqB;4BAC1B,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EACrC,CAAC;4BACD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC5D,IAAI,CAAC,qBAAqB,EAC1B,MAAM,CACP,CAAC;4BAEF,IAAI,kBAAkB,EAAE,CAAC;gCACvB,MAAM,kBAAkB,GAAG,eAQ1B,CAAC;gCACF,kBAAkB,CAAC,0BAA0B;oCAC3C,kBAAkB,CAAC;4BACvB,CAAC;wBACH,CAAC;wBAED,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;oBACvC,CAAC;oBAAC,OAAO,eAAe,EAAE,CAAC;wBACzB,IAAI,eAAe,YAAY,oBAAoB,EAAE,CAAC;4BACpD,MAAM,eAAe,CAAC;wBACxB,CAAC;wBACD,wDAAwD;wBACxD,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,+BAA+B,MAAM,CAAC,eAAe,CAAC,EAAE;4BACjE,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;wBAEH,uEAAuE;wBACvE,cAAc,CAAC,IAAI,CAAC;4BAClB,IAAI,EAAE,sBAAsB;4BAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,UAAU,YAAY,EAAE;yBACZ,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,IACL,IAAI,CAAC,IAAI,KAAK,eAAe;gBAC7B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAC7B,CAAC;gBACD,qCAAqC;gBACrC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBAEtD,IAAI,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACnC,CAAC;oBAED,gCAAgC;oBAChC,IAAI,UAAU,GAAG,4BAA4B,CAAC;oBAC9C,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;wBAC1C,IAAI,CAAC;4BACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACnC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;4BAExC,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,SAAS,EAAE;gCACzE,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;4BAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;gCACtC,UAAU,EAAE,IAAI,CAAC,OAAO;gCACxB,QAAQ,EAAE,EAAE;6BACb,CAAC,CAAC;4BACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BAEpC,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,QAAQ,IAAI,CAAC,IAAI,oCAAoC,UAAU,EAAE;gCAC1E,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,SAAS,EAAE,CAAC;4BACnB,MAAM,YAAY,GAChB,SAAS,YAAY,KAAK;gCACxB,CAAC,CAAC,SAAS,CAAC,OAAO;gCACnB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;4BACxB,UAAU,GAAG,yBAAyB,YAAY,EAAE,CAAC;4BAErD,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,KAAK,YAAY,EAAE;gCAC7D,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,qDAAqD;oBACrD,MAAM,UAAU,GAAsB;wBACpC,IAAI,EAAE,sBAAsB;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE,UAAU;qBACnB,CAAC;oBAEF,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;wBAC1C,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAEzD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,kCAAkC,YAAY,EAAE;wBACzD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,yBAAyB;oBACzB,MAAM,eAAe,GAAsB;wBACzC,IAAI,EAAE,sBAAsB;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE,UAAU,YAAY,EAAE;qBACjC,CAAC;oBAEF,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,2BAA2B,CACjC,IAAsB;QAEtB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAExB,2FAA2F;QAC3F,6FAA6F;QAC7F,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAc;YAC3B,GAAG,MAAM,EAAE,wCAAwC;SACpD,CAAC;IACJ,CAAC;IAEO,2BAA2B,CACjC,IAAsB;QAEtB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExC,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,IAAI;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAGvB;QACC,oCAAoC;QACpC,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;YACzB,OAAO,yBAAyB,OAAO,CAAC,WAAW,EAAE,CAAC;QACxD,CAAC;QAED,2CAA2C;QAC3C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACpD,OAAO,yBAAyB,WAAW,EAAE,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;gBACpD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,MAAM,IAAI,4BAA4B,CACpC,yCAAyC;YACvC,iGAAiG,CACpG,CAAC;IACJ,CAAC;CACF","sourcesContent":["import OpenAI from \"openai\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport {\n  AgentAction,\n  AgentResult,\n  AgentType,\n  AgentExecutionOptions,\n  ResponseInputItem,\n  ResponseItem,\n  ComputerCallItem,\n  FunctionCallItem,\n  SafetyCheck,\n  SafetyConfirmationHandler,\n} from \"../types/public/agent.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport {\n  AgentScreenshotProviderError,\n  StagehandClosedError,\n} from \"../types/public/sdkErrors.js\";\nimport { ToolSet } from \"ai\";\nimport {\n  SessionFileLogger,\n  formatCuaPromptPreview,\n  formatCuaResponsePreview,\n} from \"../flowLogger.js\";\nimport { v7 as uuidv7 } from \"uuid\";\n\n/**\n * Client for OpenAI's Computer Use Assistant API\n * This implementation uses the official OpenAI Responses API for Computer Use\n */\nexport class OpenAICUAClient extends AgentClient {\n  private apiKey: string;\n  private organization?: string;\n  private baseURL: string;\n  private client: OpenAI;\n  public lastResponseId?: string;\n  private currentViewport = { width: 1288, height: 711 };\n  private currentUrl?: string;\n  private screenshotProvider?: () => Promise<string>;\n  private actionHandler?: (action: AgentAction) => Promise<void>;\n  private reasoningItems: Map<string, ResponseItem> = new Map();\n  private environment: string = \"browser\"; // \"browser\", \"mac\", \"windows\", or \"ubuntu\"\n  private tools?: ToolSet;\n  private safetyConfirmationHandler?: SafetyConfirmationHandler;\n\n  constructor(\n    type: AgentType,\n    modelName: string,\n    userProvidedInstructions?: string,\n    clientOptions?: ClientOptions,\n    tools?: ToolSet,\n  ) {\n    super(type, modelName, userProvidedInstructions);\n\n    // Process client options\n    this.apiKey =\n      (clientOptions?.apiKey as string) || process.env.OPENAI_API_KEY || \"\";\n    this.baseURL = (clientOptions?.baseURL as string) || undefined;\n    this.organization =\n      (clientOptions?.organization as string) || process.env.OPENAI_ORG;\n\n    // Get environment if specified\n    if (\n      clientOptions?.environment &&\n      typeof clientOptions.environment === \"string\"\n    ) {\n      this.environment = clientOptions.environment;\n    }\n\n    // Store client options for reference\n    this.clientOptions = {\n      apiKey: this.apiKey,\n    };\n\n    if (this.baseURL) {\n      this.clientOptions.baseURL = this.baseURL;\n    }\n\n    // Initialize the OpenAI client\n    this.client = new OpenAI(this.clientOptions);\n\n    this.tools = tools;\n  }\n\n  setViewport(width: number, height: number): void {\n    this.currentViewport = { width, height };\n  }\n\n  setCurrentUrl(url: string): void {\n    this.currentUrl = url;\n  }\n\n  setScreenshotProvider(provider: () => Promise<string>): void {\n    this.screenshotProvider = provider;\n  }\n\n  setActionHandler(handler: (action: AgentAction) => Promise<void>): void {\n    this.actionHandler = handler;\n  }\n\n  setTools(tools: ToolSet): void {\n    this.tools = tools;\n  }\n\n  setSafetyConfirmationHandler(handler?: SafetyConfirmationHandler): void {\n    this.safetyConfirmationHandler = handler;\n  }\n\n  /**\n   * Execute a task with the OpenAI CUA\n   * This is the main entry point for the agent\n   * @implements AgentClient.execute\n   */\n  async execute(executionOptions: AgentExecutionOptions): Promise<AgentResult> {\n    const { options, logger } = executionOptions;\n    const { instruction } = options;\n    const maxSteps = options.maxSteps || 10;\n\n    let currentStep = 0;\n    let completed = false;\n    const actions: AgentAction[] = [];\n    const messageList: string[] = [];\n    let finalMessage = \"\";\n    this.reasoningItems.clear(); // Clear any previous reasoning items\n\n    // Start with the initial instruction\n    let inputItems = this.createInitialInputItems(instruction);\n    let previousResponseId: string | undefined = undefined;\n    let totalInputTokens = 0;\n    let totalOutputTokens = 0;\n    let totalInferenceTime = 0;\n\n    try {\n      // Execute steps until completion or max steps reached\n      while (!completed && currentStep < maxSteps) {\n        logger({\n          category: \"agent\",\n          message: `Executing step ${currentStep + 1}/${maxSteps}`,\n          level: 1,\n        });\n\n        const result = await this.executeStep(\n          inputItems,\n          previousResponseId,\n          logger,\n        );\n        totalInputTokens += result.usage.input_tokens;\n        totalOutputTokens += result.usage.output_tokens;\n        totalInferenceTime += result.usage.inference_time_ms;\n\n        // Add actions to the list\n        actions.push(...result.actions);\n\n        // Update completion status\n        completed = result.completed;\n\n        // Store the previous response ID for the next request\n        previousResponseId = result.responseId;\n\n        // Update the input items for the next step if we're continuing\n        if (!completed) {\n          inputItems = result.nextInputItems;\n        }\n\n        // Record any message for this step\n        if (result.message) {\n          messageList.push(result.message);\n          finalMessage = result.message;\n        }\n\n        // Increment step counter\n        currentStep++;\n      }\n\n      // Return the final result\n      return {\n        success: completed,\n        actions,\n        message: finalMessage,\n        completed,\n        usage: {\n          input_tokens: totalInputTokens,\n          output_tokens: totalOutputTokens,\n          inference_time_ms: totalInferenceTime,\n        },\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      logger({\n        category: \"agent\",\n        message: `Error executing agent task: ${errorMessage}`,\n        level: 0,\n      });\n\n      return {\n        success: false,\n        actions,\n        message: `Failed to execute task: ${errorMessage}`,\n        completed: false,\n        usage: {\n          input_tokens: totalInputTokens,\n          output_tokens: totalOutputTokens,\n          inference_time_ms: totalInferenceTime,\n        },\n      };\n    }\n  }\n\n  /**\n   * Execute a single step of the agent\n   * This coordinates the flow: Request → Get Action → Execute Action\n   */\n  async executeStep(\n    inputItems: ResponseInputItem[],\n    previousResponseId: string | undefined,\n    logger: (message: LogLine) => void,\n  ): Promise<{\n    actions: AgentAction[];\n    message: string;\n    completed: boolean;\n    nextInputItems: ResponseInputItem[];\n    responseId: string;\n    usage: {\n      input_tokens: number;\n      output_tokens: number;\n      inference_time_ms: number;\n    };\n  }> {\n    try {\n      // Get response from the model\n      const result = await this.getAction(inputItems, previousResponseId);\n      const output = result.output;\n      const responseId = result.responseId;\n      const usage = {\n        input_tokens: result.usage.input_tokens,\n        output_tokens: result.usage.output_tokens,\n        inference_time_ms: result.usage.inference_time_ms,\n      };\n\n      // Add any reasoning items to our map\n      for (const item of output) {\n        if (item.type === \"reasoning\") {\n          this.reasoningItems.set(item.id, item);\n          logger({\n            category: \"agent\",\n            message: `Reasoning: ${String(item.content || \"\")}`,\n            level: 1,\n          });\n        }\n      }\n\n      // Extract actions from the output\n      const stepActions: AgentAction[] = [];\n      for (const item of output) {\n        if (item.type === \"computer_call\" && this.isComputerCallItem(item)) {\n          logger({\n            category: \"agent\",\n            message: `Found computer_call: ${item.action.type}, call_id: ${item.call_id}`,\n            level: 2,\n          });\n          const action = this.convertComputerCallToAction(item);\n          if (action) {\n            stepActions.push(action);\n            logger({\n              category: \"agent\",\n              message: `Converted computer_call to action: ${action.type}`,\n              level: 2,\n            });\n          }\n        } else if (\n          item.type === \"function_call\" &&\n          this.isFunctionCallItem(item)\n        ) {\n          logger({\n            category: \"agent\",\n            message: `Found function_call: ${item.name}, call_id: ${item.call_id}`,\n            level: 2,\n          });\n          const action = this.convertFunctionCallToAction(item);\n          if (action) {\n            stepActions.push(action);\n            logger({\n              category: \"agent\",\n              message: `Converted function_call to action: ${action.type}`,\n              level: 2,\n            });\n          }\n        }\n      }\n\n      // Extract message text\n      let message = \"\";\n      for (const item of output) {\n        if (item.type === \"message\") {\n          logger({\n            category: \"agent\",\n            message: `Found message block`,\n            level: 2,\n          });\n          if (item.content && Array.isArray(item.content)) {\n            for (const content of item.content) {\n              if (content.type === \"output_text\" && content.text) {\n                message += content.text + \"\\n\";\n                logger({\n                  category: \"agent\",\n                  message: `Message text: ${String(content.text || \"\")}`,\n                  level: 1,\n                });\n              }\n            }\n          }\n        }\n      }\n\n      // Take actions and get results\n      const nextInputItems = await this.takeAction(output, logger);\n\n      // Check if completed\n      const completed =\n        output.length === 0 ||\n        output.every(\n          (item) => item.type === \"message\" || item.type === \"reasoning\",\n        );\n\n      return {\n        actions: stepActions,\n        message: message.trim(),\n        completed,\n        nextInputItems,\n        responseId,\n        usage: usage,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      logger({\n        category: \"agent\",\n        message: `Error executing step: ${errorMessage}`,\n        level: 0,\n      });\n\n      throw error;\n    }\n  }\n\n  private isComputerCallItem(item: ResponseItem): item is ComputerCallItem {\n    return (\n      item.type === \"computer_call\" &&\n      \"call_id\" in item &&\n      \"action\" in item &&\n      typeof item.action === \"object\"\n    );\n  }\n\n  private async handleSafetyConfirmation(\n    pendingSafetyChecks: SafetyCheck[],\n    logger: (message: LogLine) => void,\n  ): Promise<SafetyCheck[] | undefined> {\n    if (this.safetyConfirmationHandler) {\n      logger({\n        category: \"agent\",\n        message: `Requesting safety confirmation for ${pendingSafetyChecks.length} check(s): ${pendingSafetyChecks.map((c) => c.code).join(\", \")}`,\n        level: 1,\n      });\n\n      const response =\n        await this.safetyConfirmationHandler(pendingSafetyChecks);\n\n      if (response.acknowledged) {\n        logger({\n          category: \"agent\",\n          message: `Safety checks acknowledged by user`,\n          level: 1,\n        });\n        return pendingSafetyChecks;\n      } else {\n        logger({\n          category: \"agent\",\n          message: `Safety checks rejected by user`,\n          level: 1,\n        });\n        return undefined;\n      }\n    }\n\n    logger({\n      category: \"agent\",\n      message: `Auto-acknowledging ${pendingSafetyChecks.length} safety check(s)`,\n      level: 2,\n    });\n    return pendingSafetyChecks;\n  }\n\n  private isFunctionCallItem(item: ResponseItem): item is FunctionCallItem {\n    return (\n      item.type === \"function_call\" &&\n      \"call_id\" in item &&\n      \"name\" in item &&\n      \"arguments\" in item\n    );\n  }\n\n  private createInitialInputItems(instruction: string): ResponseInputItem[] {\n    // For the initial request, we use a simple array with the user's instruction\n    return [\n      {\n        role: \"system\",\n        content: this.userProvidedInstructions,\n      },\n      {\n        role: \"user\",\n        content: instruction,\n      },\n    ];\n  }\n\n  async getAction(\n    inputItems: ResponseInputItem[],\n    previousResponseId?: string,\n  ): Promise<{\n    output: ResponseItem[];\n    responseId: string;\n    usage: Record<string, number>;\n  }> {\n    try {\n      // Create the request parameters\n      const requestParams: Record<string, unknown> = {\n        model: this.modelName,\n        tools: [\n          {\n            type: \"computer_use_preview\",\n            display_width: this.currentViewport.width,\n            display_height: this.currentViewport.height,\n            environment: this.environment,\n          },\n        ],\n        input: inputItems,\n        truncation: \"auto\",\n      };\n\n      // Add custom tools if available\n      if (this.tools && Object.keys(this.tools).length > 0) {\n        const customTools = Object.entries(this.tools).map(([name, tool]) => ({\n          type: \"function\" as const,\n          name,\n          function: {\n            name,\n            description: tool.description,\n            parameters: tool.inputSchema,\n          },\n        }));\n\n        requestParams.tools = [\n          ...(requestParams.tools as Record<string, unknown>[]),\n          ...customTools,\n        ];\n      }\n\n      // Add previous_response_id if available\n      if (previousResponseId) {\n        requestParams.previous_response_id = previousResponseId;\n      }\n\n      // Log LLM request\n      const llmRequestId = uuidv7();\n      SessionFileLogger.logLlmRequest({\n        requestId: llmRequestId,\n        model: this.modelName,\n        operation: \"CUA.getAction\",\n        prompt: formatCuaPromptPreview(inputItems),\n      });\n\n      const startTime = Date.now();\n      // Create the response using the OpenAI Responses API\n      // @ts-expect-error - Force type to match what the OpenAI SDK expects\n      const response = await this.client.responses.create(requestParams);\n      const endTime = Date.now();\n      const elapsedMs = endTime - startTime;\n\n      // Extract only the input_tokens and output_tokens\n      const usage = {\n        input_tokens: response.usage.input_tokens,\n        output_tokens: response.usage.output_tokens,\n        inference_time_ms: elapsedMs,\n      };\n\n      // Log LLM response\n      SessionFileLogger.logLlmResponse({\n        requestId: llmRequestId,\n        model: this.modelName,\n        operation: \"CUA.getAction\",\n        output: formatCuaResponsePreview(response.output),\n        inputTokens: response.usage.input_tokens,\n        outputTokens: response.usage.output_tokens,\n      });\n\n      // Store the response ID for future use\n      this.lastResponseId = response.id;\n\n      // Return the output and response ID\n      return {\n        output: response.output as unknown as ResponseItem[],\n        responseId: response.id,\n        usage,\n      };\n    } catch (error) {\n      console.error(\"Error getting action from OpenAI:\", error);\n      throw error;\n    }\n  }\n\n  async takeAction(\n    output: ResponseItem[],\n    logger: (message: LogLine) => void,\n  ): Promise<ResponseInputItem[]> {\n    const nextInputItems: ResponseInputItem[] = [];\n\n    // Process each output item\n    for (const item of output) {\n      if (item.type === \"computer_call\" && this.isComputerCallItem(item)) {\n        // Handle computer calls\n        try {\n          const action = this.convertComputerCallToAction(item);\n\n          if (action && this.actionHandler) {\n            logger({\n              category: \"agent\",\n              message: `Executing computer action: ${action.type}`,\n              level: 1,\n            });\n            await this.actionHandler(action);\n          }\n\n          // Capture a screenshot\n          const screenshot = await this.captureScreenshot();\n\n          // Create a computer_call_output for the next request\n          const outputItem = {\n            type: \"computer_call_output\" as const,\n            call_id: item.call_id,\n            output: {\n              type: \"input_image\" as const,\n              image_url: screenshot,\n            },\n          } as ResponseInputItem;\n\n          logger({\n            category: \"agent\",\n            message: `Added computer_call_output for call_id: ${item.call_id}`,\n            level: 2,\n          });\n\n          // Add current URL if available\n          if (this.currentUrl) {\n            const computerCallOutput = outputItem as {\n              type: \"computer_call_output\";\n              call_id: string;\n              output: {\n                type: \"input_image\";\n                image_url: string;\n                current_url?: string;\n              };\n              acknowledged_safety_checks?: SafetyCheck[];\n            };\n            computerCallOutput.output.current_url = this.currentUrl;\n          }\n\n          if (\n            item.pending_safety_checks &&\n            item.pending_safety_checks.length > 0\n          ) {\n            const acknowledgedChecks = await this.handleSafetyConfirmation(\n              item.pending_safety_checks,\n              logger,\n            );\n\n            if (acknowledgedChecks) {\n              const computerCallOutput = outputItem as {\n                type: \"computer_call_output\";\n                call_id: string;\n                output: {\n                  type: \"input_image\";\n                  image_url: string;\n                };\n                acknowledged_safety_checks?: SafetyCheck[];\n              };\n              computerCallOutput.acknowledged_safety_checks =\n                acknowledgedChecks;\n            }\n          }\n\n          nextInputItems.push(outputItem);\n        } catch (error) {\n          if (error instanceof StagehandClosedError) {\n            throw error;\n          }\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n\n          logger({\n            category: \"agent\",\n            message: `Error executing computer call: ${errorMessage}`,\n            level: 0,\n          });\n\n          try {\n            // Capture a screenshot even on error\n            const screenshot = await this.captureScreenshot();\n\n            const errorOutputItem = {\n              type: \"computer_call_output\" as const,\n              call_id: item.call_id,\n              output: {\n                type: \"input_image\" as const,\n                image_url: screenshot,\n                error: errorMessage,\n              },\n            } as ResponseInputItem;\n\n            // Add current URL if available\n            if (this.currentUrl) {\n              const computerCallOutput = errorOutputItem as {\n                type: \"computer_call_output\";\n                call_id: string;\n                output: {\n                  type: \"input_image\";\n                  image_url: string;\n                  current_url?: string;\n                };\n                acknowledged_safety_checks?: SafetyCheck[];\n              };\n              computerCallOutput.output.current_url = this.currentUrl;\n            }\n\n            if (\n              item.pending_safety_checks &&\n              item.pending_safety_checks.length > 0\n            ) {\n              const acknowledgedChecks = await this.handleSafetyConfirmation(\n                item.pending_safety_checks,\n                logger,\n              );\n\n              if (acknowledgedChecks) {\n                const computerCallOutput = errorOutputItem as {\n                  type: \"computer_call_output\";\n                  call_id: string;\n                  output: {\n                    type: \"input_image\";\n                    image_url: string;\n                  };\n                  acknowledged_safety_checks?: SafetyCheck[];\n                };\n                computerCallOutput.acknowledged_safety_checks =\n                  acknowledgedChecks;\n              }\n            }\n\n            nextInputItems.push(errorOutputItem);\n          } catch (screenshotError) {\n            if (screenshotError instanceof StagehandClosedError) {\n              throw screenshotError;\n            }\n            // If we can't capture a screenshot, just send the error\n            logger({\n              category: \"agent\",\n              message: `Error capturing screenshot: ${String(screenshotError)}`,\n              level: 0,\n            });\n\n            // For error cases without a screenshot, we need to use a string output\n            nextInputItems.push({\n              type: \"computer_call_output\",\n              call_id: item.call_id,\n              output: `Error: ${errorMessage}`,\n            } as ResponseInputItem);\n          }\n        }\n      } else if (\n        item.type === \"function_call\" &&\n        this.isFunctionCallItem(item)\n      ) {\n        // Handle function calls (tool calls)\n        try {\n          const action = this.convertFunctionCallToAction(item);\n\n          if (action && this.actionHandler) {\n            await this.actionHandler(action);\n          }\n\n          // Execute the tool if available\n          let toolResult = \"Tool executed successfully\";\n          if (this.tools && item.name in this.tools) {\n            try {\n              const tool = this.tools[item.name];\n              const args = JSON.parse(item.arguments);\n\n              logger({\n                category: \"agent\",\n                message: `Executing tool call: ${item.name} with args: ${item.arguments}`,\n                level: 1,\n              });\n\n              const result = await tool.execute(args, {\n                toolCallId: item.call_id,\n                messages: [],\n              });\n              toolResult = JSON.stringify(result);\n\n              logger({\n                category: \"agent\",\n                message: `Tool ${item.name} completed successfully. Result: ${toolResult}`,\n                level: 1,\n              });\n            } catch (toolError) {\n              const errorMessage =\n                toolError instanceof Error\n                  ? toolError.message\n                  : String(toolError);\n              toolResult = `Error executing tool: ${errorMessage}`;\n\n              logger({\n                category: \"agent\",\n                message: `Error executing tool ${item.name}: ${errorMessage}`,\n                level: 0,\n              });\n            }\n          }\n\n          // Create a function_call_output for the next request\n          const outputItem: ResponseInputItem = {\n            type: \"function_call_output\",\n            call_id: item.call_id,\n            output: toolResult,\n          };\n\n          nextInputItems.push(outputItem);\n        } catch (error) {\n          if (error instanceof StagehandClosedError) {\n            throw error;\n          }\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n\n          logger({\n            category: \"agent\",\n            message: `Error executing function call: ${errorMessage}`,\n            level: 0,\n          });\n\n          // Send error result back\n          const errorOutputItem: ResponseInputItem = {\n            type: \"function_call_output\",\n            call_id: item.call_id,\n            output: `Error: ${errorMessage}`,\n          };\n\n          nextInputItems.push(errorOutputItem);\n        }\n      }\n    }\n\n    return nextInputItems;\n  }\n\n  private convertComputerCallToAction(\n    call: ComputerCallItem,\n  ): AgentAction | null {\n    const { action } = call;\n\n    // Instead of wrapping the action in a params object, spread the action properties directly\n    // This ensures properties like x, y, button, etc. are directly accessible on the AgentAction\n    return {\n      type: action.type as string,\n      ...action, // Spread all properties from the action\n    };\n  }\n\n  private convertFunctionCallToAction(\n    call: FunctionCallItem,\n  ): AgentAction | null {\n    try {\n      const args = JSON.parse(call.arguments);\n\n      return {\n        type: call.name,\n        params: args,\n      };\n    } catch (error) {\n      console.error(\"Error parsing function call arguments:\", error);\n      return null;\n    }\n  }\n\n  async captureScreenshot(options?: {\n    base64Image?: string;\n    currentUrl?: string;\n  }): Promise<string> {\n    // Use provided options if available\n    if (options?.base64Image) {\n      return `data:image/png;base64,${options.base64Image}`;\n    }\n\n    // Use the screenshot provider if available\n    if (this.screenshotProvider) {\n      try {\n        const base64Image = await this.screenshotProvider();\n        return `data:image/png;base64,${base64Image}`;\n      } catch (error) {\n        console.error(\"Error capturing screenshot:\", error);\n        throw error;\n      }\n    }\n\n    throw new AgentScreenshotProviderError(\n      \"`screenshotProvider` has not been set. \" +\n        \"Please call `setScreenshotProvider()` with a valid function that returns a base64-encoded image\",\n    );\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.d.ts

@@ -0,0 +1,12 @@
+import type { AgentToolMode } from "../../types/public/agent.js";
+export interface AgentSystemPromptOptions {
+    url: string;
+    executionInstruction: string;
+    mode: AgentToolMode;
+    systemInstructions?: string;
+    /** Whether running on Browserbase (enables captcha solver messaging) */
+    isBrowserbase?: boolean;
+    /** Tools to exclude from the system prompt */
+    excludeTools?: string[];
+}
+export declare function buildAgentSystemPrompt(options: AgentSystemPromptOptions): string;

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js

@@ -0,0 +1,186 @@
+function buildToolsSection(isHybridMode, hasSearch, excludeTools) {
+    const excludeSet = new Set(excludeTools ?? []);
+    const hybridTools = [
+        {
+            name: "screenshot",
+            description: "Take a compressed JPEG screenshot for quick visual context",
+        },
+        {
+            name: "ariaTree",
+            description: "Get an accessibility (ARIA) hybrid tree for full page context",
+        },
+        {
+            name: "click",
+            description: "Click on an element (PREFERRED - more reliable when element is visible in viewport)",
+        },
+        {
+            name: "type",
+            description: "Type text into an element (PREFERRED - more reliable when element is visible in viewport)",
+        },
+        {
+            name: "act",
+            description: "Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.",
+        },
+        { name: "dragAndDrop", description: "Drag and drop an element" },
+        { name: "clickAndHold", description: "Click and hold on an element" },
+        { name: "keys", description: "Press a keyboard key" },
+        {
+            name: "fillFormVision",
+            description: "Fill out a form using coordinates",
+        },
+        { name: "think", description: "Think about the task" },
+        { name: "extract", description: "Extract structured data" },
+        { name: "goto", description: "Navigate to a URL" },
+        { name: "wait", description: "Wait for a specified time" },
+        { name: "navback", description: "Navigate back in browser history" },
+        { name: "scroll", description: "Scroll the page x pixels up or down" },
+    ];
+    const domTools = [
+        {
+            name: "screenshot",
+            description: "Take a compressed JPEG screenshot for quick visual context",
+        },
+        {
+            name: "ariaTree",
+            description: "Get an accessibility (ARIA) hybrid tree for full page context",
+        },
+        {
+            name: "act",
+            description: "Perform a specific atomic action (click, type)",
+        },
+        { name: "keys", description: "Press a keyboard key" },
+        { name: "fillForm", description: "Fill out a form" },
+        { name: "think", description: "Think about the task" },
+        { name: "extract", description: "Extract structured data" },
+        { name: "goto", description: "Navigate to a URL" },
+        { name: "wait", description: "Wait for a specified time" },
+        { name: "navback", description: "Navigate back in browser history" },
+        { name: "scroll", description: "Scroll the page x pixels up or down" },
+    ];
+    const baseTools = isHybridMode ? hybridTools : domTools;
+    if (hasSearch) {
+        baseTools.push({
+            name: "search",
+            description: "Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.",
+        });
+    }
+    const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));
+    const toolLines = filteredTools
+        .map((tool) => `    <tool name="${tool.name}">${tool.description}</tool>`)
+        .join("\n");
+    return `<tools>\n${toolLines}\n  </tools>`;
+}
+export function buildAgentSystemPrompt(options) {
+    const { url, executionInstruction, mode, systemInstructions, isBrowserbase = false, excludeTools, } = options;
+    const localeDate = new Date().toLocaleDateString();
+    const isoDate = new Date().toISOString();
+    const cdata = (text) => `<![CDATA[${text}]]>`;
+    const isHybridMode = mode === "hybrid";
+    const hasSearch = Boolean(process.env.BRAVE_API_KEY);
+    // Tools section differs based on mode and excluded tools
+    const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);
+    // Strategy differs based on mode
+    const strategyItems = isHybridMode
+        ? [
+            `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,
+            `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,
+            `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,
+            `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,
+            `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,
+        ]
+        : [
+            `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,
+            `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,
+            `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,
+            `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,
+            `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,
+        ];
+    const strategySection = strategyItems.join("\n    ");
+    const commonStrategyItems = `
+    <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., "get job listings", "extract product details"). For reading page content or understanding elements, always use ${isHybridMode ? "screenshot or ariaTree" : "ariaTree"} instead - it's faster and more reliable.</item>
+    <item>Keep actions atomic and verify outcomes before proceeding.</item>
+    <item>For each action, provide clear reasoning about why you're taking that step.</item>
+    <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>
+    `;
+    // Page understanding protocol differs based on mode
+    const pageUnderstandingProtocol = isHybridMode
+        ? `<page_understanding_protocol>
+    <step_1>
+      <title>UNDERSTAND THE PAGE</title>
+      <primary_tool>
+        <name>screenshot</name>
+        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>
+        </primary_tool>
+      <secondary_tool>
+        <name>ariaTree</name>
+        <usage>Get complete page context before taking actions</usage>
+        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>
+      </secondary_tool>
+    </step_1>
+  </page_understanding_protocol>`
+        : `<page_understanding_protocol>
+    <step_1>
+      <title>UNDERSTAND THE PAGE</title>
+      <primary_tool>
+        <name>ariaTree</name>
+        <usage>Get complete page context before taking actions</usage>
+        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>
+        </primary_tool>
+      <secondary_tool>
+        <name>screenshot</name>
+        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>
+      </secondary_tool>
+    </step_1>
+  </page_understanding_protocol>`;
+    // Roadblocks section only shown when running on Browserbase (has captcha solver)
+    const roadblocksSection = isBrowserbase
+        ? `<roadblocks>
+    <note>captchas, popups, etc.</note>
+    <captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>
+  </roadblocks>`
+        : "";
+    // Build customInstructions block only if provided
+    const customInstructionsBlock = systemInstructions
+        ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\n  `
+        : "";
+    return `<system>
+  <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>
+  ${customInstructionsBlock}<task>
+    <goal>${cdata(executionInstruction)}</goal>
+    <date display="local" iso="${isoDate}">${localeDate}</date>
+    <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>
+  </task>
+  <page>
+    <startingUrl>you are starting your task on this url: ${url}</startingUrl>
+  </page>
+  <mindset>
+    <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>
+    <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>
+    <note>When the task is complete, do not seek more information; you have completed the task.</note>
+  </mindset>
+  <guidelines>
+    <item>Always start by understanding the current page state</item>
+    <item>Use the screenshot tool to verify page state when needed</item>
+    <item>Use appropriate tools for each action</item>
+  </guidelines>
+  ${pageUnderstandingProtocol}
+  <navigation>
+    <rule>If you are confident in the URL, navigate directly to it.</rule>
+    ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}
+  </navigation>
+  ${toolsSection}
+  <strategy>
+    ${strategySection}
+    ${commonStrategyItems}
+  </strategy>
+  ${roadblocksSection}
+  <completion>
+    <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>
+    <examples>
+      <example>If you were asked for specific flights, list the flights you found.</example>
+      <example>If you were asked for information about a product, list the product information you were asked for.</example>
+    </examples>
+  </completion>
+</system>`;
+}
+//# sourceMappingURL=agentSystemPrompt.js.map

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentSystemPrompt.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/prompts/agentSystemPrompt.ts"],"names":[],"mappings":"AAwBA,SAAS,iBAAiB,CACxB,YAAqB,EACrB,SAAkB,EAClB,YAAuB;IAEvB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,WAAW,GAAqB;QACpC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,OAAO;YACb,WAAW,EACT,qFAAqF;SACxF;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,2FAA2F;SAC9F;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EACT,0LAA0L;SAC7L;QACD,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE;QAChE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,8BAA8B,EAAE;QACrE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,mCAAmC;SACjD;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gDAAgD;SAC9D;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExD,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,8IAA8I;SACjJ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,MAAM,SAAS,GAAG,aAAa;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,SAAS,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,YAAY,SAAS,cAAc,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAiC;IAEjC,MAAM,EACJ,GAAG,EACH,oBAAoB,EACpB,IAAI,EACJ,kBAAkB,EAClB,aAAa,GAAG,KAAK,EACrB,YAAY,GACb,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,IAAI,KAAK,CAAC;IAEtD,MAAM,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC;IACvC,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAErD,yDAAyD;IACzD,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9E,iCAAiC;IACjC,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC;YACE,uIAAuI;YACvI,4GAA4G;YAC5G,sIAAsI;YACtI,uHAAuH;YACvH,sFAAsF;SACvF;QACH,CAAC,CAAC;YACE,2FAA2F;YAC3F,oJAAoJ;YACpJ,6HAA6H;YAC7H,mIAAmI;YACnI,wHAAwH;SACzH,CAAC;IAEN,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,mBAAmB,GAAG;sNACwL,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU;;;;KAIrQ,CAAC;IAEJ,oDAAoD;IACpD,MAAM,yBAAyB,GAAG,YAAY;QAC5C,CAAC,CAAC;;;;;;;;;;;;;iCAa2B;QAC7B,CAAC,CAAC;;;;;;;;;;;;;iCAa2B,CAAC;IAEhC,iFAAiF;IACjF,MAAM,iBAAiB,GAAG,aAAa;QACrC,CAAC,CAAC;;;gBAGU;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,kBAAkB;QAChD,CAAC,CAAC,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,2BAA2B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;IAEL,uBAAuB;YACf,KAAK,CAAC,oBAAoB,CAAC;iCACN,OAAO,KAAK,UAAU;;;;2DAII,GAAG;;;;;;;;;;;;IAY1D,yBAAyB;;;MAGvB,SAAS,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE;;IAEtG,YAAY;;MAEV,eAAe;MACf,mBAAmB;;IAErB,iBAAiB;;;;;;;;UAQX,CAAC;AACX,CAAC","sourcesContent":["import type { AgentToolMode } from \"../../types/public/agent.js\";\n\nexport interface AgentSystemPromptOptions {\n  url: string;\n  executionInstruction: string;\n  mode: AgentToolMode;\n  systemInstructions?: string;\n  /** Whether running on Browserbase (enables captcha solver messaging) */\n  isBrowserbase?: boolean;\n  /** Tools to exclude from the system prompt */\n  excludeTools?: string[];\n}\n\n/**\n * Builds the system prompt for the agent based on the tool mode.\n *\n * @param options - The prompt configuration options\n * @returns The formatted system prompt string\n */\ninterface ToolDefinition {\n  name: string;\n  description: string;\n}\n\nfunction buildToolsSection(\n  isHybridMode: boolean,\n  hasSearch: boolean,\n  excludeTools?: string[],\n): string {\n  const excludeSet = new Set(excludeTools ?? []);\n\n  const hybridTools: ToolDefinition[] = [\n    {\n      name: \"screenshot\",\n      description: \"Take a compressed JPEG screenshot for quick visual context\",\n    },\n    {\n      name: \"ariaTree\",\n      description:\n        \"Get an accessibility (ARIA) hybrid tree for full page context\",\n    },\n    {\n      name: \"click\",\n      description:\n        \"Click on an element (PREFERRED - more reliable when element is visible in viewport)\",\n    },\n    {\n      name: \"type\",\n      description:\n        \"Type text into an element (PREFERRED - more reliable when element is visible in viewport)\",\n    },\n    {\n      name: \"act\",\n      description:\n        \"Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.\",\n    },\n    { name: \"dragAndDrop\", description: \"Drag and drop an element\" },\n    { name: \"clickAndHold\", description: \"Click and hold on an element\" },\n    { name: \"keys\", description: \"Press a keyboard key\" },\n    {\n      name: \"fillFormVision\",\n      description: \"Fill out a form using coordinates\",\n    },\n    { name: \"think\", description: \"Think about the task\" },\n    { name: \"extract\", description: \"Extract structured data\" },\n    { name: \"goto\", description: \"Navigate to a URL\" },\n    { name: \"wait\", description: \"Wait for a specified time\" },\n    { name: \"navback\", description: \"Navigate back in browser history\" },\n    { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n  ];\n\n  const domTools: ToolDefinition[] = [\n    {\n      name: \"screenshot\",\n      description: \"Take a compressed JPEG screenshot for quick visual context\",\n    },\n    {\n      name: \"ariaTree\",\n      description:\n        \"Get an accessibility (ARIA) hybrid tree for full page context\",\n    },\n    {\n      name: \"act\",\n      description: \"Perform a specific atomic action (click, type)\",\n    },\n    { name: \"keys\", description: \"Press a keyboard key\" },\n    { name: \"fillForm\", description: \"Fill out a form\" },\n    { name: \"think\", description: \"Think about the task\" },\n    { name: \"extract\", description: \"Extract structured data\" },\n    { name: \"goto\", description: \"Navigate to a URL\" },\n    { name: \"wait\", description: \"Wait for a specified time\" },\n    { name: \"navback\", description: \"Navigate back in browser history\" },\n    { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n  ];\n\n  const baseTools = isHybridMode ? hybridTools : domTools;\n\n  if (hasSearch) {\n    baseTools.push({\n      name: \"search\",\n      description:\n        \"Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.\",\n    });\n  }\n\n  const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));\n\n  const toolLines = filteredTools\n    .map((tool) => `    <tool name=\"${tool.name}\">${tool.description}</tool>`)\n    .join(\"\\n\");\n\n  return `<tools>\\n${toolLines}\\n  </tools>`;\n}\n\nexport function buildAgentSystemPrompt(\n  options: AgentSystemPromptOptions,\n): string {\n  const {\n    url,\n    executionInstruction,\n    mode,\n    systemInstructions,\n    isBrowserbase = false,\n    excludeTools,\n  } = options;\n  const localeDate = new Date().toLocaleDateString();\n  const isoDate = new Date().toISOString();\n  const cdata = (text: string) => `<![CDATA[${text}]]>`;\n\n  const isHybridMode = mode === \"hybrid\";\n  const hasSearch = Boolean(process.env.BRAVE_API_KEY);\n\n  // Tools section differs based on mode and excluded tools\n  const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);\n\n  // Strategy differs based on mode\n  const strategyItems = isHybridMode\n    ? [\n        `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,\n        `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,\n        `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,\n        `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,\n        `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,\n      ]\n    : [\n        `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,\n        `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,\n        `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,\n        `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,\n        `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,\n      ];\n\n  const strategySection = strategyItems.join(\"\\n    \");\n\n  const commonStrategyItems = `\n    <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., \"get job listings\", \"extract product details\"). For reading page content or understanding elements, always use ${isHybridMode ? \"screenshot or ariaTree\" : \"ariaTree\"} instead - it's faster and more reliable.</item>\n    <item>Keep actions atomic and verify outcomes before proceeding.</item>\n    <item>For each action, provide clear reasoning about why you're taking that step.</item>\n    <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>\n    `;\n\n  // Page understanding protocol differs based on mode\n  const pageUnderstandingProtocol = isHybridMode\n    ? `<page_understanding_protocol>\n    <step_1>\n      <title>UNDERSTAND THE PAGE</title>\n      <primary_tool>\n        <name>screenshot</name>\n        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n        </primary_tool>\n      <secondary_tool>\n        <name>ariaTree</name>\n        <usage>Get complete page context before taking actions</usage>\n        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n      </secondary_tool>\n    </step_1>\n  </page_understanding_protocol>`\n    : `<page_understanding_protocol>\n    <step_1>\n      <title>UNDERSTAND THE PAGE</title>\n      <primary_tool>\n        <name>ariaTree</name>\n        <usage>Get complete page context before taking actions</usage>\n        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n        </primary_tool>\n      <secondary_tool>\n        <name>screenshot</name>\n        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n      </secondary_tool>\n    </step_1>\n  </page_understanding_protocol>`;\n\n  // Roadblocks section only shown when running on Browserbase (has captcha solver)\n  const roadblocksSection = isBrowserbase\n    ? `<roadblocks>\n    <note>captchas, popups, etc.</note>\n    <captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>\n  </roadblocks>`\n    : \"\";\n\n  // Build customInstructions block only if provided\n  const customInstructionsBlock = systemInstructions\n    ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\\n  `\n    : \"\";\n\n  return `<system>\n  <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>\n  ${customInstructionsBlock}<task>\n    <goal>${cdata(executionInstruction)}</goal>\n    <date display=\"local\" iso=\"${isoDate}\">${localeDate}</date>\n    <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>\n  </task>\n  <page>\n    <startingUrl>you are starting your task on this url: ${url}</startingUrl>\n  </page>\n  <mindset>\n    <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>\n    <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>\n    <note>When the task is complete, do not seek more information; you have completed the task.</note>\n  </mindset>\n  <guidelines>\n    <item>Always start by understanding the current page state</item>\n    <item>Use the screenshot tool to verify page state when needed</item>\n    <item>Use appropriate tools for each action</item>\n  </guidelines>\n  ${pageUnderstandingProtocol}\n  <navigation>\n    <rule>If you are confident in the URL, navigate directly to it.</rule>\n    ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}\n  </navigation>\n  ${toolsSection}\n  <strategy>\n    ${strategySection}\n    ${commonStrategyItems}\n  </strategy>\n  ${roadblocksSection}\n  <completion>\n    <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>\n    <examples>\n      <example>If you were asked for specific flights, list the flights you found.</example>\n      <example>If you were asked for information about a product, list the product information you were asked for.</example>\n    </examples>\n  </completion>\n</system>`;\n}\n"]}

package/dist/esm/lib/v3/agent/tools/act.d.ts

@@ -0,0 +1,12 @@
+import type { V3 } from "../../v3.js";
+import type { Action } from "../../types/public/methods.js";
+export declare const actTool: (v3: V3, executionModel?: string) => import("ai").Tool<{
+    action: string;
+}, {
+    success: boolean;
+    action: string;
+    playwrightArguments?: Action;
+} | {
+    success: boolean;
+    error: any;
+}>;

package/dist/esm/lib/v3/agent/tools/act.js

@@ -0,0 +1,49 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const actTool = (v3, executionModel) => tool({
+    description: "Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.",
+    inputSchema: z.object({
+        action: z
+            .string()
+            .describe('Describe what to click or type, e.g. "click the Login button" or "type "John" into the first name input"'),
+    }),
+    execute: async ({ action }) => {
+        try {
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: act`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: action,
+                        type: "string",
+                    },
+                },
+            });
+            const options = executionModel ? { model: executionModel } : undefined;
+            const result = await v3.act(action, options);
+            const actions = result.actions ?? [];
+            v3.recordAgentReplayStep({
+                type: "act",
+                instruction: action,
+                actions,
+                actionDescription: result.actionDescription,
+                message: result.message,
+            });
+            // Only include playwrightArguments when actions exist
+            // (undefined is not valid JSON and breaks AI SDK validation)
+            const response = {
+                success: result.success ?? true,
+                action: result?.actionDescription ?? action,
+            };
+            if (actions.length > 0) {
+                response.playwrightArguments = actions[0];
+            }
+            return response;
+        }
+        catch (error) {
+            return { success: false, error: error?.message ?? String(error) };
+        }
+    },
+});
+//# sourceMappingURL=act.js.map

package/dist/esm/lib/v3/agent/tools/act.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"act.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/act.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAM,EAAE,cAAuB,EAAE,EAAE,CACzD,IAAI,CAAC;IACH,WAAW,EACT,+GAA+G;IACjH,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC;aACN,MAAM,EAAE;aACR,QAAQ,CACP,0GAA0G,CAC3G;KACJ,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;QAC5B,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,yBAAyB;gBAClC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,MAAM;wBACb,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACvE,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAI,MAAM,CAAC,OAAgC,IAAI,EAAE,CAAC;YAC/D,EAAE,CAAC,qBAAqB,CAAC;gBACvB,IAAI,EAAE,KAAK;gBACX,WAAW,EAAE,MAAM;gBACnB,OAAO;gBACP,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC,CAAC;YACH,sDAAsD;YACtD,6DAA6D;YAC7D,MAAM,QAAQ,GAIV;gBACF,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;gBAC/B,MAAM,EAAE,MAAM,EAAE,iBAAiB,IAAI,MAAM;aAC5C,CAAC;YACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\n\nexport const actTool = (v3: V3, executionModel?: string) =>\n  tool({\n    description:\n      \"Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.\",\n    inputSchema: z.object({\n      action: z\n        .string()\n        .describe(\n          'Describe what to click or type, e.g. \"click the Login button\" or \"type \"John\" into the first name input\"',\n        ),\n    }),\n    execute: async ({ action }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: act`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: action,\n              type: \"string\",\n            },\n          },\n        });\n        const options = executionModel ? { model: executionModel } : undefined;\n        const result = await v3.act(action, options);\n        const actions = (result.actions as Action[] | undefined) ?? [];\n        v3.recordAgentReplayStep({\n          type: \"act\",\n          instruction: action,\n          actions,\n          actionDescription: result.actionDescription,\n          message: result.message,\n        });\n        // Only include playwrightArguments when actions exist\n        // (undefined is not valid JSON and breaks AI SDK validation)\n        const response: {\n          success: boolean;\n          action: string;\n          playwrightArguments?: Action;\n        } = {\n          success: result.success ?? true,\n          action: result?.actionDescription ?? action,\n        };\n        if (actions.length > 0) {\n          response.playwrightArguments = actions[0];\n        }\n        return response;\n      } catch (error) {\n        return { success: false, error: error?.message ?? String(error) };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/ariaTree.d.ts

@@ -0,0 +1,5 @@
+import type { V3 } from "../../v3.js";
+export declare const ariaTreeTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+    content: string;
+    pageUrl: string;
+}>;

package/dist/esm/lib/v3/agent/tools/ariaTree.js

@@ -0,0 +1,31 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const ariaTreeTool = (v3) => tool({
+    description: "gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.",
+    inputSchema: z.object({}),
+    execute: async () => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: ariaTree`,
+            level: 1,
+        });
+        const page = await v3.context.awaitActivePage();
+        const { pageText } = (await v3.extract());
+        const pageUrl = page.url();
+        let content = pageText;
+        const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation
+        const estimatedTokens = Math.ceil(content.length / 4);
+        if (estimatedTokens > MAX_TOKENS) {
+            const maxChars = MAX_TOKENS * 4;
+            content =
+                content.substring(0, maxChars) +
+                    "\n\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]";
+        }
+        return { content, pageUrl };
+    },
+    toModelOutput: (result) => ({
+        type: "content",
+        value: [{ type: "text", text: `Accessibility Tree:\n${result.content}` }],
+    }),
+});
+//# sourceMappingURL=ariaTree.js.map

package/dist/esm/lib/v3/agent/tools/ariaTree.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ariaTree.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/ariaTree.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAM,EAAE,EAAE,CACrC,IAAI,CAAC;IACH,WAAW,EACT,oHAAoH;IACtH,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,8BAA8B;YACvC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,CAAyB,CAAC;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,IAAI,OAAO,GAAG,QAAQ,CAAC;QACvB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,mEAAmE;QAC7F,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtD,IAAI,eAAe,GAAG,UAAU,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,CAAC;YAChC,OAAO;gBACL,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC;oBAC9B,sDAAsD,CAAC;QAC3D,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;KAC1E,CAAC;CACH,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const ariaTreeTool = (v3: V3) =>\n  tool({\n    description:\n      \"gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: ariaTree`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      const { pageText } = (await v3.extract()) as { pageText: string };\n      const pageUrl = page.url();\n\n      let content = pageText;\n      const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation\n      const estimatedTokens = Math.ceil(content.length / 4);\n      if (estimatedTokens > MAX_TOKENS) {\n        const maxChars = MAX_TOKENS * 4;\n        content =\n          content.substring(0, maxChars) +\n          \"\\n\\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]\";\n      }\n\n      return { content, pageUrl };\n    },\n    toModelOutput: (result) => ({\n      type: \"content\",\n      value: [{ type: \"text\", text: `Accessibility Tree:\\n${result.content}` }],\n    }),\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/click.d.ts

@@ -0,0 +1,6 @@
+import type { V3 } from "../../v3.js";
+import type { ClickToolResult } from "../../types/public/agent.js";
+export declare const clickTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    describe: string;
+    coordinates: number[];
+}, ClickToolResult>;

package/dist/esm/lib/v3/agent/tools/click.js

@@ -0,0 +1,104 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const clickTool = (v3, provider) => tool({
+    description: "Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
+    inputSchema: z.object({
+        describe: z
+            .string()
+            .describe("Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description"),
+        coordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to click on"),
+    }),
+    execute: async ({ describe, coordinates }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: click`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({ describe }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            const xpath = await page.click(processed.x, processed.y, {
+                returnXpath: shouldCollectXpath,
+            });
+            const screenshotBase64 = await waitAndCaptureScreenshot(page);
+            // Record as an "act" step with proper Action for deterministic replay (only when caching)
+            if (shouldCollectXpath) {
+                const normalizedXpath = ensureXPath(xpath);
+                if (normalizedXpath) {
+                    const action = {
+                        selector: normalizedXpath,
+                        description: describe,
+                        method: "click",
+                        arguments: [],
+                    };
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: describe,
+                        actions: [action],
+                        actionDescription: describe,
+                    });
+                }
+            }
+            return {
+                success: true,
+                describe,
+                coordinates: [processed.x, processed.y],
+                screenshotBase64,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error clicking: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success) {
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        describe: result.describe,
+                        coordinates: result.coordinates,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
+        }
+        return {
+            type: "content",
+            value: [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        error: result.error,
+                    }),
+                },
+            ],
+        };
+    },
+});
+//# sourceMappingURL=click.js.map

package/dist/esm/lib/v3/agent/tools/click.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;qBAChC,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n              coordinates: result.coordinates,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/clickAndHold.d.ts

@@ -0,0 +1,14 @@
+import type { V3 } from "../../v3.js";
+export declare const clickAndHoldTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    describe: string;
+    duration: number;
+    coordinates: number[];
+}, {
+    success: boolean;
+    describe: string;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    describe?: undefined;
+}>;