npm - @browserbasehq/orca - Versions diffs - 3.1.0-patch.3 → 3.2.0-preview.1 - Mend

@browserbasehq/orca 3.1.0-patch.3 → 3.2.0-preview.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1460) hide show

package/dist/esm/lib/v3/agent/MicrosoftCUAClient.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"MicrosoftCUAClient.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/MicrosoftCUAClient.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,4BAA4B,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AA+B9D;;;GAGG;AACH,MAAM,OAAO,kBAAmB,SAAQ,WAAW;IACzC,MAAM,CAAS;IACf,OAAO,CAAS;IAChB,MAAM,CAAS;IACf,eAAe,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC/C,UAAU,CAAU;IACpB,kBAAkB,CAAyB;IAC3C,aAAa,CAA0C;IAE/D,sBAAsB;IACd,mBAAmB,GAAkB,EAAE,CAAC,CAAC,kBAAkB;IAC3D,aAAa,GAAkB,EAAE,CAAC,CAAC,sBAAsB;IAEzD,SAAS,GAAW,CAAC,CAAC;IACtB,WAAW,GAAW,CAAC,CAAC;IACxB,KAAK,GAAa,EAAE,CAAC;IAE7B,qCAAqC;IACpB,oBAAoB,GAAG;QACtC,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,QAAQ;QACpB,UAAU,EAAE,EAAE;QACd,UAAU,EAAE,CAAC;KACd,CAAC;IAEF,qCAAqC;IAC7B,eAAe,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAEvD,YACE,IAAe,EACf,SAAiB,EACjB,wBAAiC,EACjC,aAA6B;QAE7B,KAAK,CAAC,IAAI,EAAE,SAAS,IAAI,SAAS,EAAE,wBAAwB,CAAC,CAAC;QAE9D,yBAAyB;QACzB,IAAI,CAAC,MAAM;YACR,aAAa,EAAE,MAAiB;gBACjC,OAAO,CAAC,GAAG,CAAC,aAAa;gBACzB,OAAO,CAAC,GAAG,CAAC,iBAAiB;gBAC7B,EAAE,CAAC;QACL,IAAI,CAAC,OAAO;YACT,aAAa,EAAE,OAAkB;gBAClC,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,kBAAkB;gBAC9B,EAAE,CAAC;QAEL,qCAAqC;QACrC,IAAI,CAAC,aAAa,GAAG;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;QAEF,mBAAmB;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CACb,8HAA8H,CAC/H,CAAC;QACJ,CAAC;QAED,iEAAiE;QACjE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,gCAAgC;QAChC,IAAI,aAAa,EAAE,SAAS,KAAK,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,SAAmB,CAAC;QACrD,CAAC;QAED,cAAc;QACd,IAAI,aAAa,EAAE,WAAW,KAAK,SAAS,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,GAAG,aAAa,CAAC,WAAqB,CAAC;QACzD,CAAC;IACH,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,MAAc;QACvC,IAAI,CAAC,eAAe,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QACzC,oDAAoD;QACpD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACxB,CAAC;IAED,qBAAqB,CAAC,QAA+B;QACnD,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAED,gBAAgB,CAAC,OAA+C;QAC9D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACK,WAAW,CACjB,KAAa,EACb,MAAc;QAEd,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,GACtD,IAAI,CAAC,oBAAoB,CAAC;QAC5B,MAAM,MAAM,GAAG,UAAU,GAAG,UAAU,CAAC;QAEvC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1E,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACxE,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAE1E,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE3D,IAAI,KAAK,GAAG,KAAK,GAAG,UAAU,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,UAAU,CAAC,CAAC;YACtD,KAAK,GAAG,aAAa,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7C,KAAK,GAAG,aAAa,CAAC,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9C,CAAC;aAAM,IAAI,KAAK,GAAG,KAAK,GAAG,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;YACtD,KAAK,GAAG,YAAY,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;YAC5C,KAAK,GAAG,YAAY,CAAC,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;IAED;;;OAGG;IACK,oBAAoB;QAC1B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QAE/C,uCAAuC;QACvC,IAAI,UAAU,GAAG,8BAA8B,CAAC;QAEhD,8CAA8C;QAC9C,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAClC,UAAU,GAAG,GAAG,UAAU,OAAO,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnE,CAAC;QAED,wCAAwC;QACxC,MAAM,eAAe,GAAG;;;+BAGG,KAAK,IAAI,MAAM;;;;;;;uLAOyI,CAAC;QAEpL,8BAA8B;QAC9B,MAAM,kBAAkB,GAAG;;;;;;;;;;;8EAW+C,CAAC;QAE3E,mBAAmB;QACnB,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,cAAc;YACpB,WAAW,EAAE,eAAe;YAC5B,UAAU,EAAE;gBACV,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,UAAU,EAAE;oBACV,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kBAAkB;wBAC/B,IAAI,EAAE;4BACJ,KAAK;4BACL,MAAM;4BACN,YAAY;4BACZ,YAAY;4BACZ,QAAQ;4BACR,WAAW;4BACX,YAAY;4BACZ,cAAc;4BACd,yBAAyB;4BACzB,MAAM;4BACN,WAAW;yBACZ;qBACF;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,OAAO;wBACb,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,iCAAiC;qBAC/C;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,SAAS;wBACf,WAAW,EACT,8EAA8E;qBACjF;oBACD,oBAAoB,EAAE;wBACpB,IAAI,EAAE,SAAS;wBACf,WAAW,EACT,gFAAgF;qBACnF;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,OAAO;wBACb,WAAW,EACT,4LAA4L;qBAC/L;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,+HAA+H;qBAClI;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,yFAAyF;qBAC5F;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sDAAsD;qBACpE;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,8DAA8D;wBAChE,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;qBAC7B;iBACF;aACF;SACF,CAAC;QAEF,qEAAqE;QACrE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,oBAAoB,GAAG;;;EAG/B,SAAS;;;;;;aAME,CAAC;QAEV,OAAO,GAAG,UAAU,OAAO,oBAAoB,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,QAAgB;QAI7C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9D,IAAI,YAAY,CAAC;YACjB,IAAI,CAAC;gBACH,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,qEAAqE;gBACrE,0DAA0D;gBAC1D,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5D,iCAAiC;oBACjC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtC,IAAI,CAAC;wBACH,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBACvC,CAAC;oBAAC,OAAO,UAAU,EAAE,CAAC;wBACpB,MAAM,IAAI,KAAK,CACb,4EAA4E,UAAU,YAAY,SAAS,YAAY,UAAU,EAAE,CACpI,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,wCAAwC,UAAU,YAAY,SAAS,EAAE,CAC1E,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,OAAO;gBACL,QAAQ;gBACR,YAAY,EAAE;oBACZ,IAAI,EAAE,YAAY,CAAC,IAAI,IAAI,cAAc;oBACzC,SAAS,EAAE;wBACT,GAAG,YAAY,CAAC,SAAS;wBACzB,QAAQ;qBACT;iBACF;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,iDAAiD,QAAQ,YAAY,KAAK,EAAE,CAC7E,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,2BAA2B,CACjC,YAA8B;QAE9B,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAgB,CAAC;QAErC,0DAA0D;QAC1D,MAAM,mBAAmB,GAAG,CAAC,KAAe,EAAY,EAAE;YACxD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/C,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;YACrB,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YACvE,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC;QAEF,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,IAAI,CAAC,QAAkB;SACnC,CAAC;QAEF,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,UAAsB,CAAC,CAAC;gBACpE,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,OAAO;oBACb,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;oBAChB,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;oBAChB,MAAM,EAAE,MAAe;iBACxB,CAAC;YACJ,CAAC;YAED,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,UAAsB,CAAC,CAAC;gBACnE,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,SAAS;iBACtB,CAAC;YACJ,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU;oBAC/B,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAsB,CAAC;oBAClD,CAAC,CAAC,SAAS,CAAC;gBACd,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,IAAI,CAAC,IAAc;oBACzB,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtD,WAAW,EACT,IAAI,CAAC,WAAW,KAAK,SAAS;wBAC5B,CAAC,CAAE,IAAI,CAAC,WAAuB;wBAC/B,CAAC,CAAC,IAAI;oBACV,GAAG,CAAC,IAAI,CAAC,oBAAoB,KAAK,SAAS,IAAI;wBAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAA+B;qBAC3D,CAAC;iBACH,CAAC;YACJ,CAAC;YAED,KAAK,KAAK,CAAC;YACX,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,IAAI,GAAI,IAAI,CAAC,IAAiB,IAAI,EAAE,CAAC;gBAC3C,sCAAsC;gBACtC,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9D,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,cAAc;iBACrB,CAAC;YACJ,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,MAAM,GAAI,IAAI,CAAC,MAAiB,IAAI,CAAC,CAAC;gBAC5C,qDAAqD;gBACrD,+BAA+B;gBAC/B,OAAO;oBACL,GAAG,UAAU;oBACb,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,CAAC,MAAM,EAAE,wCAAwC;iBAC5D,CAAC;YACJ,CAAC;YAED,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,IAAI,GAAG,GAAG,IAAI,CAAC,GAAa,CAAC;gBAC7B,sCAAsC;gBACtC,IACE,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC;oBAC3B,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC1B,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC1B,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EACzB,CAAC;oBACD,+CAA+C;oBAC/C,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBACtB,GAAG,GAAG,iCAAiC,kBAAkB,CAAC,GAAG,CAAC,YAAY,CAAC;oBAC7E,CAAC;yBAAM,CAAC;wBACN,iCAAiC;wBACjC,GAAG,GAAG,UAAU,GAAG,GAAG,CAAC;oBACzB,CAAC;gBACH,CAAC;gBACD,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;oBACZ,GAAG;iBACJ,CAAC;YACJ,CAAC;YAED,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,mDAAmD;gBACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAe,CAAC;gBACnC,MAAM,SAAS,GAAG,iCAAiC,kBAAkB,CAAC,KAAK,CAAC,YAAY,CAAC;gBACzF,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;oBACZ,GAAG,EAAE,SAAS;iBACf,CAAC;YACJ,CAAC;YAED,KAAK,cAAc;gBACjB,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;iBACb,CAAC;YAEJ,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,8EAA8E;gBAC9E,MAAM,eAAe,GAClB,IAAI,CAAC,IAAe,IAAK,IAAI,CAAC,QAAmB,IAAI,GAAG,CAAC;gBAC5D,OAAO;oBACL,GAAG,UAAU;oBACb,MAAM,EAAE,eAAe,GAAG,IAAI,EAAE,wBAAwB;iBACzD,CAAC;YACJ,CAAC;YAED,KAAK,yBAAyB,CAAC,CAAC,CAAC;gBAC/B,uDAAuD;gBACvD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAc,CAAC;gBACjC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI;iBACL,CAAC;YACJ,CAAC;YAED,KAAK,WAAW;gBACd,OAAO;oBACL,GAAG,UAAU;oBACb,MAAM,EAAE,IAAI,CAAC,MAAgB;iBAC9B,CAAC;YAEJ;gBACE,OAAO;oBACL,GAAG,UAAU;oBACb,GAAG,IAAI;iBACR,CAAC;QACN,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,MAAM,IAAI,4BAA4B,CAAC,6BAA6B,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzD,OAAO,yBAAyB,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;;OAGG;IACK,yBAAyB,CAC/B,OAAsB,EACtB,kBAA2B,KAAK;QAEhC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACxE,MAAM,UAAU,GAAkB,EAAE,CAAC;QACrC,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,oBAAoB;QACpB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,6BAA6B;YAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YAC7D,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;gBACpC,qDAAqD;gBACrD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC/B,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;oBACrE,0FAA0F;oBAC1F,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,SAAS;oBACX,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;gBACnD,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,OAAO,GAAG,SAAS,EAAE,CAAC;oBACxB,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACrB,OAAO,EAAE,CAAC;gBACZ,CAAC;qBAAM,CAAC;oBACN,0BAA0B;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC/B,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAC9B,CAAC;wBACF,4FAA4F;wBAC5F,yCAAyC;wBACzC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC1B,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;wBACnD,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACvB,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACK,kBAAkB;QACxB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzD,MAAM,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC3B,IAAI,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;oBAC5C,wCAAwC;oBACxC,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;oBAC7C,UAAU,EAAE,CAAC;gBACf,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CACvB,MAAkC,EAClC,eAAwB,KAAK;QAU7B,qBAAqB;QACrB,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzD,0DAA0D;QAC1D,IAAI,YAAY,EAAE,CAAC;YACjB,wFAAwF;YACxF,MAAM,WAAW,GACf,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,WAAW,IAAI,WAAW,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC/C,MAAM,eAAe,GACnB,OAAO,WAAW,CAAC,OAAO,KAAK,QAAQ;oBACrC,CAAC,CAAC,WAAW,CAAC,OAAO;oBACrB,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI;wBACzD,YAAY,CAAC,CAAC;gBAEpB,WAAW,CAAC,OAAO,GAAG;oBACpB;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;qBACtC;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,eAAe;qBACtB;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0DAA0D;YAC1D,MAAM,WAAW,GAAyB;gBACxC;oBACE,IAAI,EAAE,WAAW;oBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;iBACtC;aACF,CAAC;YAEF,+BAA+B;YAC/B,IAAI,UAAU,GACZ,2DAA2D,CAAC;YAC9D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,MAAM,UAAU,GACd,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG;oBAC1B,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;oBACvC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACtB,UAAU,GAAG,gBAAgB,UAAU,KAAK,UAAU,EAAE,CAAC;YAC3D,CAAC;YAED,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YAEH,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;QACL,CAAC;QAED,qCAAqC;QACrC,IAAI,OAAO,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAExC,0CAA0C;QAC1C,MAAM,aAAa,GAAgB;YACjC,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,CAAC,oBAAoB,EAAE;SACrC,CAAC;QACF,OAAO,GAAG,CAAC,aAAa,EAAE,GAAG,OAAO,CAAC,CAAC;QAEtC,gBAAgB;QAChB,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,sCAAsC,OAAO,CAAC,MAAM,WAAW;YACxE,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,QAAQ,EAAE,OAAkD;gBAC5D,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,oBAAoB,QAAQ,YAAY,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;gBAC9F,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,QAAQ,CAAC;QACjB,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAE7C,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,yBAAyB,aAAa,IAAI;YACnD,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI;YAC9B,aAAa,EAAE,CAAC;YAChB,iBAAiB,EAAE,CAAC;YACpB,YAAY,EAAE,CAAC;SAChB,CAAC;QAEF,2CAA2C;QAC3C,MAAM,YAAY,GAAgB;YAChC,IAAI,EAAE,WAAW;YACjB,OAAO;SACR,CAAC;QACF,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAEtC,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,mBAAmB,OAAO,EAAE;YACrC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,kBAAkB;QAClB,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAExE,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,aAAa,QAAQ,EAAE;YAChC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,WAAW,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;YAC5D,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,2BAA2B,CAAC,YAAY,CAAC,CAAC;QAEnE,iEAAiE;QACjE,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,IACE,WAAW,CAAC,IAAI,KAAK,MAAM;YAC3B,OAAO,WAAW,CAAC,CAAC,KAAK,QAAQ;YACjC,OAAO,WAAW,CAAC,CAAC,KAAK,QAAQ,EACjC,CAAC;YACD,qDAAqD;YACrD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,CAAC,EAAE,WAAW,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;gBAChB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,yDAAyD;YACzD,IAAI,WAAW,CAAC,oBAAoB,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,CAAC,WAAW,CAAC;iBACpB,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,CAAC,WAAW,CAAC;iBACpB,CAAC,CAAC;YACL,CAAC;YAED,qEAAqE;YACrE,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,WAAW,CAAC,IAAI;aACvB,CAAC,CAAC;YAEH,6DAA6D;YAC7D,IAAI,WAAW,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,CAAC,OAAO,CAAC;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5B,CAAC;QAED,8CAA8C;QAC9C,IAAI,IAAI,CAAC,aAAa,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC,MAAM,KAAK,WAAW,CAAC;QAEhE,OAAO;YACL,OAAO;YACP,SAAS;YACT,KAAK,EAAE;gBACL,YAAY,EAAE,KAAK,CAAC,aAAa;gBACjC,aAAa,EAAE,KAAK,CAAC,iBAAiB;gBACtC,iBAAiB,EAAE,aAAa;aACjC;SACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,gBAAuC;QACnD,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC7C,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAExC,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,gDAAgD;QAChD,yEAAyE;QACzE,IAAI,CAAC,mBAAmB,GAAG;YACzB;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB;SACF,CAAC;QACF,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QAExB,IAAI,CAAC;YACH,sDAAsD;YACtD,OAAO,CAAC,SAAS,IAAI,WAAW,GAAG,QAAQ,EAAE,CAAC;gBAC5C,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,kBAAkB,WAAW,GAAG,CAAC,IAAI,QAAQ,EAAE;oBACxD,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBAEH,MAAM,YAAY,GAAG,WAAW,KAAK,CAAC,CAAC;gBACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;gBAC5D,gBAAgB,IAAI,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;gBAC9C,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC;gBAChD,kBAAkB,IAAI,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;gBAErD,0BAA0B;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;gBAEhC,2BAA2B;gBAC3B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAE7B,WAAW,EAAE,CAAC;gBAEd,+BAA+B;gBAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC7D,IAAI,UAAU,EAAE,SAAS,EAAE,CAAC;oBAC1B,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAED,yBAAyB;YACzB,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC/C,YAAY;oBACT,UAAkC,EAAE,MAAM,KAAK,SAAS;wBACvD,CAAC,CAAC,8BAA8B;wBAChC,CAAC,CAAC,+BAA+B,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,0BAA0B,QAAQ,uBAAuB,CAAC;YAC3E,CAAC;YAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,YAAY,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,YAAY,EAAE,CAAC;YAClE,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,SAAS;gBAClB,SAAS;gBACT,OAAO,EAAE,YAAY;gBACrB,OAAO;gBACP,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B,KAAK,EAAE;gBAC3C,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,wEAAwE;YACxE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF","sourcesContent":["import OpenAI from \"openai\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport {\n AgentAction,\n AgentResult,\n AgentType,\n AgentExecutionOptions,\n} from \"../types/public/agent.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AgentScreenshotProviderError } from \"../types/public/sdkErrors.js\";\nimport { mapKeyToPlaywright } from \"./utils/cuaKeyMapping.js\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\n\n/*\n Message types for FARA agent\n /\ninterface FaraMessage {\n role: \"system\" \| \"user\" \| \"assistant\";\n content: string \| FaraMessageContent[];\n}\n\ninterface FaraMessageContent {\n type: \"text\" \| \"image_url\";\n text?: string;\n image_url?: {\n url: string; // data:image/png;base64,...\n };\n}\n\n/\n FARA function call structure (parsed from XML tags)\n /\ninterface FaraFunctionCall {\n name: string; // Always \"computer_use\"\n arguments: {\n action: string;\n thoughts?: string;\n [key: string]: unknown;\n };\n}\n\n/\n Client for FARA (Function-based Autonomous Research Agent) by Microsoft\n * This implementation uses OpenAI-compatible API with XML-based tool calling\n /\nexport class MicrosoftCUAClient extends AgentClient {\n private apiKey: string;\n private baseURL: string;\n private client: OpenAI;\n private currentViewport = { width: 1288, height: 711 };\n private currentUrl?: string;\n private screenshotProvider?: () => Promise<string>;\n private actionHandler?: (action: AgentAction) => Promise<void>;\n\n // Dual history system\n private conversationHistory: FaraMessage[] = []; // Conceptual flow\n private actionHistory: FaraMessage[] = []; // Raw model responses\n\n private maxImages: number = 3;\n private temperature: number = 0;\n private facts: string[] = [];\n\n // FARA-specific MLM processor config\n private readonly MLM_PROCESSOR_IM_CFG = {\n min_pixels: 3136,\n max_pixels: 12845056,\n patch_size: 14,\n merge_size: 2,\n };\n\n // Resized dimensions for model input\n private resizedViewport = { width: 1288, height: 711 };\n\n constructor(\n type: AgentType,\n modelName: string,\n userProvidedInstructions?: string,\n clientOptions?: ClientOptions,\n ) {\n super(type, modelName \|\| \"fara-7b\", userProvidedInstructions);\n\n // Process client options\n this.apiKey =\n (clientOptions?.apiKey as string) \|\|\n process.env.AZURE_API_KEY \|\|\n process.env.FIREWORKS_API_KEY \|\|\n \"\";\n this.baseURL =\n (clientOptions?.baseURL as string) \|\|\n process.env.AZURE_ENDPOINT \|\|\n process.env.FIREWORKS_ENDPOINT \|\|\n \"\";\n\n // Store client options for reference\n this.clientOptions = {\n apiKey: this.apiKey,\n baseURL: this.baseURL,\n };\n\n // Validate API key\n if (!this.apiKey \|\| this.apiKey === \"\") {\n throw new Error(\n \"API key is required. Please provide it via clientOptions.apiKey or AZURE_API_KEY or FIREWORKS_API_KEY environment variables.\",\n );\n }\n\n // Initialize the OpenAI client (FARA uses OpenAI-compatible API)\n this.client = new OpenAI({\n apiKey: this.apiKey,\n baseURL: this.baseURL,\n });\n\n // Max images to keep in history\n if (clientOptions?.maxImages !== undefined) {\n this.maxImages = clientOptions.maxImages as number;\n }\n\n // Temperature\n if (clientOptions?.temperature !== undefined) {\n this.temperature = clientOptions.temperature as number;\n }\n }\n\n setViewport(width: number, height: number): void {\n this.currentViewport = { width, height };\n // Compute resized viewport using smart_resize logic\n this.resizedViewport = this.smartResize(width, height);\n }\n\n setCurrentUrl(url: string): void {\n this.currentUrl = url;\n }\n\n setScreenshotProvider(provider: () => Promise<string>): void {\n this.screenshotProvider = provider;\n }\n\n setActionHandler(handler: (action: AgentAction) => Promise<void>): void {\n this.actionHandler = handler;\n }\n\n /\n Smart resize algorithm from FARA\n * Ensures dimensions are divisible by factor and within pixel limits\n /\n private smartResize(\n width: number,\n height: number,\n ): { width: number; height: number } {\n const { patch_size, merge_size, min_pixels, max_pixels } =\n this.MLM_PROCESSOR_IM_CFG;\n const factor = patch_size merge_size;\n\n const roundByFactor = (num: number, f: number) => Math.round(num / f) * f;\n const ceilByFactor = (num: number, f: number) => Math.ceil(num / f) * f;\n const floorByFactor = (num: number, f: number) => Math.floor(num / f) * f;\n\n let h_bar = Math.max(factor, roundByFactor(height, factor));\n let w_bar = Math.max(factor, roundByFactor(width, factor));\n\n if (h_bar * w_bar > max_pixels) {\n const beta = Math.sqrt((height * width) / max_pixels);\n h_bar = floorByFactor(height / beta, factor);\n w_bar = floorByFactor(width / beta, factor);\n } else if (h_bar * w_bar < min_pixels) {\n const beta = Math.sqrt(min_pixels / (height * width));\n h_bar = ceilByFactor(height * beta, factor);\n w_bar = ceilByFactor(width * beta, factor);\n }\n\n return { width: w_bar, height: h_bar };\n }\n\n /*\n Generate system prompt with tool description\n * Simplified to match Python's minimal approach\n /\n private generateSystemPrompt(): string {\n const { width, height } = this.resizedViewport;\n\n // Base prompt - Minimalist like Python\n let basePrompt = \"You are a helpful assistant.\";\n\n // Add user-provided instructions if available\n if (this.userProvidedInstructions) {\n basePrompt = `${basePrompt}\\n\\n${this.userProvidedInstructions}`;\n }\n\n // Tool description from FaraComputerUse\n const toolDescription = `Use a mouse and keyboard to interact with a computer, and take screenshots.\n This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\n* The screen's resolution is ${width}x${height}.\n* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.\n* When a separate scrollable container prominently overlays the webpage, if you want to scroll within it, you typically need to mouse_move() over it first and then scroll().\n* If a popup window appears that you want to close, if left_click() on the 'X' or close button doesn't work, try key(keys=['Escape']) to close it.\n* On some search bars, when you type(), you may need to press_enter=False and instead separately call left_click() on the search button to submit the search query. This is especially true of search bars that have auto-suggest popups for e.g. locations\n* For calendar widgets, you usually need to left_click() on arrows to move between months and left_click() on dates to select them; type() is not typically used to input dates there.`;\n\n // Tool parameters description\n const actionsDescription = `The action to perform. The available actions are:\n* \\`key\\`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order. Includes \"Enter\", \"Alt\", \"Shift\", \"Tab\", \"Control\", \"Backspace\", \"Delete\", \"Escape\", \"ArrowUp\", \"ArrowDown\", \"ArrowLeft\", \"ArrowRight\", \"PageDown\", \"PageUp\", \"Shift\", etc.\n* \\`type\\`: Type a string of text on the keyboard.\n* \\`mouse_move\\`: Move the cursor to a specified (x, y) pixel coordinate on the screen.\n* \\`left_click\\`: Click the left mouse button.\n* \\`scroll\\`: Performs a scroll of the mouse scroll wheel.\n* \\`history_back\\`: Go back to the previous page in the browser history.\n* \\`pause_and_memorize_fact\\`: Pause and memorize a fact for future reference.\n* \\`visit_url\\`: Visit a specified URL.\n* \\`web_search\\`: Perform a web search with a specified query.\n* \\`wait\\`: Wait specified seconds for the change to happen.\n* \\`terminate\\`: Terminate the current task and report its completion status.`;\n\n // Tool JSON schema\n const toolSchema = {\n name: \"computer_use\",\n description: toolDescription,\n parameters: {\n type: \"object\",\n required: [\"action\"],\n properties: {\n action: {\n type: \"string\",\n description: actionsDescription,\n enum: [\n \"key\",\n \"type\",\n \"mouse_move\",\n \"left_click\",\n \"scroll\",\n \"visit_url\",\n \"web_search\",\n \"history_back\",\n \"pause_and_memorize_fact\",\n \"wait\",\n \"terminate\",\n ],\n },\n keys: {\n type: \"array\",\n description: \"Required only by `action=key`.\",\n },\n text: {\n type: \"string\",\n description: \"Required only by `action=type`.\",\n },\n press_enter: {\n type: \"boolean\",\n description:\n \"Whether to press the Enter key after typing. Required only by `action=type`.\",\n },\n delete_existing_text: {\n type: \"boolean\",\n description:\n \"Whether to delete existing text before typing. Required only by `action=type`.\",\n },\n coordinate: {\n type: \"array\",\n description:\n \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=left_click`, `action=mouse_move`, and `action=type`.\",\n },\n pixels: {\n type: \"number\",\n description:\n \"The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll`.\",\n },\n fact: {\n type: \"string\",\n description:\n \"The fact to remember for the future. Required only by `action=pause_and_memorize_fact`.\",\n },\n time: {\n type: \"number\",\n description: \"The seconds to wait. Required only by `action=wait`.\",\n },\n status: {\n type: \"string\",\n description:\n \"The status of the task. Required only by `action=terminate`.\",\n enum: [\"success\", \"failure\"],\n },\n },\n },\n };\n\n // Format as FARA function calling template (FN_CALL_TEMPLATE format)\n const toolDescs = JSON.stringify(toolSchema, null, 2);\n const functionCallTemplate = `\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n${toolDescs}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{{\"name\": <function-name>, \"arguments\": <args-json-object>}}\n</tool_call>`;\n\n return `${basePrompt}\\n\\n${functionCallTemplate}`;\n }\n\n /*\n Parse thoughts and action from model response\n * FARA uses XML-based tool calling: <tool_call>\\n{...}\\n</tool_call>\n /\n private parseThoughtsAndAction(response: string): {\n thoughts: string;\n functionCall: FaraFunctionCall;\n } {\n try {\n const parts = response.split(\"<tool_call>\\n\");\n const thoughts = parts[0].trim();\n const actionText = parts[1].split(\"\\n</tool_call>\")[0].trim();\n\n let parsedAction;\n try {\n parsedAction = JSON.parse(actionText);\n } catch (jsonError) {\n // Fix common malformed JSON: double opening brackets {{\"name\": ...}}\n // This happens when the model adds an extra opening brace\n if (actionText.startsWith(\"{{\") && actionText.endsWith(\"}\")) {\n // Remove the extra opening brace\n const fixedText = actionText.slice(1);\n try {\n parsedAction = JSON.parse(fixedText);\n } catch (retryError) {\n throw new Error(\n `Failed to parse action text even after fixing double brackets. Original: ${actionText}. Fixed: ${fixedText}. Error: ${retryError}`,\n );\n }\n } else {\n throw new Error(\n `Failed to parse action text as JSON: ${actionText}. Error: ${jsonError}`,\n );\n }\n }\n\n return {\n thoughts,\n functionCall: {\n name: parsedAction.name \|\| \"computer_use\",\n arguments: {\n ...parsedAction.arguments,\n thoughts,\n },\n },\n };\n } catch (error) {\n throw new Error(\n `Failed to parse FARA tool call from response: ${response}. Error: ${error}`,\n );\n }\n }\n\n /\n Convert FARA function call to Stagehand AgentAction\n /\n private convertFunctionCallToAction(\n functionCall: FaraFunctionCall,\n ): AgentAction {\n const args = functionCall.arguments;\n const action = args.action as string;\n\n // Transform coordinates from resized to original viewport\n const transformCoordinate = (coord: number[]): number[] => {\n if (!coord \|\| coord.length !== 2) return coord;\n const [x, y] = coord;\n const scaleX = this.currentViewport.width / this.resizedViewport.width;\n const scaleY = this.currentViewport.height / this.resizedViewport.height;\n return [Math.round(x scaleX), Math.round(y * scaleY)];\n };\n\n const baseAction = {\n type: action,\n reasoning: args.thoughts as string,\n };\n\n switch (action) {\n case \"left_click\": {\n const clickCoord = transformCoordinate(args.coordinate as number[]);\n return {\n ...baseAction,\n type: \"click\",\n x: clickCoord[0],\n y: clickCoord[1],\n button: \"left\" as const,\n };\n }\n\n case \"mouse_move\": {\n const moveCoord = transformCoordinate(args.coordinate as number[]);\n return {\n ...baseAction,\n type: \"move\",\n coordinate: moveCoord,\n };\n }\n\n case \"type\": {\n const typeCoord = args.coordinate\n ? transformCoordinate(args.coordinate as number[])\n : undefined;\n return {\n ...baseAction,\n text: args.text as string,\n ...(typeCoord && { x: typeCoord[0], y: typeCoord[1] }),\n press_enter:\n args.press_enter !== undefined\n ? (args.press_enter as boolean)\n : true,\n ...(args.delete_existing_text !== undefined && {\n delete_existing_text: args.delete_existing_text as boolean,\n }),\n };\n }\n\n case \"key\":\n case \"keypress\": {\n const keys = (args.keys as string[]) \|\| [];\n // Normalize keys to Playwright format\n const normalizedKeys = keys.map((k) => mapKeyToPlaywright(k));\n return {\n ...baseAction,\n type: \"keypress\",\n keys: normalizedKeys,\n };\n }\n\n case \"scroll\": {\n const pixels = (args.pixels as number) \|\| 0;\n // FARA: positive = scroll up, negative = scroll down\n // Convert to scroll_x/scroll_y\n return {\n ...baseAction,\n scroll_x: 0,\n scroll_y: -pixels, // Invert: negative pixels = scroll down\n };\n }\n\n case \"visit_url\": {\n let url = args.url as string;\n // Enhanced URL processing like Python\n if (\n !url.startsWith(\"https://\") &&\n !url.startsWith(\"http://\") &&\n !url.startsWith(\"file://\") &&\n !url.startsWith(\"about:\")\n ) {\n // If URL contains space, treat as search query\n if (url.includes(\" \")) {\n url = `https://www.bing.com/search?q=${encodeURIComponent(url)}&FORM=QBLH`;\n } else {\n // Otherwise prefix with https://\n url = \"https://\" + url;\n }\n }\n return {\n ...baseAction,\n type: \"goto\",\n url,\n };\n }\n\n case \"web_search\": {\n // Convert web search to visit_url with Bing search\n const query = args.query as string;\n const searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&FORM=QBLH`;\n return {\n ...baseAction,\n type: \"goto\",\n url: searchUrl,\n };\n }\n\n case \"history_back\":\n return {\n ...baseAction,\n type: \"back\",\n };\n\n case \"wait\": {\n // Support both 'time' and 'duration' parameters with default (matches Python)\n const durationSeconds =\n (args.time as number) \|\| (args.duration as number) \|\| 3.0;\n return {\n ...baseAction,\n timeMs: durationSeconds * 1000, // Convert seconds to ms\n };\n }\n\n case \"pause_and_memorize_fact\": {\n // Store the fact for future reference (matches Python)\n const fact = args.fact as string;\n this.facts.push(fact);\n return {\n ...baseAction,\n fact,\n };\n }\n\n case \"terminate\":\n return {\n ...baseAction,\n status: args.status as string,\n };\n\n default:\n return {\n ...baseAction,\n ...args,\n };\n }\n }\n\n /*\n Capture a screenshot and return as base64 data URL\n /\n async captureScreenshot(): Promise<string> {\n if (!this.screenshotProvider) {\n throw new AgentScreenshotProviderError(\"Screenshot provider not set\");\n }\n\n const base64Screenshot = await this.screenshotProvider();\n return `data:image/png;base64,${base64Screenshot}`;\n }\n\n /\n Remove old screenshots from history\n * Matches Python's maybe_remove_old_screenshots\n /\n private maybeRemoveOldScreenshots(\n history: FaraMessage[],\n includesCurrent: boolean = false,\n ): FaraMessage[] {\n if (this.maxImages <= 0) {\n return history;\n }\n\n const maxImages = includesCurrent ? this.maxImages : this.maxImages - 1;\n const newHistory: FaraMessage[] = [];\n let nImages = 0;\n\n // Iterate backwards\n for (let i = history.length - 1; i >= 0; i--) {\n const msg = history[i];\n\n // Check if message has image\n let hasImage = false;\n if (Array.isArray(msg.content)) {\n hasImage = msg.content.some((c) => c.type === \"image_url\");\n }\n\n if (i === 0 && nImages >= maxImages) {\n // First message (task) - preserve text, remove image\n if (Array.isArray(msg.content)) {\n const newContent = msg.content.filter((c) => c.type !== \"image_url\");\n // If no content left, skip (unless it's the only message, but Python logic says continue)\n if (newContent.length === 0) {\n continue;\n }\n newHistory.push({ ...msg, content: newContent });\n } else {\n newHistory.push(msg);\n }\n continue;\n }\n\n if (hasImage) {\n if (nImages < maxImages) {\n newHistory.push(msg);\n nImages++;\n } else {\n // Remove image, keep text\n if (Array.isArray(msg.content)) {\n const newContent = msg.content.filter(\n (c) => c.type !== \"image_url\",\n );\n // If content becomes empty, we can skip this message entirely (unless it's meaningful text)\n // Python logic: if msg is None continue.\n if (newContent.length > 0) {\n newHistory.push({ ...msg, content: newContent });\n }\n } else {\n newHistory.push(msg);\n }\n }\n } else {\n newHistory.push(msg);\n }\n }\n\n return newHistory.reverse();\n }\n\n /\n Reconstruct history for API call\n * Merges conceptual chat history with raw action history\n /\n private reconstructHistory(): FaraMessage[] {\n const history: FaraMessage[] = [];\n let actionTurn = 0;\n\n for (let i = 0; i < this.conversationHistory.length; i++) {\n const m = this.conversationHistory[i];\n if (m.role === \"assistant\") {\n if (actionTurn >= this.actionHistory.length) {\n // Should not happen if synced correctly\n console.warn(\"OUT OF SYNC: Action history shorter than chat history\");\n history.push(m);\n } else {\n history.push(this.actionHistory[actionTurn]);\n actionTurn++;\n }\n } else {\n history.push(m);\n }\n }\n\n return this.maybeRemoveOldScreenshots(history);\n }\n\n /\n Execute a single step\n /\n private async executeStep(\n logger: (message: LogLine) => void,\n isFirstRound: boolean = false,\n ): Promise<{\n actions: AgentAction[];\n completed: boolean;\n usage: {\n input_tokens: number;\n output_tokens: number;\n inference_time_ms: number;\n };\n }> {\n // Capture screenshot\n const screenshotDataUrl = await this.captureScreenshot();\n\n // Update conversation history with new screenshot/message\n if (isFirstRound) {\n // First round: modify the last message (initial user instruction) to include screenshot\n const lastMessage =\n this.conversationHistory[this.conversationHistory.length - 1];\n if (lastMessage && lastMessage.role === \"user\") {\n const originalContent =\n typeof lastMessage.content === \"string\"\n ? lastMessage.content\n : (lastMessage.content.find((c) => c.type === \"text\")?.text ??\n \"Start task\");\n\n lastMessage.content = [\n {\n type: \"image_url\",\n image_url: { url: screenshotDataUrl },\n },\n {\n type: \"text\",\n text: originalContent,\n },\n ];\n }\n } else {\n // Subsequent rounds: add new user message with screenshot\n const userContent: FaraMessageContent[] = [\n {\n type: \"image_url\",\n image_url: { url: screenshotDataUrl },\n },\n ];\n\n // Add current URL if available\n let textPrompt =\n \"Here is the next screenshot. Think about what to do next.\";\n if (this.currentUrl) {\n const trimmedUrl =\n this.currentUrl.length > 100\n ? this.currentUrl.slice(0, 100) + \"...\"\n : this.currentUrl;\n textPrompt = `Current URL: ${trimmedUrl}\\n${textPrompt}`;\n }\n\n userContent.push({\n type: \"text\",\n text: textPrompt,\n });\n\n this.conversationHistory.push({\n role: \"user\",\n content: userContent,\n });\n }\n\n // Reconstruct history for model call\n let history = this.reconstructHistory();\n\n // Prepend system prompt (generated fresh)\n const systemMessage: FaraMessage = {\n role: \"system\",\n content: this.generateSystemPrompt(),\n };\n history = [systemMessage, ...history];\n\n // Make API call\n logger({\n category: \"agent\",\n message: `Making API call to FARA model with ${history.length} messages`,\n level: 2,\n });\n\n const startTime = Date.now();\n let response;\n try {\n response = await this.client.chat.completions.create({\n model: this.modelName,\n messages: history as unknown as ChatCompletionMessageParam[],\n temperature: this.temperature,\n });\n } catch (apiError) {\n logger({\n category: \"agent\",\n message: `API call failed: ${apiError instanceof Error ? apiError.message : String(apiError)}`,\n level: 0,\n });\n throw apiError;\n }\n const inferenceTime = Date.now() - startTime;\n\n logger({\n category: \"agent\",\n message: `API call completed in ${inferenceTime}ms`,\n level: 2,\n });\n\n const content = response.choices[0].message.content \|\| \"\";\n const usage = response.usage \|\| {\n prompt_tokens: 0,\n completion_tokens: 0,\n total_tokens: 0,\n };\n\n // Add assistant response to both histories\n const assistantMsg: FaraMessage = {\n role: \"assistant\",\n content,\n };\n this.conversationHistory.push(assistantMsg);\n this.actionHistory.push(assistantMsg);\n\n logger({\n category: \"agent\",\n message: `Model response: ${content}`,\n level: 2,\n });\n\n // Parse tool call\n const { thoughts, functionCall } = this.parseThoughtsAndAction(content);\n\n logger({\n category: \"agent\",\n message: `Thoughts: ${thoughts}`,\n level: 2,\n });\n\n logger({\n category: \"agent\",\n message: `Action: ${JSON.stringify(functionCall.arguments)}`,\n level: 2,\n });\n\n // Convert to AgentAction\n const agentAction = this.convertFunctionCallToAction(functionCall);\n\n // Expand type action into multiple actions if it has coordinates\n const actions: AgentAction[] = [];\n if (\n agentAction.type === \"type\" &&\n typeof agentAction.x === \"number\" &&\n typeof agentAction.y === \"number\"\n ) {\n // First, click at the coordinates to focus the field\n actions.push({\n type: \"click\",\n x: agentAction.x,\n y: agentAction.y,\n button: \"left\",\n });\n\n // If delete_existing_text is true, clear the field first\n if (agentAction.delete_existing_text) {\n actions.push({\n type: \"keypress\",\n keys: [\"Command+A\"],\n });\n actions.push({\n type: \"keypress\",\n keys: [\"Backspace\"],\n });\n }\n\n // Add the type action (without coordinates since we already clicked)\n actions.push({\n type: \"type\",\n text: agentAction.text,\n });\n\n // If press_enter is true (default), press Enter after typing\n if (agentAction.press_enter !== false) {\n actions.push({\n type: \"keypress\",\n keys: [\"Enter\"],\n });\n }\n } else {\n // For all other actions, just add as-is\n actions.push(agentAction);\n }\n\n // Execute all actions if handler is available\n if (this.actionHandler && agentAction.type !== \"terminate\") {\n for (const action of actions) {\n await this.actionHandler(action);\n }\n }\n\n // Check if completed\n const completed = functionCall.arguments.action === \"terminate\";\n\n return {\n actions,\n completed,\n usage: {\n input_tokens: usage.prompt_tokens,\n output_tokens: usage.completion_tokens,\n inference_time_ms: inferenceTime,\n },\n };\n }\n\n /\n Execute a task with the FARA CUA\n * This is the main entry point for the agent\n * @implements AgentClient.execute\n */\n async execute(executionOptions: AgentExecutionOptions): Promise<AgentResult> {\n const { options, logger } = executionOptions;\n const { instruction } = options;\n const maxSteps = options.maxSteps \|\| 10;\n\n let currentStep = 0;\n let completed = false;\n const actions: AgentAction[] = [];\n const messageList: string[] = [];\n let finalMessage = \"\";\n let totalInputTokens = 0;\n let totalOutputTokens = 0;\n let totalInferenceTime = 0;\n\n // Initialize conversation with user instruction\n // System prompt is NOT added here, it's added dynamically in executeStep\n this.conversationHistory = [\n {\n role: \"user\",\n content: instruction,\n },\n ];\n this.actionHistory = [];\n\n try {\n // Execute steps until completion or max steps reached\n while (!completed && currentStep < maxSteps) {\n logger({\n category: \"agent\",\n message: `Executing step ${currentStep + 1}/${maxSteps}`,\n level: 1,\n });\n\n const isFirstRound = currentStep === 0;\n const result = await this.executeStep(logger, isFirstRound);\n totalInputTokens += result.usage.input_tokens;\n totalOutputTokens += result.usage.output_tokens;\n totalInferenceTime += result.usage.inference_time_ms;\n\n // Add actions to the list\n actions.push(...result.actions);\n\n // Update completion status\n completed = result.completed;\n\n currentStep++;\n\n // Record message for this step\n const lastAction = result.actions[result.actions.length - 1];\n if (lastAction?.reasoning) {\n messageList.push(lastAction.reasoning);\n }\n }\n\n // Generate final message\n if (completed) {\n const lastAction = actions[actions.length - 1];\n finalMessage =\n (lastAction as { status?: string })?.status === \"success\"\n ? \"Task completed successfully.\"\n : \"Task completed with failures.\";\n } else {\n finalMessage = `Reached maximum steps (${maxSteps}) without completion.`;\n }\n\n if (messageList.length > 0) {\n finalMessage = `${messageList.join(\"\\n\\n\")}\\n\\n${finalMessage}`;\n }\n\n return {\n success: completed,\n completed,\n message: finalMessage,\n actions,\n usage: {\n input_tokens: totalInputTokens,\n output_tokens: totalOutputTokens,\n inference_time_ms: totalInferenceTime,\n },\n };\n } catch (error) {\n logger({\n category: \"agent\",\n message: `Error during execution: ${error}`,\n level: 0,\n });\n\n // Rethrow to allow eval runner's retry logic to handle transient errors\n throw error;\n }\n }\n}\n"]}
1	+ {"version":3,"file":"MicrosoftCUAClient.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/MicrosoftCUAClient.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,4BAA4B,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AA+B9D;;;GAGG;AACH,MAAM,OAAO,kBAAmB,SAAQ,WAAW;IACzC,MAAM,CAAS;IACf,OAAO,CAAS;IAChB,MAAM,CAAS;IACf,eAAe,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC/C,UAAU,CAAU;IACpB,kBAAkB,CAAyB;IAC3C,aAAa,CAA0C;IAE/D,sBAAsB;IACd,mBAAmB,GAAkB,EAAE,CAAC,CAAC,kBAAkB;IAC3D,aAAa,GAAkB,EAAE,CAAC,CAAC,sBAAsB;IAEzD,SAAS,GAAW,CAAC,CAAC;IACtB,WAAW,GAAW,CAAC,CAAC;IACxB,KAAK,GAAa,EAAE,CAAC;IAE7B,qCAAqC;IACpB,oBAAoB,GAAG;QACtC,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,QAAQ;QACpB,UAAU,EAAE,EAAE;QACd,UAAU,EAAE,CAAC;KACd,CAAC;IAEF,qCAAqC;IAC7B,eAAe,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAEvD,YACE,IAAe,EACf,SAAiB,EACjB,wBAAiC,EACjC,aAA6B;QAE7B,KAAK,CAAC,IAAI,EAAE,SAAS,IAAI,SAAS,EAAE,wBAAwB,CAAC,CAAC;QAE9D,yBAAyB;QACzB,IAAI,CAAC,MAAM;YACR,aAAa,EAAE,MAAiB;gBACjC,OAAO,CAAC,GAAG,CAAC,aAAa;gBACzB,OAAO,CAAC,GAAG,CAAC,iBAAiB;gBAC7B,EAAE,CAAC;QACL,IAAI,CAAC,OAAO;YACT,aAAa,EAAE,OAAkB;gBAClC,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,kBAAkB;gBAC9B,EAAE,CAAC;QAEL,qCAAqC;QACrC,IAAI,CAAC,aAAa,GAAG;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;QAEF,mBAAmB;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CACb,8HAA8H,CAC/H,CAAC;QACJ,CAAC;QAED,iEAAiE;QACjE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,gCAAgC;QAChC,IAAI,aAAa,EAAE,SAAS,KAAK,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,SAAmB,CAAC;QACrD,CAAC;QAED,cAAc;QACd,IAAI,aAAa,EAAE,WAAW,KAAK,SAAS,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,GAAG,aAAa,CAAC,WAAqB,CAAC;QACzD,CAAC;IACH,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,MAAc;QACvC,IAAI,CAAC,eAAe,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QACzC,oDAAoD;QACpD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACxB,CAAC;IAED,qBAAqB,CAAC,QAA+B;QACnD,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAED,gBAAgB,CAAC,OAA+C;QAC9D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACK,WAAW,CACjB,KAAa,EACb,MAAc;QAEd,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,GACtD,IAAI,CAAC,oBAAoB,CAAC;QAC5B,MAAM,MAAM,GAAG,UAAU,GAAG,UAAU,CAAC;QAEvC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1E,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACxE,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAE1E,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE3D,IAAI,KAAK,GAAG,KAAK,GAAG,UAAU,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,UAAU,CAAC,CAAC;YACtD,KAAK,GAAG,aAAa,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7C,KAAK,GAAG,aAAa,CAAC,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9C,CAAC;aAAM,IAAI,KAAK,GAAG,KAAK,GAAG,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;YACtD,KAAK,GAAG,YAAY,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;YAC5C,KAAK,GAAG,YAAY,CAAC,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;IAED;;;OAGG;IACK,oBAAoB;QAC1B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QAE/C,uCAAuC;QACvC,IAAI,UAAU,GAAG,8BAA8B,CAAC;QAEhD,8CAA8C;QAC9C,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAClC,UAAU,GAAG,GAAG,UAAU,OAAO,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnE,CAAC;QAED,wCAAwC;QACxC,MAAM,eAAe,GAAG;;;+BAGG,KAAK,IAAI,MAAM;;;;;;;uLAOyI,CAAC;QAEpL,8BAA8B;QAC9B,MAAM,kBAAkB,GAAG;;;;;;;;;;;8EAW+C,CAAC;QAE3E,mBAAmB;QACnB,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,cAAc;YACpB,WAAW,EAAE,eAAe;YAC5B,UAAU,EAAE;gBACV,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,UAAU,EAAE;oBACV,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,kBAAkB;wBAC/B,IAAI,EAAE;4BACJ,KAAK;4BACL,MAAM;4BACN,YAAY;4BACZ,YAAY;4BACZ,QAAQ;4BACR,WAAW;4BACX,YAAY;4BACZ,cAAc;4BACd,yBAAyB;4BACzB,MAAM;4BACN,WAAW;yBACZ;qBACF;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,OAAO;wBACb,WAAW,EAAE,gCAAgC;qBAC9C;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,iCAAiC;qBAC/C;oBACD,WAAW,EAAE;wBACX,IAAI,EAAE,SAAS;wBACf,WAAW,EACT,8EAA8E;qBACjF;oBACD,oBAAoB,EAAE;wBACpB,IAAI,EAAE,SAAS;wBACf,WAAW,EACT,gFAAgF;qBACnF;oBACD,UAAU,EAAE;wBACV,IAAI,EAAE,OAAO;wBACb,WAAW,EACT,4LAA4L;qBAC/L;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,+HAA+H;qBAClI;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,yFAAyF;qBAC5F;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sDAAsD;qBACpE;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,WAAW,EACT,8DAA8D;wBAChE,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;qBAC7B;iBACF;aACF;SACF,CAAC;QAEF,qEAAqE;QACrE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,oBAAoB,GAAG;;;EAG/B,SAAS;;;;;;aAME,CAAC;QAEV,OAAO,GAAG,UAAU,OAAO,oBAAoB,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,QAAgB;QAI7C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9D,IAAI,YAAY,CAAC;YACjB,IAAI,CAAC;gBACH,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,qEAAqE;gBACrE,0DAA0D;gBAC1D,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5D,iCAAiC;oBACjC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtC,IAAI,CAAC;wBACH,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBACvC,CAAC;oBAAC,OAAO,UAAU,EAAE,CAAC;wBACpB,MAAM,IAAI,KAAK,CACb,4EAA4E,UAAU,YAAY,SAAS,YAAY,UAAU,EAAE,EACnI,EAAE,KAAK,EAAE,UAAU,EAAE,CACtB,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,wCAAwC,UAAU,YAAY,SAAS,EAAE,EACzE,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,OAAO;gBACL,QAAQ;gBACR,YAAY,EAAE;oBACZ,IAAI,EAAE,YAAY,CAAC,IAAI,IAAI,cAAc;oBACzC,SAAS,EAAE;wBACT,GAAG,YAAY,CAAC,SAAS;wBACzB,QAAQ;qBACT;iBACF;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,iDAAiD,QAAQ,YAAY,KAAK,EAAE,EAC5E,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,2BAA2B,CACjC,YAA8B;QAE9B,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAgB,CAAC;QAErC,0DAA0D;QAC1D,MAAM,mBAAmB,GAAG,CAAC,KAAe,EAAY,EAAE;YACxD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/C,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;YACrB,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YACvE,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC;QAEF,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,IAAI,CAAC,QAAkB;SACnC,CAAC;QAEF,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,UAAsB,CAAC,CAAC;gBACpE,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,OAAO;oBACb,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;oBAChB,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;oBAChB,MAAM,EAAE,MAAe;iBACxB,CAAC;YACJ,CAAC;YAED,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,UAAsB,CAAC,CAAC;gBACnE,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,SAAS;iBACtB,CAAC;YACJ,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU;oBAC/B,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAsB,CAAC;oBAClD,CAAC,CAAC,SAAS,CAAC;gBACd,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,IAAI,CAAC,IAAc;oBACzB,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtD,WAAW,EACT,IAAI,CAAC,WAAW,KAAK,SAAS;wBAC5B,CAAC,CAAE,IAAI,CAAC,WAAuB;wBAC/B,CAAC,CAAC,IAAI;oBACV,GAAG,CAAC,IAAI,CAAC,oBAAoB,KAAK,SAAS,IAAI;wBAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAA+B;qBAC3D,CAAC;iBACH,CAAC;YACJ,CAAC;YAED,KAAK,KAAK,CAAC;YACX,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,IAAI,GAAI,IAAI,CAAC,IAAiB,IAAI,EAAE,CAAC;gBAC3C,sCAAsC;gBACtC,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9D,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,cAAc;iBACrB,CAAC;YACJ,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,MAAM,GAAI,IAAI,CAAC,MAAiB,IAAI,CAAC,CAAC;gBAC5C,qDAAqD;gBACrD,+BAA+B;gBAC/B,OAAO;oBACL,GAAG,UAAU;oBACb,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,CAAC,MAAM,EAAE,wCAAwC;iBAC5D,CAAC;YACJ,CAAC;YAED,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,IAAI,GAAG,GAAG,IAAI,CAAC,GAAa,CAAC;gBAC7B,sCAAsC;gBACtC,IACE,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC;oBAC3B,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC1B,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC1B,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EACzB,CAAC;oBACD,+CAA+C;oBAC/C,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBACtB,GAAG,GAAG,iCAAiC,kBAAkB,CAAC,GAAG,CAAC,YAAY,CAAC;oBAC7E,CAAC;yBAAM,CAAC;wBACN,iCAAiC;wBACjC,GAAG,GAAG,UAAU,GAAG,GAAG,CAAC;oBACzB,CAAC;gBACH,CAAC;gBACD,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;oBACZ,GAAG;iBACJ,CAAC;YACJ,CAAC;YAED,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,mDAAmD;gBACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAe,CAAC;gBACnC,MAAM,SAAS,GAAG,iCAAiC,kBAAkB,CAAC,KAAK,CAAC,YAAY,CAAC;gBACzF,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;oBACZ,GAAG,EAAE,SAAS;iBACf,CAAC;YACJ,CAAC;YAED,KAAK,cAAc;gBACjB,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI,EAAE,MAAM;iBACb,CAAC;YAEJ,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,8EAA8E;gBAC9E,MAAM,eAAe,GAClB,IAAI,CAAC,IAAe,IAAK,IAAI,CAAC,QAAmB,IAAI,GAAG,CAAC;gBAC5D,OAAO;oBACL,GAAG,UAAU;oBACb,MAAM,EAAE,eAAe,GAAG,IAAI,EAAE,wBAAwB;iBACzD,CAAC;YACJ,CAAC;YAED,KAAK,yBAAyB,CAAC,CAAC,CAAC;gBAC/B,uDAAuD;gBACvD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAc,CAAC;gBACjC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,OAAO;oBACL,GAAG,UAAU;oBACb,IAAI;iBACL,CAAC;YACJ,CAAC;YAED,KAAK,WAAW;gBACd,OAAO;oBACL,GAAG,UAAU;oBACb,MAAM,EAAE,IAAI,CAAC,MAAgB;iBAC9B,CAAC;YAEJ;gBACE,OAAO;oBACL,GAAG,UAAU;oBACb,GAAG,IAAI;iBACR,CAAC;QACN,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,MAAM,IAAI,4BAA4B,CAAC,6BAA6B,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzD,OAAO,yBAAyB,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;;OAGG;IACK,yBAAyB,CAC/B,OAAsB,EACtB,kBAA2B,KAAK;QAEhC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACxE,MAAM,UAAU,GAAkB,EAAE,CAAC;QACrC,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,oBAAoB;QACpB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,6BAA6B;YAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YAC7D,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;gBACpC,qDAAqD;gBACrD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC/B,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;oBACrE,0FAA0F;oBAC1F,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,SAAS;oBACX,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;gBACnD,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,OAAO,GAAG,SAAS,EAAE,CAAC;oBACxB,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACrB,OAAO,EAAE,CAAC;gBACZ,CAAC;qBAAM,CAAC;oBACN,0BAA0B;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC/B,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAC9B,CAAC;wBACF,4FAA4F;wBAC5F,yCAAyC;wBACzC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC1B,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;wBACnD,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACvB,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACK,kBAAkB;QACxB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzD,MAAM,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC3B,IAAI,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;oBAC5C,wCAAwC;oBACxC,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;oBAC7C,UAAU,EAAE,CAAC;gBACf,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CACvB,MAAkC,EAClC,eAAwB,KAAK;QAU7B,qBAAqB;QACrB,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzD,0DAA0D;QAC1D,IAAI,YAAY,EAAE,CAAC;YACjB,wFAAwF;YACxF,MAAM,WAAW,GACf,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,WAAW,IAAI,WAAW,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC/C,MAAM,eAAe,GACnB,OAAO,WAAW,CAAC,OAAO,KAAK,QAAQ;oBACrC,CAAC,CAAC,WAAW,CAAC,OAAO;oBACrB,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI;wBACzD,YAAY,CAAC,CAAC;gBAEpB,WAAW,CAAC,OAAO,GAAG;oBACpB;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;qBACtC;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,eAAe;qBACtB;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0DAA0D;YAC1D,MAAM,WAAW,GAAyB;gBACxC;oBACE,IAAI,EAAE,WAAW;oBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;iBACtC;aACF,CAAC;YAEF,+BAA+B;YAC/B,IAAI,UAAU,GACZ,2DAA2D,CAAC;YAC9D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,MAAM,UAAU,GACd,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG;oBAC1B,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;oBACvC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACtB,UAAU,GAAG,gBAAgB,UAAU,KAAK,UAAU,EAAE,CAAC;YAC3D,CAAC;YAED,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YAEH,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;QACL,CAAC;QAED,qCAAqC;QACrC,IAAI,OAAO,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAExC,0CAA0C;QAC1C,MAAM,aAAa,GAAgB;YACjC,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,CAAC,oBAAoB,EAAE;SACrC,CAAC;QACF,OAAO,GAAG,CAAC,aAAa,EAAE,GAAG,OAAO,CAAC,CAAC;QAEtC,gBAAgB;QAChB,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,sCAAsC,OAAO,CAAC,MAAM,WAAW;YACxE,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,QAAQ,EAAE,OAAkD;gBAC5D,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,oBAAoB,QAAQ,YAAY,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;gBAC9F,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,QAAQ,CAAC;QACjB,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAE7C,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,yBAAyB,aAAa,IAAI;YACnD,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI;YAC9B,aAAa,EAAE,CAAC;YAChB,iBAAiB,EAAE,CAAC;YACpB,YAAY,EAAE,CAAC;SAChB,CAAC;QAEF,2CAA2C;QAC3C,MAAM,YAAY,GAAgB;YAChC,IAAI,EAAE,WAAW;YACjB,OAAO;SACR,CAAC;QACF,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAEtC,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,mBAAmB,OAAO,EAAE;YACrC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,kBAAkB;QAClB,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAExE,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,aAAa,QAAQ,EAAE;YAChC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,WAAW,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;YAC5D,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,2BAA2B,CAAC,YAAY,CAAC,CAAC;QAEnE,iEAAiE;QACjE,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,IACE,WAAW,CAAC,IAAI,KAAK,MAAM;YAC3B,OAAO,WAAW,CAAC,CAAC,KAAK,QAAQ;YACjC,OAAO,WAAW,CAAC,CAAC,KAAK,QAAQ,EACjC,CAAC;YACD,qDAAqD;YACrD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,CAAC,EAAE,WAAW,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;gBAChB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,yDAAyD;YACzD,IAAI,WAAW,CAAC,oBAAoB,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,CAAC,WAAW,CAAC;iBACpB,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,CAAC,WAAW,CAAC;iBACpB,CAAC,CAAC;YACL,CAAC;YAED,qEAAqE;YACrE,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,WAAW,CAAC,IAAI;aACvB,CAAC,CAAC;YAEH,6DAA6D;YAC7D,IAAI,WAAW,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,CAAC,OAAO,CAAC;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5B,CAAC;QAED,8CAA8C;QAC9C,IAAI,IAAI,CAAC,aAAa,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC,MAAM,KAAK,WAAW,CAAC;QAEhE,OAAO;YACL,OAAO;YACP,SAAS;YACT,KAAK,EAAE;gBACL,YAAY,EAAE,KAAK,CAAC,aAAa;gBACjC,aAAa,EAAE,KAAK,CAAC,iBAAiB;gBACtC,iBAAiB,EAAE,aAAa;aACjC;SACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,gBAAuC;QACnD,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC7C,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAExC,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,YAAoB,CAAC;QACzB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,gDAAgD;QAChD,yEAAyE;QACzE,IAAI,CAAC,mBAAmB,GAAG;YACzB;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB;SACF,CAAC;QACF,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QAExB,IAAI,CAAC;YACH,sDAAsD;YACtD,OAAO,CAAC,SAAS,IAAI,WAAW,GAAG,QAAQ,EAAE,CAAC;gBAC5C,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,kBAAkB,WAAW,GAAG,CAAC,IAAI,QAAQ,EAAE;oBACxD,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBAEH,MAAM,YAAY,GAAG,WAAW,KAAK,CAAC,CAAC;gBACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;gBAC5D,gBAAgB,IAAI,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;gBAC9C,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC;gBAChD,kBAAkB,IAAI,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;gBAErD,0BAA0B;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;gBAEhC,2BAA2B;gBAC3B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAE7B,WAAW,EAAE,CAAC;gBAEd,+BAA+B;gBAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC7D,IAAI,UAAU,EAAE,SAAS,EAAE,CAAC;oBAC1B,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAED,yBAAyB;YACzB,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC/C,YAAY;oBACT,UAAkC,EAAE,MAAM,KAAK,SAAS;wBACvD,CAAC,CAAC,8BAA8B;wBAChC,CAAC,CAAC,+BAA+B,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,0BAA0B,QAAQ,uBAAuB,CAAC;YAC3E,CAAC;YAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,YAAY,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,YAAY,EAAE,CAAC;YAClE,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,SAAS;gBAClB,SAAS;gBACT,OAAO,EAAE,YAAY;gBACrB,OAAO;gBACP,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B,KAAK,EAAE;gBAC3C,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,wEAAwE;YACxE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF","sourcesContent":["import OpenAI from \"openai\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport {\n AgentAction,\n AgentResult,\n AgentType,\n AgentExecutionOptions,\n} from \"../types/public/agent.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AgentScreenshotProviderError } from \"../types/public/sdkErrors.js\";\nimport { mapKeyToPlaywright } from \"./utils/cuaKeyMapping.js\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\n\n/*\n Message types for FARA agent\n /\ninterface FaraMessage {\n role: \"system\" \| \"user\" \| \"assistant\";\n content: string \| FaraMessageContent[];\n}\n\ninterface FaraMessageContent {\n type: \"text\" \| \"image_url\";\n text?: string;\n image_url?: {\n url: string; // data:image/png;base64,...\n };\n}\n\n/\n FARA function call structure (parsed from XML tags)\n /\ninterface FaraFunctionCall {\n name: string; // Always \"computer_use\"\n arguments: {\n action: string;\n thoughts?: string;\n [key: string]: unknown;\n };\n}\n\n/\n Client for FARA (Function-based Autonomous Research Agent) by Microsoft\n * This implementation uses OpenAI-compatible API with XML-based tool calling\n /\nexport class MicrosoftCUAClient extends AgentClient {\n private apiKey: string;\n private baseURL: string;\n private client: OpenAI;\n private currentViewport = { width: 1288, height: 711 };\n private currentUrl?: string;\n private screenshotProvider?: () => Promise<string>;\n private actionHandler?: (action: AgentAction) => Promise<void>;\n\n // Dual history system\n private conversationHistory: FaraMessage[] = []; // Conceptual flow\n private actionHistory: FaraMessage[] = []; // Raw model responses\n\n private maxImages: number = 3;\n private temperature: number = 0;\n private facts: string[] = [];\n\n // FARA-specific MLM processor config\n private readonly MLM_PROCESSOR_IM_CFG = {\n min_pixels: 3136,\n max_pixels: 12845056,\n patch_size: 14,\n merge_size: 2,\n };\n\n // Resized dimensions for model input\n private resizedViewport = { width: 1288, height: 711 };\n\n constructor(\n type: AgentType,\n modelName: string,\n userProvidedInstructions?: string,\n clientOptions?: ClientOptions,\n ) {\n super(type, modelName \|\| \"fara-7b\", userProvidedInstructions);\n\n // Process client options\n this.apiKey =\n (clientOptions?.apiKey as string) \|\|\n process.env.AZURE_API_KEY \|\|\n process.env.FIREWORKS_API_KEY \|\|\n \"\";\n this.baseURL =\n (clientOptions?.baseURL as string) \|\|\n process.env.AZURE_ENDPOINT \|\|\n process.env.FIREWORKS_ENDPOINT \|\|\n \"\";\n\n // Store client options for reference\n this.clientOptions = {\n apiKey: this.apiKey,\n baseURL: this.baseURL,\n };\n\n // Validate API key\n if (!this.apiKey \|\| this.apiKey === \"\") {\n throw new Error(\n \"API key is required. Please provide it via clientOptions.apiKey or AZURE_API_KEY or FIREWORKS_API_KEY environment variables.\",\n );\n }\n\n // Initialize the OpenAI client (FARA uses OpenAI-compatible API)\n this.client = new OpenAI({\n apiKey: this.apiKey,\n baseURL: this.baseURL,\n });\n\n // Max images to keep in history\n if (clientOptions?.maxImages !== undefined) {\n this.maxImages = clientOptions.maxImages as number;\n }\n\n // Temperature\n if (clientOptions?.temperature !== undefined) {\n this.temperature = clientOptions.temperature as number;\n }\n }\n\n setViewport(width: number, height: number): void {\n this.currentViewport = { width, height };\n // Compute resized viewport using smart_resize logic\n this.resizedViewport = this.smartResize(width, height);\n }\n\n setCurrentUrl(url: string): void {\n this.currentUrl = url;\n }\n\n setScreenshotProvider(provider: () => Promise<string>): void {\n this.screenshotProvider = provider;\n }\n\n setActionHandler(handler: (action: AgentAction) => Promise<void>): void {\n this.actionHandler = handler;\n }\n\n /\n Smart resize algorithm from FARA\n * Ensures dimensions are divisible by factor and within pixel limits\n /\n private smartResize(\n width: number,\n height: number,\n ): { width: number; height: number } {\n const { patch_size, merge_size, min_pixels, max_pixels } =\n this.MLM_PROCESSOR_IM_CFG;\n const factor = patch_size merge_size;\n\n const roundByFactor = (num: number, f: number) => Math.round(num / f) * f;\n const ceilByFactor = (num: number, f: number) => Math.ceil(num / f) * f;\n const floorByFactor = (num: number, f: number) => Math.floor(num / f) * f;\n\n let h_bar = Math.max(factor, roundByFactor(height, factor));\n let w_bar = Math.max(factor, roundByFactor(width, factor));\n\n if (h_bar * w_bar > max_pixels) {\n const beta = Math.sqrt((height * width) / max_pixels);\n h_bar = floorByFactor(height / beta, factor);\n w_bar = floorByFactor(width / beta, factor);\n } else if (h_bar * w_bar < min_pixels) {\n const beta = Math.sqrt(min_pixels / (height * width));\n h_bar = ceilByFactor(height * beta, factor);\n w_bar = ceilByFactor(width * beta, factor);\n }\n\n return { width: w_bar, height: h_bar };\n }\n\n /*\n Generate system prompt with tool description\n * Simplified to match Python's minimal approach\n /\n private generateSystemPrompt(): string {\n const { width, height } = this.resizedViewport;\n\n // Base prompt - Minimalist like Python\n let basePrompt = \"You are a helpful assistant.\";\n\n // Add user-provided instructions if available\n if (this.userProvidedInstructions) {\n basePrompt = `${basePrompt}\\n\\n${this.userProvidedInstructions}`;\n }\n\n // Tool description from FaraComputerUse\n const toolDescription = `Use a mouse and keyboard to interact with a computer, and take screenshots.\n This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\n* The screen's resolution is ${width}x${height}.\n* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.\n* When a separate scrollable container prominently overlays the webpage, if you want to scroll within it, you typically need to mouse_move() over it first and then scroll().\n* If a popup window appears that you want to close, if left_click() on the 'X' or close button doesn't work, try key(keys=['Escape']) to close it.\n* On some search bars, when you type(), you may need to press_enter=False and instead separately call left_click() on the search button to submit the search query. This is especially true of search bars that have auto-suggest popups for e.g. locations\n* For calendar widgets, you usually need to left_click() on arrows to move between months and left_click() on dates to select them; type() is not typically used to input dates there.`;\n\n // Tool parameters description\n const actionsDescription = `The action to perform. The available actions are:\n* \\`key\\`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order. Includes \"Enter\", \"Alt\", \"Shift\", \"Tab\", \"Control\", \"Backspace\", \"Delete\", \"Escape\", \"ArrowUp\", \"ArrowDown\", \"ArrowLeft\", \"ArrowRight\", \"PageDown\", \"PageUp\", \"Shift\", etc.\n* \\`type\\`: Type a string of text on the keyboard.\n* \\`mouse_move\\`: Move the cursor to a specified (x, y) pixel coordinate on the screen.\n* \\`left_click\\`: Click the left mouse button.\n* \\`scroll\\`: Performs a scroll of the mouse scroll wheel.\n* \\`history_back\\`: Go back to the previous page in the browser history.\n* \\`pause_and_memorize_fact\\`: Pause and memorize a fact for future reference.\n* \\`visit_url\\`: Visit a specified URL.\n* \\`web_search\\`: Perform a web search with a specified query.\n* \\`wait\\`: Wait specified seconds for the change to happen.\n* \\`terminate\\`: Terminate the current task and report its completion status.`;\n\n // Tool JSON schema\n const toolSchema = {\n name: \"computer_use\",\n description: toolDescription,\n parameters: {\n type: \"object\",\n required: [\"action\"],\n properties: {\n action: {\n type: \"string\",\n description: actionsDescription,\n enum: [\n \"key\",\n \"type\",\n \"mouse_move\",\n \"left_click\",\n \"scroll\",\n \"visit_url\",\n \"web_search\",\n \"history_back\",\n \"pause_and_memorize_fact\",\n \"wait\",\n \"terminate\",\n ],\n },\n keys: {\n type: \"array\",\n description: \"Required only by `action=key`.\",\n },\n text: {\n type: \"string\",\n description: \"Required only by `action=type`.\",\n },\n press_enter: {\n type: \"boolean\",\n description:\n \"Whether to press the Enter key after typing. Required only by `action=type`.\",\n },\n delete_existing_text: {\n type: \"boolean\",\n description:\n \"Whether to delete existing text before typing. Required only by `action=type`.\",\n },\n coordinate: {\n type: \"array\",\n description:\n \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=left_click`, `action=mouse_move`, and `action=type`.\",\n },\n pixels: {\n type: \"number\",\n description:\n \"The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll`.\",\n },\n fact: {\n type: \"string\",\n description:\n \"The fact to remember for the future. Required only by `action=pause_and_memorize_fact`.\",\n },\n time: {\n type: \"number\",\n description: \"The seconds to wait. Required only by `action=wait`.\",\n },\n status: {\n type: \"string\",\n description:\n \"The status of the task. Required only by `action=terminate`.\",\n enum: [\"success\", \"failure\"],\n },\n },\n },\n };\n\n // Format as FARA function calling template (FN_CALL_TEMPLATE format)\n const toolDescs = JSON.stringify(toolSchema, null, 2);\n const functionCallTemplate = `\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n${toolDescs}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{{\"name\": <function-name>, \"arguments\": <args-json-object>}}\n</tool_call>`;\n\n return `${basePrompt}\\n\\n${functionCallTemplate}`;\n }\n\n /*\n Parse thoughts and action from model response\n * FARA uses XML-based tool calling: <tool_call>\\n{...}\\n</tool_call>\n /\n private parseThoughtsAndAction(response: string): {\n thoughts: string;\n functionCall: FaraFunctionCall;\n } {\n try {\n const parts = response.split(\"<tool_call>\\n\");\n const thoughts = parts[0].trim();\n const actionText = parts[1].split(\"\\n</tool_call>\")[0].trim();\n\n let parsedAction;\n try {\n parsedAction = JSON.parse(actionText);\n } catch (jsonError) {\n // Fix common malformed JSON: double opening brackets {{\"name\": ...}}\n // This happens when the model adds an extra opening brace\n if (actionText.startsWith(\"{{\") && actionText.endsWith(\"}\")) {\n // Remove the extra opening brace\n const fixedText = actionText.slice(1);\n try {\n parsedAction = JSON.parse(fixedText);\n } catch (retryError) {\n throw new Error(\n `Failed to parse action text even after fixing double brackets. Original: ${actionText}. Fixed: ${fixedText}. Error: ${retryError}`,\n { cause: retryError },\n );\n }\n } else {\n throw new Error(\n `Failed to parse action text as JSON: ${actionText}. Error: ${jsonError}`,\n { cause: jsonError },\n );\n }\n }\n\n return {\n thoughts,\n functionCall: {\n name: parsedAction.name \|\| \"computer_use\",\n arguments: {\n ...parsedAction.arguments,\n thoughts,\n },\n },\n };\n } catch (error) {\n throw new Error(\n `Failed to parse FARA tool call from response: ${response}. Error: ${error}`,\n { cause: error },\n );\n }\n }\n\n /\n Convert FARA function call to Stagehand AgentAction\n /\n private convertFunctionCallToAction(\n functionCall: FaraFunctionCall,\n ): AgentAction {\n const args = functionCall.arguments;\n const action = args.action as string;\n\n // Transform coordinates from resized to original viewport\n const transformCoordinate = (coord: number[]): number[] => {\n if (!coord \|\| coord.length !== 2) return coord;\n const [x, y] = coord;\n const scaleX = this.currentViewport.width / this.resizedViewport.width;\n const scaleY = this.currentViewport.height / this.resizedViewport.height;\n return [Math.round(x scaleX), Math.round(y * scaleY)];\n };\n\n const baseAction = {\n type: action,\n reasoning: args.thoughts as string,\n };\n\n switch (action) {\n case \"left_click\": {\n const clickCoord = transformCoordinate(args.coordinate as number[]);\n return {\n ...baseAction,\n type: \"click\",\n x: clickCoord[0],\n y: clickCoord[1],\n button: \"left\" as const,\n };\n }\n\n case \"mouse_move\": {\n const moveCoord = transformCoordinate(args.coordinate as number[]);\n return {\n ...baseAction,\n type: \"move\",\n coordinate: moveCoord,\n };\n }\n\n case \"type\": {\n const typeCoord = args.coordinate\n ? transformCoordinate(args.coordinate as number[])\n : undefined;\n return {\n ...baseAction,\n text: args.text as string,\n ...(typeCoord && { x: typeCoord[0], y: typeCoord[1] }),\n press_enter:\n args.press_enter !== undefined\n ? (args.press_enter as boolean)\n : true,\n ...(args.delete_existing_text !== undefined && {\n delete_existing_text: args.delete_existing_text as boolean,\n }),\n };\n }\n\n case \"key\":\n case \"keypress\": {\n const keys = (args.keys as string[]) \|\| [];\n // Normalize keys to Playwright format\n const normalizedKeys = keys.map((k) => mapKeyToPlaywright(k));\n return {\n ...baseAction,\n type: \"keypress\",\n keys: normalizedKeys,\n };\n }\n\n case \"scroll\": {\n const pixels = (args.pixels as number) \|\| 0;\n // FARA: positive = scroll up, negative = scroll down\n // Convert to scroll_x/scroll_y\n return {\n ...baseAction,\n scroll_x: 0,\n scroll_y: -pixels, // Invert: negative pixels = scroll down\n };\n }\n\n case \"visit_url\": {\n let url = args.url as string;\n // Enhanced URL processing like Python\n if (\n !url.startsWith(\"https://\") &&\n !url.startsWith(\"http://\") &&\n !url.startsWith(\"file://\") &&\n !url.startsWith(\"about:\")\n ) {\n // If URL contains space, treat as search query\n if (url.includes(\" \")) {\n url = `https://www.bing.com/search?q=${encodeURIComponent(url)}&FORM=QBLH`;\n } else {\n // Otherwise prefix with https://\n url = \"https://\" + url;\n }\n }\n return {\n ...baseAction,\n type: \"goto\",\n url,\n };\n }\n\n case \"web_search\": {\n // Convert web search to visit_url with Bing search\n const query = args.query as string;\n const searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&FORM=QBLH`;\n return {\n ...baseAction,\n type: \"goto\",\n url: searchUrl,\n };\n }\n\n case \"history_back\":\n return {\n ...baseAction,\n type: \"back\",\n };\n\n case \"wait\": {\n // Support both 'time' and 'duration' parameters with default (matches Python)\n const durationSeconds =\n (args.time as number) \|\| (args.duration as number) \|\| 3.0;\n return {\n ...baseAction,\n timeMs: durationSeconds * 1000, // Convert seconds to ms\n };\n }\n\n case \"pause_and_memorize_fact\": {\n // Store the fact for future reference (matches Python)\n const fact = args.fact as string;\n this.facts.push(fact);\n return {\n ...baseAction,\n fact,\n };\n }\n\n case \"terminate\":\n return {\n ...baseAction,\n status: args.status as string,\n };\n\n default:\n return {\n ...baseAction,\n ...args,\n };\n }\n }\n\n /*\n Capture a screenshot and return as base64 data URL\n /\n async captureScreenshot(): Promise<string> {\n if (!this.screenshotProvider) {\n throw new AgentScreenshotProviderError(\"Screenshot provider not set\");\n }\n\n const base64Screenshot = await this.screenshotProvider();\n return `data:image/png;base64,${base64Screenshot}`;\n }\n\n /\n Remove old screenshots from history\n * Matches Python's maybe_remove_old_screenshots\n /\n private maybeRemoveOldScreenshots(\n history: FaraMessage[],\n includesCurrent: boolean = false,\n ): FaraMessage[] {\n if (this.maxImages <= 0) {\n return history;\n }\n\n const maxImages = includesCurrent ? this.maxImages : this.maxImages - 1;\n const newHistory: FaraMessage[] = [];\n let nImages = 0;\n\n // Iterate backwards\n for (let i = history.length - 1; i >= 0; i--) {\n const msg = history[i];\n\n // Check if message has image\n let hasImage = false;\n if (Array.isArray(msg.content)) {\n hasImage = msg.content.some((c) => c.type === \"image_url\");\n }\n\n if (i === 0 && nImages >= maxImages) {\n // First message (task) - preserve text, remove image\n if (Array.isArray(msg.content)) {\n const newContent = msg.content.filter((c) => c.type !== \"image_url\");\n // If no content left, skip (unless it's the only message, but Python logic says continue)\n if (newContent.length === 0) {\n continue;\n }\n newHistory.push({ ...msg, content: newContent });\n } else {\n newHistory.push(msg);\n }\n continue;\n }\n\n if (hasImage) {\n if (nImages < maxImages) {\n newHistory.push(msg);\n nImages++;\n } else {\n // Remove image, keep text\n if (Array.isArray(msg.content)) {\n const newContent = msg.content.filter(\n (c) => c.type !== \"image_url\",\n );\n // If content becomes empty, we can skip this message entirely (unless it's meaningful text)\n // Python logic: if msg is None continue.\n if (newContent.length > 0) {\n newHistory.push({ ...msg, content: newContent });\n }\n } else {\n newHistory.push(msg);\n }\n }\n } else {\n newHistory.push(msg);\n }\n }\n\n return newHistory.reverse();\n }\n\n /\n Reconstruct history for API call\n * Merges conceptual chat history with raw action history\n /\n private reconstructHistory(): FaraMessage[] {\n const history: FaraMessage[] = [];\n let actionTurn = 0;\n\n for (let i = 0; i < this.conversationHistory.length; i++) {\n const m = this.conversationHistory[i];\n if (m.role === \"assistant\") {\n if (actionTurn >= this.actionHistory.length) {\n // Should not happen if synced correctly\n console.warn(\"OUT OF SYNC: Action history shorter than chat history\");\n history.push(m);\n } else {\n history.push(this.actionHistory[actionTurn]);\n actionTurn++;\n }\n } else {\n history.push(m);\n }\n }\n\n return this.maybeRemoveOldScreenshots(history);\n }\n\n /\n Execute a single step\n /\n private async executeStep(\n logger: (message: LogLine) => void,\n isFirstRound: boolean = false,\n ): Promise<{\n actions: AgentAction[];\n completed: boolean;\n usage: {\n input_tokens: number;\n output_tokens: number;\n inference_time_ms: number;\n };\n }> {\n // Capture screenshot\n const screenshotDataUrl = await this.captureScreenshot();\n\n // Update conversation history with new screenshot/message\n if (isFirstRound) {\n // First round: modify the last message (initial user instruction) to include screenshot\n const lastMessage =\n this.conversationHistory[this.conversationHistory.length - 1];\n if (lastMessage && lastMessage.role === \"user\") {\n const originalContent =\n typeof lastMessage.content === \"string\"\n ? lastMessage.content\n : (lastMessage.content.find((c) => c.type === \"text\")?.text ??\n \"Start task\");\n\n lastMessage.content = [\n {\n type: \"image_url\",\n image_url: { url: screenshotDataUrl },\n },\n {\n type: \"text\",\n text: originalContent,\n },\n ];\n }\n } else {\n // Subsequent rounds: add new user message with screenshot\n const userContent: FaraMessageContent[] = [\n {\n type: \"image_url\",\n image_url: { url: screenshotDataUrl },\n },\n ];\n\n // Add current URL if available\n let textPrompt =\n \"Here is the next screenshot. Think about what to do next.\";\n if (this.currentUrl) {\n const trimmedUrl =\n this.currentUrl.length > 100\n ? this.currentUrl.slice(0, 100) + \"...\"\n : this.currentUrl;\n textPrompt = `Current URL: ${trimmedUrl}\\n${textPrompt}`;\n }\n\n userContent.push({\n type: \"text\",\n text: textPrompt,\n });\n\n this.conversationHistory.push({\n role: \"user\",\n content: userContent,\n });\n }\n\n // Reconstruct history for model call\n let history = this.reconstructHistory();\n\n // Prepend system prompt (generated fresh)\n const systemMessage: FaraMessage = {\n role: \"system\",\n content: this.generateSystemPrompt(),\n };\n history = [systemMessage, ...history];\n\n // Make API call\n logger({\n category: \"agent\",\n message: `Making API call to FARA model with ${history.length} messages`,\n level: 2,\n });\n\n const startTime = Date.now();\n let response;\n try {\n response = await this.client.chat.completions.create({\n model: this.modelName,\n messages: history as unknown as ChatCompletionMessageParam[],\n temperature: this.temperature,\n });\n } catch (apiError) {\n logger({\n category: \"agent\",\n message: `API call failed: ${apiError instanceof Error ? apiError.message : String(apiError)}`,\n level: 0,\n });\n throw apiError;\n }\n const inferenceTime = Date.now() - startTime;\n\n logger({\n category: \"agent\",\n message: `API call completed in ${inferenceTime}ms`,\n level: 2,\n });\n\n const content = response.choices[0].message.content \|\| \"\";\n const usage = response.usage \|\| {\n prompt_tokens: 0,\n completion_tokens: 0,\n total_tokens: 0,\n };\n\n // Add assistant response to both histories\n const assistantMsg: FaraMessage = {\n role: \"assistant\",\n content,\n };\n this.conversationHistory.push(assistantMsg);\n this.actionHistory.push(assistantMsg);\n\n logger({\n category: \"agent\",\n message: `Model response: ${content}`,\n level: 2,\n });\n\n // Parse tool call\n const { thoughts, functionCall } = this.parseThoughtsAndAction(content);\n\n logger({\n category: \"agent\",\n message: `Thoughts: ${thoughts}`,\n level: 2,\n });\n\n logger({\n category: \"agent\",\n message: `Action: ${JSON.stringify(functionCall.arguments)}`,\n level: 2,\n });\n\n // Convert to AgentAction\n const agentAction = this.convertFunctionCallToAction(functionCall);\n\n // Expand type action into multiple actions if it has coordinates\n const actions: AgentAction[] = [];\n if (\n agentAction.type === \"type\" &&\n typeof agentAction.x === \"number\" &&\n typeof agentAction.y === \"number\"\n ) {\n // First, click at the coordinates to focus the field\n actions.push({\n type: \"click\",\n x: agentAction.x,\n y: agentAction.y,\n button: \"left\",\n });\n\n // If delete_existing_text is true, clear the field first\n if (agentAction.delete_existing_text) {\n actions.push({\n type: \"keypress\",\n keys: [\"Command+A\"],\n });\n actions.push({\n type: \"keypress\",\n keys: [\"Backspace\"],\n });\n }\n\n // Add the type action (without coordinates since we already clicked)\n actions.push({\n type: \"type\",\n text: agentAction.text,\n });\n\n // If press_enter is true (default), press Enter after typing\n if (agentAction.press_enter !== false) {\n actions.push({\n type: \"keypress\",\n keys: [\"Enter\"],\n });\n }\n } else {\n // For all other actions, just add as-is\n actions.push(agentAction);\n }\n\n // Execute all actions if handler is available\n if (this.actionHandler && agentAction.type !== \"terminate\") {\n for (const action of actions) {\n await this.actionHandler(action);\n }\n }\n\n // Check if completed\n const completed = functionCall.arguments.action === \"terminate\";\n\n return {\n actions,\n completed,\n usage: {\n input_tokens: usage.prompt_tokens,\n output_tokens: usage.completion_tokens,\n inference_time_ms: inferenceTime,\n },\n };\n }\n\n /\n Execute a task with the FARA CUA\n * This is the main entry point for the agent\n * @implements AgentClient.execute\n */\n async execute(executionOptions: AgentExecutionOptions): Promise<AgentResult> {\n const { options, logger } = executionOptions;\n const { instruction } = options;\n const maxSteps = options.maxSteps \|\| 10;\n\n let currentStep = 0;\n let completed = false;\n const actions: AgentAction[] = [];\n const messageList: string[] = [];\n let finalMessage: string;\n let totalInputTokens = 0;\n let totalOutputTokens = 0;\n let totalInferenceTime = 0;\n\n // Initialize conversation with user instruction\n // System prompt is NOT added here, it's added dynamically in executeStep\n this.conversationHistory = [\n {\n role: \"user\",\n content: instruction,\n },\n ];\n this.actionHistory = [];\n\n try {\n // Execute steps until completion or max steps reached\n while (!completed && currentStep < maxSteps) {\n logger({\n category: \"agent\",\n message: `Executing step ${currentStep + 1}/${maxSteps}`,\n level: 1,\n });\n\n const isFirstRound = currentStep === 0;\n const result = await this.executeStep(logger, isFirstRound);\n totalInputTokens += result.usage.input_tokens;\n totalOutputTokens += result.usage.output_tokens;\n totalInferenceTime += result.usage.inference_time_ms;\n\n // Add actions to the list\n actions.push(...result.actions);\n\n // Update completion status\n completed = result.completed;\n\n currentStep++;\n\n // Record message for this step\n const lastAction = result.actions[result.actions.length - 1];\n if (lastAction?.reasoning) {\n messageList.push(lastAction.reasoning);\n }\n }\n\n // Generate final message\n if (completed) {\n const lastAction = actions[actions.length - 1];\n finalMessage =\n (lastAction as { status?: string })?.status === \"success\"\n ? \"Task completed successfully.\"\n : \"Task completed with failures.\";\n } else {\n finalMessage = `Reached maximum steps (${maxSteps}) without completion.`;\n }\n\n if (messageList.length > 0) {\n finalMessage = `${messageList.join(\"\\n\\n\")}\\n\\n${finalMessage}`;\n }\n\n return {\n success: completed,\n completed,\n message: finalMessage,\n actions,\n usage: {\n input_tokens: totalInputTokens,\n output_tokens: totalOutputTokens,\n inference_time_ms: totalInferenceTime,\n },\n };\n } catch (error) {\n logger({\n category: \"agent\",\n message: `Error during execution: ${error}`,\n level: 0,\n });\n\n // Rethrow to allow eval runner's retry logic to handle transient errors\n throw error;\n }\n }\n}\n"]}

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.d.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import type { AgentToolMode } from "../../types/public/agent.js";
+import type { AgentToolMode, Variables } from "../../types/public/agent.js";
 export interface AgentSystemPromptOptions {
     url: string;
     executionInstruction: string;
@@ -8,5 +8,7 @@ export interface AgentSystemPromptOptions {
     isBrowserbase?: boolean;
     /** Tools to exclude from the system prompt */
     excludeTools?: string[];
+    /** Variables available to the agent for use in act/type tools */
+    variables?: Variables;
 }
 export declare function buildAgentSystemPrompt(options: AgentSystemPromptOptions): string;

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js CHANGED Viewed

@@ -71,7 +71,7 @@ function buildToolsSection(isHybridMode, hasSearch, excludeTools) {
     return `<tools>\n${toolLines}\n  </tools>`;
 }
 export function buildAgentSystemPrompt(options) {
-    const { url, executionInstruction, mode, systemInstructions, isBrowserbase = false, excludeTools, } = options;
+    const { url, executionInstruction, mode, systemInstructions, isBrowserbase = false, excludeTools, variables, } = options;
     const localeDate = new Date().toLocaleDateString();
     const isoDate = new Date().toISOString();
     const cdata = (text) => `<![CDATA[${text}]]>`;
@@ -143,6 +143,28 @@ export function buildAgentSystemPrompt(options) {
     const customInstructionsBlock = systemInstructions
         ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\n  `
         : "";
+    // Build variables section only if variables are provided
+    const hasVariables = variables && Object.keys(variables).length > 0;
+    const variableToolsNote = isHybridMode
+        ? "Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields."
+        : "Use %variableName% syntax in the act or fillForm tool's value/action fields.";
+    const variablesSection = hasVariables
+        ? `<variables>
+    <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>
+    <usage>${variableToolsNote}</usage>
+    <example>To type a password, use: type %password% into the password field</example>
+    ${Object.entries(variables)
+            .map(([name, v]) => {
+            const description = typeof v === "object" && v !== null && "value" in v
+                ? v.description
+                : undefined;
+            return description
+                ? `<variable name="${name}">${description}</variable>`
+                : `<variable name="${name}" />`;
+        })
+            .join("\n    ")}
+  </variables>`
+        : "";
     return `<system>
   <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>
   ${customInstructionsBlock}<task>
@@ -174,6 +196,7 @@ export function buildAgentSystemPrompt(options) {
     ${commonStrategyItems}
   </strategy>
   ${roadblocksSection}
+  ${variablesSection}
   <completion>
     <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>
     <examples>

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agentSystemPrompt.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/prompts/agentSystemPrompt.ts"],"names":[],"mappings":"AAwBA,SAAS,iBAAiB,CACxB,YAAqB,EACrB,SAAkB,EAClB,YAAuB;IAEvB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,WAAW,GAAqB;QACpC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,OAAO;YACb,WAAW,EACT,qFAAqF;SACxF;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,2FAA2F;SAC9F;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EACT,0LAA0L;SAC7L;QACD,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE;QAChE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,8BAA8B,EAAE;QACrE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,mCAAmC;SACjD;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gDAAgD;SAC9D;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExD,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,8IAA8I;SACjJ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,MAAM,SAAS,GAAG,aAAa;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,SAAS,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,YAAY,SAAS,cAAc,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAiC;IAEjC,MAAM,EACJ,GAAG,EACH,oBAAoB,EACpB,IAAI,EACJ,kBAAkB,EAClB,aAAa,GAAG,KAAK,EACrB,YAAY,GACb,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,IAAI,KAAK,CAAC;IAEtD,MAAM,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC;IACvC,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAErD,yDAAyD;IACzD,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9E,iCAAiC;IACjC,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC;YACE,uIAAuI;YACvI,4GAA4G;YAC5G,sIAAsI;YACtI,uHAAuH;YACvH,sFAAsF;SACvF;QACH,CAAC,CAAC;YACE,2FAA2F;YAC3F,oJAAoJ;YACpJ,6HAA6H;YAC7H,mIAAmI;YACnI,wHAAwH;SACzH,CAAC;IAEN,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,mBAAmB,GAAG;sNACwL,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU;;;;KAIrQ,CAAC;IAEJ,oDAAoD;IACpD,MAAM,yBAAyB,GAAG,YAAY;QAC5C,CAAC,CAAC;;;;;;;;;;;;;iCAa2B;QAC7B,CAAC,CAAC;;;;;;;;;;;;;iCAa2B,CAAC;IAEhC,iFAAiF;IACjF,MAAM,iBAAiB,GAAG,aAAa;QACrC,CAAC,CAAC;;;gBAGU;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,kBAAkB;QAChD,CAAC,CAAC,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,2BAA2B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;IAEL,uBAAuB;YACf,KAAK,CAAC,oBAAoB,CAAC;iCACN,OAAO,KAAK,UAAU;;;;2DAII,GAAG;;;;;;;;;;;;IAY1D,yBAAyB;;;MAGvB,SAAS,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE;;IAEtG,YAAY;;MAEV,eAAe;MACf,mBAAmB;;IAErB,iBAAiB;;;;;;;;UAQX,CAAC;AACX,CAAC","sourcesContent":["import type { AgentToolMode } from \"../../types/public/agent.js\";\n\nexport interface AgentSystemPromptOptions {\n url: string;\n executionInstruction: string;\n mode: AgentToolMode;\n systemInstructions?: string;\n /** Whether running on Browserbase (enables captcha solver messaging) /\n isBrowserbase?: boolean;\n /* Tools to exclude from the system prompt /\n excludeTools?: string[];\n}\n\n/\n Builds the system prompt for the agent based on the tool mode.\n \n @param options - The prompt configuration options\n * @returns The formatted system prompt string\n */\ninterface ToolDefinition {\n name: string;\n description: string;\n}\n\nfunction buildToolsSection(\n isHybridMode: boolean,\n hasSearch: boolean,\n excludeTools?: string[],\n): string {\n const excludeSet = new Set(excludeTools ?? []);\n\n const hybridTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"click\",\n description:\n \"Click on an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"type\",\n description:\n \"Type text into an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"act\",\n description:\n \"Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.\",\n },\n { name: \"dragAndDrop\", description: \"Drag and drop an element\" },\n { name: \"clickAndHold\", description: \"Click and hold on an element\" },\n { name: \"keys\", description: \"Press a keyboard key\" },\n {\n name: \"fillFormVision\",\n description: \"Fill out a form using coordinates\",\n },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const domTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"act\",\n description: \"Perform a specific atomic action (click, type)\",\n },\n { name: \"keys\", description: \"Press a keyboard key\" },\n { name: \"fillForm\", description: \"Fill out a form\" },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const baseTools = isHybridMode ? hybridTools : domTools;\n\n if (hasSearch) {\n baseTools.push({\n name: \"search\",\n description:\n \"Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.\",\n });\n }\n\n const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));\n\n const toolLines = filteredTools\n .map((tool) => ` <tool name=\"${tool.name}\">${tool.description}</tool>`)\n .join(\"\\n\");\n\n return `<tools>\\n${toolLines}\\n </tools>`;\n}\n\nexport function buildAgentSystemPrompt(\n options: AgentSystemPromptOptions,\n): string {\n const {\n url,\n executionInstruction,\n mode,\n systemInstructions,\n isBrowserbase = false,\n excludeTools,\n } = options;\n const localeDate = new Date().toLocaleDateString();\n const isoDate = new Date().toISOString();\n const cdata = (text: string) => `<![CDATA[${text}]]>`;\n\n const isHybridMode = mode === \"hybrid\";\n const hasSearch = Boolean(process.env.BRAVE_API_KEY);\n\n // Tools section differs based on mode and excluded tools\n const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);\n\n // Strategy differs based on mode\n const strategyItems = isHybridMode\n ? [\n `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,\n `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,\n `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,\n `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,\n `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,\n ]\n : [\n `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,\n `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,\n `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,\n `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,\n `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,\n ];\n\n const strategySection = strategyItems.join(\"\\n \");\n\n const commonStrategyItems = `\n <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., \"get job listings\", \"extract product details\"). For reading page content or understanding elements, always use ${isHybridMode ? \"screenshot or ariaTree\" : \"ariaTree\"} instead - it's faster and more reliable.</item>\n <item>Keep actions atomic and verify outcomes before proceeding.</item>\n <item>For each action, provide clear reasoning about why you're taking that step.</item>\n <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>\n `;\n\n // Page understanding protocol differs based on mode\n const pageUnderstandingProtocol = isHybridMode\n ? `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </primary_tool>\n <secondary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`\n : `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </primary_tool>\n <secondary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`;\n\n // Roadblocks section only shown when running on Browserbase (has captcha solver)\n const roadblocksSection = isBrowserbase\n ? `<roadblocks>\n <note>captchas, popups, etc.</note>\n <captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>\n </roadblocks>`\n : \"\";\n\n // Build customInstructions block only if provided\n const customInstructionsBlock = systemInstructions\n ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\\n `\n : \"\";\n\n return `<system>\n <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>\n ${customInstructionsBlock}<task>\n <goal>${cdata(executionInstruction)}</goal>\n <date display=\"local\" iso=\"${isoDate}\">${localeDate}</date>\n <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>\n </task>\n <page>\n <startingUrl>you are starting your task on this url: ${url}</startingUrl>\n </page>\n <mindset>\n <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>\n <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>\n <note>When the task is complete, do not seek more information; you have completed the task.</note>\n </mindset>\n <guidelines>\n <item>Always start by understanding the current page state</item>\n <item>Use the screenshot tool to verify page state when needed</item>\n <item>Use appropriate tools for each action</item>\n </guidelines>\n ${pageUnderstandingProtocol}\n <navigation>\n <rule>If you are confident in the URL, navigate directly to it.</rule>\n ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}\n </navigation>\n ${toolsSection}\n <strategy>\n ${strategySection}\n ${commonStrategyItems}\n </strategy>\n ${roadblocksSection}\n <completion>\n <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>\n <examples>\n <example>If you were asked for specific flights, list the flights you found.</example>\n <example>If you were asked for information about a product, list the product information you were asked for.</example>\n </examples>\n </completion>\n</system>`;\n}\n"]}
1	+ {"version":3,"file":"agentSystemPrompt.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/prompts/agentSystemPrompt.ts"],"names":[],"mappings":"AA0BA,SAAS,iBAAiB,CACxB,YAAqB,EACrB,SAAkB,EAClB,YAAuB;IAEvB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,WAAW,GAAqB;QACpC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,OAAO;YACb,WAAW,EACT,qFAAqF;SACxF;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,2FAA2F;SAC9F;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EACT,0LAA0L;SAC7L;QACD,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE;QAChE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,8BAA8B,EAAE;QACrE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,mCAAmC;SACjD;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gDAAgD;SAC9D;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExD,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,8IAA8I;SACjJ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,MAAM,SAAS,GAAG,aAAa;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,SAAS,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,YAAY,SAAS,cAAc,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAiC;IAEjC,MAAM,EACJ,GAAG,EACH,oBAAoB,EACpB,IAAI,EACJ,kBAAkB,EAClB,aAAa,GAAG,KAAK,EACrB,YAAY,EACZ,SAAS,GACV,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,IAAI,KAAK,CAAC;IAEtD,MAAM,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC;IACvC,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAErD,yDAAyD;IACzD,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9E,iCAAiC;IACjC,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC;YACE,uIAAuI;YACvI,4GAA4G;YAC5G,sIAAsI;YACtI,uHAAuH;YACvH,sFAAsF;SACvF;QACH,CAAC,CAAC;YACE,2FAA2F;YAC3F,oJAAoJ;YACpJ,6HAA6H;YAC7H,mIAAmI;YACnI,wHAAwH;SACzH,CAAC;IAEN,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,mBAAmB,GAAG;sNACwL,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU;;;;KAIrQ,CAAC;IAEJ,oDAAoD;IACpD,MAAM,yBAAyB,GAAG,YAAY;QAC5C,CAAC,CAAC;;;;;;;;;;;;;iCAa2B;QAC7B,CAAC,CAAC;;;;;;;;;;;;;iCAa2B,CAAC;IAEhC,iFAAiF;IACjF,MAAM,iBAAiB,GAAG,aAAa;QACrC,CAAC,CAAC;;;gBAGU;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,kBAAkB;QAChD,CAAC,CAAC,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,2BAA2B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,yDAAyD;IACzD,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,gGAAgG;QAClG,CAAC,CAAC,8EAA8E,CAAC;IACnF,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC;;aAEO,iBAAiB;;MAExB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;aACxB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE;YACjB,MAAM,WAAW,GACf,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC;gBACjD,CAAC,CAAC,CAAC,CAAC,WAAW;gBACf,CAAC,CAAC,SAAS,CAAC;YAChB,OAAO,WAAW;gBAChB,CAAC,CAAC,mBAAmB,IAAI,KAAK,WAAW,aAAa;gBACtD,CAAC,CAAC,mBAAmB,IAAI,MAAM,CAAC;QACpC,CAAC,CAAC;aACD,IAAI,CAAC,QAAQ,CAAC;eACN;QACX,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;IAEL,uBAAuB;YACf,KAAK,CAAC,oBAAoB,CAAC;iCACN,OAAO,KAAK,UAAU;;;;2DAII,GAAG;;;;;;;;;;;;IAY1D,yBAAyB;;;MAGvB,SAAS,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE;;IAEtG,YAAY;;MAEV,eAAe;MACf,mBAAmB;;IAErB,iBAAiB;IACjB,gBAAgB;;;;;;;;UAQV,CAAC;AACX,CAAC","sourcesContent":["import type { AgentToolMode, Variables } from \"../../types/public/agent.js\";\n\nexport interface AgentSystemPromptOptions {\n url: string;\n executionInstruction: string;\n mode: AgentToolMode;\n systemInstructions?: string;\n /** Whether running on Browserbase (enables captcha solver messaging) /\n isBrowserbase?: boolean;\n /* Tools to exclude from the system prompt /\n excludeTools?: string[];\n /* Variables available to the agent for use in act/type tools /\n variables?: Variables;\n}\n\n/\n Builds the system prompt for the agent based on the tool mode.\n \n @param options - The prompt configuration options\n * @returns The formatted system prompt string\n */\ninterface ToolDefinition {\n name: string;\n description: string;\n}\n\nfunction buildToolsSection(\n isHybridMode: boolean,\n hasSearch: boolean,\n excludeTools?: string[],\n): string {\n const excludeSet = new Set(excludeTools ?? []);\n\n const hybridTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"click\",\n description:\n \"Click on an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"type\",\n description:\n \"Type text into an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"act\",\n description:\n \"Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.\",\n },\n { name: \"dragAndDrop\", description: \"Drag and drop an element\" },\n { name: \"clickAndHold\", description: \"Click and hold on an element\" },\n { name: \"keys\", description: \"Press a keyboard key\" },\n {\n name: \"fillFormVision\",\n description: \"Fill out a form using coordinates\",\n },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const domTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"act\",\n description: \"Perform a specific atomic action (click, type)\",\n },\n { name: \"keys\", description: \"Press a keyboard key\" },\n { name: \"fillForm\", description: \"Fill out a form\" },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const baseTools = isHybridMode ? hybridTools : domTools;\n\n if (hasSearch) {\n baseTools.push({\n name: \"search\",\n description:\n \"Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.\",\n });\n }\n\n const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));\n\n const toolLines = filteredTools\n .map((tool) => ` <tool name=\"${tool.name}\">${tool.description}</tool>`)\n .join(\"\\n\");\n\n return `<tools>\\n${toolLines}\\n </tools>`;\n}\n\nexport function buildAgentSystemPrompt(\n options: AgentSystemPromptOptions,\n): string {\n const {\n url,\n executionInstruction,\n mode,\n systemInstructions,\n isBrowserbase = false,\n excludeTools,\n variables,\n } = options;\n const localeDate = new Date().toLocaleDateString();\n const isoDate = new Date().toISOString();\n const cdata = (text: string) => `<![CDATA[${text}]]>`;\n\n const isHybridMode = mode === \"hybrid\";\n const hasSearch = Boolean(process.env.BRAVE_API_KEY);\n\n // Tools section differs based on mode and excluded tools\n const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);\n\n // Strategy differs based on mode\n const strategyItems = isHybridMode\n ? [\n `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,\n `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,\n `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,\n `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,\n `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,\n ]\n : [\n `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,\n `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,\n `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,\n `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,\n `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,\n ];\n\n const strategySection = strategyItems.join(\"\\n \");\n\n const commonStrategyItems = `\n <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., \"get job listings\", \"extract product details\"). For reading page content or understanding elements, always use ${isHybridMode ? \"screenshot or ariaTree\" : \"ariaTree\"} instead - it's faster and more reliable.</item>\n <item>Keep actions atomic and verify outcomes before proceeding.</item>\n <item>For each action, provide clear reasoning about why you're taking that step.</item>\n <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>\n `;\n\n // Page understanding protocol differs based on mode\n const pageUnderstandingProtocol = isHybridMode\n ? `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </primary_tool>\n <secondary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`\n : `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </primary_tool>\n <secondary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`;\n\n // Roadblocks section only shown when running on Browserbase (has captcha solver)\n const roadblocksSection = isBrowserbase\n ? `<roadblocks>\n <note>captchas, popups, etc.</note>\n <captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>\n </roadblocks>`\n : \"\";\n\n // Build customInstructions block only if provided\n const customInstructionsBlock = systemInstructions\n ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\\n `\n : \"\";\n\n // Build variables section only if variables are provided\n const hasVariables = variables && Object.keys(variables).length > 0;\n const variableToolsNote = isHybridMode\n ? \"Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields.\"\n : \"Use %variableName% syntax in the act or fillForm tool's value/action fields.\";\n const variablesSection = hasVariables\n ? `<variables>\n <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>\n <usage>${variableToolsNote}</usage>\n <example>To type a password, use: type %password% into the password field</example>\n ${Object.entries(variables)\n .map(([name, v]) => {\n const description =\n typeof v === \"object\" && v !== null && \"value\" in v\n ? v.description\n : undefined;\n return description\n ? `<variable name=\"${name}\">${description}</variable>`\n : `<variable name=\"${name}\" />`;\n })\n .join(\"\\n \")}\n </variables>`\n : \"\";\n\n return `<system>\n <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>\n ${customInstructionsBlock}<task>\n <goal>${cdata(executionInstruction)}</goal>\n <date display=\"local\" iso=\"${isoDate}\">${localeDate}</date>\n <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>\n </task>\n <page>\n <startingUrl>you are starting your task on this url: ${url}</startingUrl>\n </page>\n <mindset>\n <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>\n <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>\n <note>When the task is complete, do not seek more information; you have completed the task.</note>\n </mindset>\n <guidelines>\n <item>Always start by understanding the current page state</item>\n <item>Use the screenshot tool to verify page state when needed</item>\n <item>Use appropriate tools for each action</item>\n </guidelines>\n ${pageUnderstandingProtocol}\n <navigation>\n <rule>If you are confident in the URL, navigate directly to it.</rule>\n ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}\n </navigation>\n ${toolsSection}\n <strategy>\n ${strategySection}\n ${commonStrategyItems}\n </strategy>\n ${roadblocksSection}\n ${variablesSection}\n <completion>\n <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>\n <examples>\n <example>If you were asked for specific flights, list the flights you found.</example>\n <example>If you were asked for information about a product, list the product information you were asked for.</example>\n </examples>\n </completion>\n</system>`;\n}\n"]}

package/dist/esm/lib/v3/agent/tools/act.d.ts CHANGED Viewed

@@ -1,7 +1,7 @@
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
-import type { AgentModelConfig } from "../../types/public/agent.js";
-export declare const actTool: (v3: V3, executionModel?: string | AgentModelConfig) => import("ai").Tool<{
+import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
+export declare const actTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
     action: string;
 }, {
     success: boolean;

package/dist/esm/lib/v3/agent/tools/act.js CHANGED Viewed

@@ -1,49 +1,71 @@
 import { tool } from "ai";
 import { z } from "zod";
-export const actTool = (v3, executionModel) => tool({
-    description: "Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.",
-    inputSchema: z.object({
-        action: z
-            .string()
-            .describe('Describe what to click or type, e.g. "click the Login button" or "type "John" into the first name input"'),
-    }),
-    execute: async ({ action }) => {
-        try {
-            v3.logger({
-                category: "agent",
-                message: `Agent calling tool: act`,
-                level: 1,
-                auxiliary: {
-                    arguments: {
-                        value: action,
-                        type: "string",
+import { TimeoutError } from "../../types/public/sdkErrors.js";
+export const actTool = (v3, executionModel, variables, toolTimeout) => {
+    const hasVariables = variables && Object.keys(variables).length > 0;
+    const actionDescription = hasVariables
+        ? `Describe what to click or type, e.g. "click the Login button" or "type %variableName% into the input". Available variables: ${Object.keys(variables).join(", ")}`
+        : 'Describe what to click or type, e.g. "click the Login button" or "type "John" into the first name input"';
+    return tool({
+        description: "Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.",
+        inputSchema: z.object({
+            action: z.string().describe(actionDescription),
+        }),
+        execute: async ({ action }) => {
+            try {
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: act`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: action,
+                            type: "string",
+                        },
                     },
-                },
-            });
-            const options = executionModel ? { model: executionModel } : undefined;
-            const result = await v3.act(action, options);
-            const actions = result.actions ?? [];
-            v3.recordAgentReplayStep({
-                type: "act",
-                instruction: action,
-                actions,
-                actionDescription: result.actionDescription,
-                message: result.message,
-            });
-            // Only include playwrightArguments when actions exist
-            // (undefined is not valid JSON and breaks AI SDK validation)
-            const response = {
-                success: result.success ?? true,
-                action: result?.actionDescription ?? action,
-            };
-            if (actions.length > 0) {
-                response.playwrightArguments = actions[0];
+                });
+                const options = executionModel
+                    ? { model: executionModel, variables, timeout: toolTimeout }
+                    : { variables, timeout: toolTimeout };
+                const result = await v3.act(action, options);
+                const actions = result.actions ?? [];
+                v3.recordAgentReplayStep({
+                    type: "act",
+                    instruction: action,
+                    actions,
+                    actionDescription: result.actionDescription,
+                    message: result.message,
+                });
+                // Only include playwrightArguments when actions exist
+                // (undefined is not valid JSON and breaks AI SDK validation)
+                const response = {
+                    success: result.success ?? true,
+                    action: result?.actionDescription ?? action,
+                };
+                if (actions.length > 0) {
+                    response.playwrightArguments = actions[0];
+                }
+                return response;
             }
-            return response;
-        }
-        catch (error) {
-            return { success: false, error: error?.message ?? String(error) };
-        }
-    },
-});
+            catch (error) {
+                if (error instanceof TimeoutError) {
+                    const timeoutMessage = `TimeoutError while waiting for act() to complete (it may continue executing in the background)`;
+                    v3.logger({
+                        category: "agent",
+                        message: timeoutMessage,
+                        level: 0,
+                    });
+                    return {
+                        success: false,
+                        error: `${timeoutMessage} — try using a different description for the action`,
+                    };
+                }
+                return {
+                    success: false,
+                    error: error?.message ?? String(error),
+                };
+            }
+        },
+    });
+};
 //# sourceMappingURL=act.js.map

package/dist/esm/lib/v3/agent/tools/act.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"act.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/act.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;~~AAKxB~~,MAAM,CAAC,MAAM,OAAO,GAAG,~~CAAC~~,EAAM,~~EAAE~~,cAA0C,EAAE,EAAE,~~CAC5E~~,IAAI,CAAC;~~IACH~~,WAAW,EACT,+GAA+G;~~IACjH~~,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;~~QACpB~~,MAAM,EAAE,CAAC~~;aACN~~,MAAM,EAAE~~;aACR~~,QAAQ,~~CACP~~,~~0GAA0G~~,~~CAC3G~~;~~KACJ~~,CAAC;~~IACF~~,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;~~QAC5B~~,IAAI,CAAC;~~YACH~~,EAAE,CAAC,MAAM,CAAC;~~gBACR~~,QAAQ,EAAE,OAAO;~~gBACjB~~,OAAO,EAAE,yBAAyB;~~gBAClC~~,KAAK,EAAE,CAAC;~~gBACR~~,SAAS,EAAE;~~oBACT~~,SAAS,EAAE;~~wBACT~~,KAAK,EAAE,MAAM;~~wBACb~~,IAAI,EAAE,QAAQ;~~qBACf~~;iBACF~~;aACF~~,CAAC,CAAC;~~YACH~~,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,~~CAAC,~~EAAE,KAAK,EAAE,cAAc,EAAE,~~CAAC~~,CAAC,CAAC,SAAS,CAAC;~~YACvE~~,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;~~YAC7C~~,MAAM,OAAO,GAAI,MAAM,CAAC,OAAgC,IAAI,EAAE,CAAC;~~YAC~~/D,EAAE,CAAC,qBAAqB,CAAC;~~gBACvB~~,IAAI,EAAE,KAAK;~~gBACX~~,WAAW,EAAE,MAAM;~~gBACnB~~,OAAO;~~gBACP~~,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;~~gBAC3C~~,OAAO,EAAE,MAAM,CAAC,OAAO;~~aACxB~~,CAAC,CAAC;~~YACH~~,sDAAsD;~~YACtD~~,6DAA6D;~~YAC7D~~,MAAM,QAAQ,GAIV;~~gBACF~~,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;~~gBAC~~/B,MAAM,EAAE,MAAM,EAAE,iBAAiB,IAAI,MAAM;~~aAC5C~~,CAAC;~~YACF~~,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;~~gBACvB~~,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;~~YAC5C~~,CAAC;~~YACD~~,OAAO,QAAQ,CAAC;~~QAClB~~,CAAC;~~QAAC~~,OAAO,KAAK,EAAE,CAAC;~~YACf~~,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,~~EAAE~~,CAAC;~~QACpE~~,CAAC;~~IACH~~,CAAC;~~CACF~~,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\n\nexport const actTool = (v3: V3, executionModel?: string \| AgentModelConfig) ~~=>\~~n ~~tool~~({\n ~~description:\n \"Perform~~ an ~~action~~ on ~~the~~ ~~page~~ ~~(click~~, ~~type)~~. ~~Provide~~ a ~~short,~~ ~~specific~~ ~~phrase~~ ~~that~~ ~~mentions~~ the ~~element~~ ~~type.\",\n inputSchema~~: ~~z.object(~~{~~\n action: z\n~~ .~~string~~()\n .~~describe~~(\n 'Describe what to click or type, e.g. \"click the Login button\" or \"type \"John\" into the first name input\"',\n ),\n }),\n execute: async ({ action }) => {\n try {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: act`,\n level: 1,\n auxiliary: {\n arguments: {\n value: action,\n type: \"string\",\n },\n },\n });\n const options = executionModel ? { model: executionModel } : ~~undefined~~;\n const result = await v3.act(action, options);\n const actions = (result.actions as Action[] \| undefined) ?? [];\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: action,\n actions,\n actionDescription: result.actionDescription,\n message: result.message,\n });\n // Only include playwrightArguments when actions exist\n // (undefined is not valid JSON and breaks AI SDK validation)\n const response: {\n success: boolean;\n action: string;\n playwrightArguments?: Action;\n } = {\n success: result.success ?? true,\n action: result?.actionDescription ?? action,\n };\n if (actions.length > 0) {\n response.playwrightArguments = actions[0];\n }\n return response;\n } catch (error) {\n return { success: false, error: error?.message ?? String(error) };\n }\n },\n });\n"]}
1	+ {"version":3,"file":"act.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/act.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,+HAA+H,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpK,CAAC,CAAC,0GAA0G,CAAC;IAE/G,OAAO,IAAI,CAAC;QACV,WAAW,EACT,+GAA+G;QACjH,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;SAC/C,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,yBAAyB;oBAClC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,MAAM;4BACb,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,cAAc;oBAC5B,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;oBAC5D,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAI,MAAM,CAAC,OAAgC,IAAI,EAAE,CAAC;gBAC/D,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,KAAK;oBACX,WAAW,EAAE,MAAM;oBACnB,OAAO;oBACP,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;oBAC3C,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC;gBACH,sDAAsD;gBACtD,6DAA6D;gBAC7D,MAAM,QAAQ,GAIV;oBACF,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;oBAC/B,MAAM,EAAE,MAAM,EAAE,iBAAiB,IAAI,MAAM;iBAC5C,CAAC;gBACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5C,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,cAAc,GAAG,gGAAgG,CAAC;oBACxH,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc;wBACvB,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,GAAG,cAAc,qDAAqD;qBAC9E,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const actTool = (\n v3: V3,\n executionModel?: string \| AgentModelConfig,\n variables?: Variables,\n toolTimeout?: number,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const actionDescription = hasVariables\n ? `Describe what to click or type, e.g. \"click the Login button\" or \"type %variableName% into the input\". Available variables: ${Object.keys(variables).join(\", \")}`\n : 'Describe what to click or type, e.g. \"click the Login button\" or \"type \"John\" into the first name input\"';\n\n return tool({\n description:\n \"Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.\",\n inputSchema: z.object({\n action: z.string().describe(actionDescription),\n }),\n execute: async ({ action }) => {\n try {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: act`,\n level: 1,\n auxiliary: {\n arguments: {\n value: action,\n type: \"string\",\n },\n },\n });\n const options = executionModel\n ? { model: executionModel, variables, timeout: toolTimeout }\n : { variables, timeout: toolTimeout };\n\n const result = await v3.act(action, options);\n const actions = (result.actions as Action[] \| undefined) ?? [];\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: action,\n actions,\n actionDescription: result.actionDescription,\n message: result.message,\n });\n // Only include playwrightArguments when actions exist\n // (undefined is not valid JSON and breaks AI SDK validation)\n const response: {\n success: boolean;\n action: string;\n playwrightArguments?: Action;\n } = {\n success: result.success ?? true,\n action: result?.actionDescription ?? action,\n };\n if (actions.length > 0) {\n response.playwrightArguments = actions[0];\n }\n return response;\n } catch (error) {\n if (error instanceof TimeoutError) {\n const timeoutMessage = `TimeoutError while waiting for act() to complete (it may continue executing in the background)`;\n v3.logger({\n category: \"agent\",\n message: timeoutMessage,\n level: 0,\n });\n return {\n success: false,\n error: `${timeoutMessage} — try using a different description for the action`,\n };\n }\n return {\n success: false,\n error: error?.message ?? String(error),\n };\n }\n },\n });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/ariaTree.d.ts CHANGED Viewed

@@ -1,5 +1,12 @@
 import type { V3 } from "../../v3.js";
-export declare const ariaTreeTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+export declare const ariaTreeTool: (v3: V3, toolTimeout?: number) => import("ai").Tool<Record<string, never>, {
+    success: boolean;
     content: string;
     pageUrl: string;
+    error?: undefined;
+} | {
+    content: string;
+    error: any;
+    success: boolean;
+    pageUrl: string;
 }>;