@browserbasehq/orca 3.2.0-preview.4 → 3.2.0-preview.5

package/dist/esm/lib/v3/agent/OpenAICUAClient.js

@@ -7,7 +7,10 @@ import { v7 as uuidv7 } from "uuid";
  * Client for OpenAI's Computer Use Assistant API
  * This implementation uses the official OpenAI Responses API for Computer Use
  */
+const CAPTCHA_PROCEED_TOOL = "captchaSolvedProceed";
 export class OpenAICUAClient extends AgentClient {
+    pendingContextNotes = [];
+    captchaSolvedToolActive = false;
     apiKey;
     organization;
     baseURL;
@@ -63,6 +66,15 @@ export class OpenAICUAClient extends AgentClient {
     setSafetyConfirmationHandler(handler) {
         this.safetyConfirmationHandler = handler;
     }
+    addContextNote(note) {
+        this.pendingContextNotes.push(note);
+        // When a captcha-related note arrives, expose a tool that the model can
+        // call instead of asking the user for confirmation.  This replaces
+        // fragile English-phrase parsing with a structured tool call.
+        if (note.toLowerCase().includes("captcha")) {
+            this.captchaSolvedToolActive = true;
+        }
+    }
     /**
      * Execute a task with the OpenAI CUA
      * This is the main entry point for the agent
@@ -87,6 +99,7 @@ export class OpenAICUAClient extends AgentClient {
         try {
             // Execute steps until completion or max steps reached
             while (!completed && currentStep < maxSteps) {
+                await this.preStepHook?.();
                 logger({
                     category: "agent",
                     message: `Executing step ${currentStep + 1}/${maxSteps}`,
@@ -105,6 +118,16 @@ export class OpenAICUAClient extends AgentClient {
                 // Update the input items for the next step if we're continuing
                 if (!completed) {
                     inputItems = result.nextInputItems;
+                    const contextNotes = this.drainContextNotes();
+                    if (contextNotes.length > 0) {
+                        inputItems = [
+                            ...inputItems,
+                            ...contextNotes.map((note) => ({
+                                role: "user",
+                                content: note,
+                            })),
+                        ];
+                    }
                 }
                 // Record any message for this step
                 if (result.message) {
@@ -179,7 +202,7 @@ export class OpenAICUAClient extends AgentClient {
                 if (item.type === "computer_call" && this.isComputerCallItem(item)) {
                     logger({
                         category: "agent",
-                        message: `Found computer_call: ${item.action.type}, call_id: ${item.call_id}`,
+                        message: `Found computer_call: ${item.action.type}, payload: ${JSON.stringify(item.action)}, call_id: ${item.call_id}`,
                         level: 2,
                     });
                     const action = this.convertComputerCallToAction(item);
@@ -346,6 +369,25 @@ export class OpenAICUAClient extends AgentClient {
                     ...customTools,
                 ];
             }
+            // When a captcha was just solved, expose a tool the model can call
+            // to confirm it should proceed.  This avoids fragile English-phrase
+            // parsing and works regardless of the model's output language.
+            if (this.captchaSolvedToolActive) {
+                requestParams.tools = [
+                    ...requestParams.tools,
+                    {
+                        type: "function",
+                        name: CAPTCHA_PROCEED_TOOL,
+                        function: {
+                            name: CAPTCHA_PROCEED_TOOL,
+                            description: "The captcha on this page was solved automatically. " +
+                                "Call this tool to confirm and continue with your task " +
+                                "instead of asking the user for permission.",
+                            parameters: { type: "object", properties: {}, required: [] },
+                        },
+                    },
+                ];
+            }
             // Add previous_response_id if available
             if (previousResponseId) {
                 requestParams.previous_response_id = previousResponseId;
@@ -498,6 +540,17 @@ export class OpenAICUAClient extends AgentClient {
             }
             else if (item.type === "function_call" &&
                 this.isFunctionCallItem(item)) {
+                // Handle the captcha-proceed tool — just return a confirmation and
+                // deactivate the tool so it doesn't appear on subsequent steps.
+                if (item.name === CAPTCHA_PROCEED_TOOL) {
+                    this.captchaSolvedToolActive = false;
+                    nextInputItems.push({
+                        type: "function_call_output",
+                        call_id: item.call_id,
+                        output: "Confirmed. The captcha is solved. Continue completing the original task autonomously without asking for further confirmation.",
+                    });
+                    continue;
+                }
                 // Handle function calls (tool calls)
                 try {
                     const action = this.convertFunctionCallToAction(item);
@@ -577,6 +630,14 @@ export class OpenAICUAClient extends AgentClient {
             ...action, // Spread all properties from the action
         };
     }
+    drainContextNotes() {
+        if (this.pendingContextNotes.length === 0) {
+            return [];
+        }
+        const notes = [...this.pendingContextNotes];
+        this.pendingContextNotes = [];
+        return notes;
+    }
     convertFunctionCallToAction(call) {
         try {
             const args = JSON.parse(call.arguments);

package/dist/esm/lib/v3/agent/OpenAICUAClient.js.map

@@ -1 +1 @@
-{"version":3,"file":"OpenAICUAClient.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/OpenAICUAClient.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAe5B,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EACL,4BAA4B,EAC5B,oBAAoB,GACrB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,UAAU,EACV,0BAA0B,EAC1B,4BAA4B,GAC7B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAEpC;;;GAGG;AACH,MAAM,OAAO,eAAgB,SAAQ,WAAW;IACtC,MAAM,CAAS;IACf,YAAY,CAAU;IACtB,OAAO,CAAS;IAChB,MAAM,CAAS;IAChB,cAAc,CAAU;IACvB,eAAe,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC/C,UAAU,CAAU;IACpB,kBAAkB,CAAyB;IAC3C,aAAa,CAA0C;IACvD,cAAc,GAA8B,IAAI,GAAG,EAAE,CAAC;IACtD,WAAW,GAAW,SAAS,CAAC,CAAC,2CAA2C;IAC5E,KAAK,CAAW;IAChB,yBAAyB,CAA6B;IAE9D,YACE,IAAe,EACf,SAAiB,EACjB,wBAAiC,EACjC,aAA6B,EAC7B,KAAe;QAEf,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,wBAAwB,CAAC,CAAC;QAEjD,yBAAyB;QACzB,IAAI,CAAC,MAAM;YACR,aAAa,EAAE,MAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,OAAO,GAAI,aAAa,EAAE,OAAkB,IAAI,SAAS,CAAC;QAC/D,IAAI,CAAC,YAAY;YACd,aAAa,EAAE,YAAuB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QAEpE,+BAA+B;QAC/B,IACE,aAAa,EAAE,WAAW;YAC1B,OAAO,aAAa,CAAC,WAAW,KAAK,QAAQ,EAC7C,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,aAAa,CAAC,WAAW,CAAC;QAC/C,CAAC;QAED,qCAAqC;QACrC,IAAI,CAAC,aAAa,GAAG;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,aAAa,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5C,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,MAAc;QACvC,IAAI,CAAC,eAAe,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACxB,CAAC;IAED,qBAAqB,CAAC,QAA+B;QACnD,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAED,gBAAgB,CAAC,OAA+C;QAC9D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IAC/B,CAAC;IAED,QAAQ,CAAC,KAAc;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,4BAA4B,CAAC,OAAmC;QAC9D,IAAI,CAAC,yBAAyB,GAAG,OAAO,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,gBAAuC;QACnD,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC7C,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAExC,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,qCAAqC;QAElE,qCAAqC;QACrC,IAAI,UAAU,GAAG,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,kBAAkB,GAAuB,SAAS,CAAC;QACvD,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,IAAI,CAAC;YACH,sDAAsD;YACtD,OAAO,CAAC,SAAS,IAAI,WAAW,GAAG,QAAQ,EAAE,CAAC;gBAC5C,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,kBAAkB,WAAW,GAAG,CAAC,IAAI,QAAQ,EAAE;oBACxD,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,UAAU,EACV,kBAAkB,EAClB,MAAM,CACP,CAAC;gBACF,gBAAgB,IAAI,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;gBAC9C,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC;gBAChD,kBAAkB,IAAI,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;gBAErD,0BAA0B;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;gBAEhC,2BAA2B;gBAC3B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAE7B,sDAAsD;gBACtD,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC;gBAEvC,+DAA+D;gBAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC;gBACrC,CAAC;gBAED,mCAAmC;gBACnC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBACjC,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC;gBAChC,CAAC;gBAED,yBAAyB;gBACzB,WAAW,EAAE,CAAC;YAChB,CAAC;YAED,0BAA0B;YAC1B,OAAO;gBACL,OAAO,EAAE,SAAS;gBAClB,OAAO;gBACP,OAAO,EAAE,YAAY;gBACrB,SAAS;gBACT,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,+BAA+B,YAAY,EAAE;gBACtD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO;gBACP,OAAO,EAAE,2BAA2B,YAAY,EAAE;gBAClD,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,UAA+B,EAC/B,kBAAsC,EACtC,MAAkC;QAalC,IAAI,CAAC;YACH,8BAA8B;YAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACrC,MAAM,KAAK,GAAG;gBACZ,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;gBACvC,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa;gBACzC,iBAAiB,EAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB;aAClD,CAAC;YAEF,qCAAqC;YACrC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBACvC,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE;wBACnD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAkB,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnE,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,MAAM,CAAC,IAAI,cAAc,IAAI,CAAC,OAAO,EAAE;wBAC7E,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBACtD,IAAI,MAAM,EAAE,CAAC;wBACX,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACzB,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,sCAAsC,MAAM,CAAC,IAAI,EAAE;4BAC5D,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;qBAAM,IACL,IAAI,CAAC,IAAI,KAAK,eAAe;oBAC7B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAC7B,CAAC;oBACD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,cAAc,IAAI,CAAC,OAAO,EAAE;wBACtE,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBACtD,IAAI,MAAM,EAAE,CAAC;wBACX,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACzB,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,sCAAsC,MAAM,CAAC,IAAI,EAAE;4BAC5D,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,uBAAuB;YACvB,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC5B,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,qBAAqB;wBAC9B,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;wBAChD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;4BACnC,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gCACnD,OAAO,IAAI,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;gCAC/B,MAAM,CAAC;oCACL,QAAQ,EAAE,OAAO;oCACjB,OAAO,EAAE,iBAAiB,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE;oCACtD,KAAK,EAAE,CAAC;iCACT,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAE7D,qBAAqB;YACrB,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,CAAC;gBACnB,MAAM,CAAC,KAAK,CACV,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,CAC/D,CAAC;YAEJ,OAAO;gBACL,OAAO,EAAE,WAAW;gBACpB,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;gBACvB,SAAS;gBACT,cAAc;gBACd,UAAU;gBACV,KAAK,EAAE,KAAK;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,yBAAyB,YAAY,EAAE;gBAChD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,kBAAkB,CAAC,IAAkB;QAC3C,OAAO,CACL,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,SAAS,IAAI,IAAI;YACjB,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAChC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,mBAAkC,EAClC,MAAkC;QAElC,IAAI,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnC,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,sCAAsC,mBAAmB,CAAC,MAAM,cAAc,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC1I,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,MAAM,QAAQ,GACZ,MAAM,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;YAE5D,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;gBAC1B,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO,mBAAmB,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,gCAAgC;oBACzC,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;QAED,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,sBAAsB,mBAAmB,CAAC,MAAM,kBAAkB;YAC3E,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IAEO,kBAAkB,CAAC,IAAkB;QAC3C,OAAO,CACL,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,SAAS,IAAI,IAAI;YACjB,MAAM,IAAI,IAAI;YACd,WAAW,IAAI,IAAI,CACpB,CAAC;IACJ,CAAC;IAEO,uBAAuB,CAAC,WAAmB;QACjD,6EAA6E;QAC7E,OAAO;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,IAAI,CAAC,wBAAwB;aACvC;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,UAA+B,EAC/B,kBAA2B;QAM3B,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,aAAa,GAA4B;gBAC7C,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,sBAAsB;wBAC5B,aAAa,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK;wBACzC,cAAc,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;wBAC3C,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B;iBACF;gBACD,KAAK,EAAE,UAAU;gBACjB,UAAU,EAAE,MAAM;aACnB,CAAC;YAEF,gCAAgC;YAChC,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;oBACpE,IAAI,EAAE,UAAmB;oBACzB,IAAI;oBACJ,QAAQ,EAAE;wBACR,IAAI;wBACJ,WAAW,EAAE,IAAI,CAAC,WAAW;wBAC7B,UAAU,EAAE,IAAI,CAAC,WAAW;qBAC7B;iBACF,CAAC,CAAC,CAAC;gBAEJ,aAAa,CAAC,KAAK,GAAG;oBACpB,GAAI,aAAa,CAAC,KAAmC;oBACrD,GAAG,WAAW;iBACf,CAAC;YACJ,CAAC;YAED,wCAAwC;YACxC,IAAI,kBAAkB,EAAE,CAAC;gBACvB,aAAa,CAAC,oBAAoB,GAAG,kBAAkB,CAAC;YAC1D,CAAC;YAED,kBAAkB;YAClB,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC;YAC9B,UAAU,CAAC,aAAa,CAAC;gBACvB,SAAS,EAAE,YAAY;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,MAAM,EAAE,0BAA0B,CAAC,UAAU,CAAC;aAC/C,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,qDAAqD;YACrD,qEAAqE;YACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;YAEtC,kDAAkD;YAClD,MAAM,KAAK,GAAG;gBACZ,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;gBACzC,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;gBAC3C,iBAAiB,EAAE,SAAS;aAC7B,CAAC;YAEF,mBAAmB;YACnB,UAAU,CAAC,cAAc,CAAC;gBACxB,SAAS,EAAE,YAAY;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,MAAM,EAAE,4BAA4B,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACrD,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;gBACxC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;aAC3C,CAAC,CAAC;YAEH,uCAAuC;YACvC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,EAAE,CAAC;YAElC,oCAAoC;YACpC,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAmC;gBACpD,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,KAAK;aACN,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CACd,MAAsB,EACtB,MAAkC;QAElC,MAAM,cAAc,GAAwB,EAAE,CAAC;QAE/C,2BAA2B;QAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnE,wBAAwB;gBACxB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBAEtD,IAAI,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,8BAA8B,MAAM,CAAC,IAAI,EAAE;4BACpD,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACnC,CAAC;oBAED,uBAAuB;oBACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAElD,qDAAqD;oBACrD,MAAM,UAAU,GAAG;wBACjB,IAAI,EAAE,sBAA+B;wBACrC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE;4BACN,IAAI,EAAE,aAAsB;4BAC5B,SAAS,EAAE,UAAU;yBACtB;qBACmB,CAAC;oBAEvB,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,2CAA2C,IAAI,CAAC,OAAO,EAAE;wBAClE,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,+BAA+B;oBAC/B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,MAAM,kBAAkB,GAAG,UAS1B,CAAC;wBACF,kBAAkB,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;oBAC1D,CAAC;oBAED,IACE,IAAI,CAAC,qBAAqB;wBAC1B,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EACrC,CAAC;wBACD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC5D,IAAI,CAAC,qBAAqB,EAC1B,MAAM,CACP,CAAC;wBAEF,IAAI,kBAAkB,EAAE,CAAC;4BACvB,MAAM,kBAAkB,GAAG,UAQ1B,CAAC;4BACF,kBAAkB,CAAC,0BAA0B;gCAC3C,kBAAkB,CAAC;wBACvB,CAAC;oBACH,CAAC;oBAED,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;wBAC1C,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAEzD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,kCAAkC,YAAY,EAAE;wBACzD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,IAAI,CAAC;wBACH,qCAAqC;wBACrC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBAElD,MAAM,eAAe,GAAG;4BACtB,IAAI,EAAE,sBAA+B;4BACrC,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE;gCACN,IAAI,EAAE,aAAsB;gCAC5B,SAAS,EAAE,UAAU;gCACrB,KAAK,EAAE,YAAY;6BACpB;yBACmB,CAAC;wBAEvB,+BAA+B;wBAC/B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;4BACpB,MAAM,kBAAkB,GAAG,eAS1B,CAAC;4BACF,kBAAkB,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;wBAC1D,CAAC;wBAED,IACE,IAAI,CAAC,qBAAqB;4BAC1B,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EACrC,CAAC;4BACD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC5D,IAAI,CAAC,qBAAqB,EAC1B,MAAM,CACP,CAAC;4BAEF,IAAI,kBAAkB,EAAE,CAAC;gCACvB,MAAM,kBAAkB,GAAG,eAQ1B,CAAC;gCACF,kBAAkB,CAAC,0BAA0B;oCAC3C,kBAAkB,CAAC;4BACvB,CAAC;wBACH,CAAC;wBAED,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;oBACvC,CAAC;oBAAC,OAAO,eAAe,EAAE,CAAC;wBACzB,IAAI,eAAe,YAAY,oBAAoB,EAAE,CAAC;4BACpD,MAAM,eAAe,CAAC;wBACxB,CAAC;wBACD,wDAAwD;wBACxD,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,+BAA+B,MAAM,CAAC,eAAe,CAAC,EAAE;4BACjE,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;wBAEH,uEAAuE;wBACvE,cAAc,CAAC,IAAI,CAAC;4BAClB,IAAI,EAAE,sBAAsB;4BAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,UAAU,YAAY,EAAE;yBACZ,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,IACL,IAAI,CAAC,IAAI,KAAK,eAAe;gBAC7B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAC7B,CAAC;gBACD,qCAAqC;gBACrC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBAEtD,IAAI,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACnC,CAAC;oBAED,gCAAgC;oBAChC,IAAI,UAAU,GAAG,4BAA4B,CAAC;oBAC9C,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;wBAC1C,IAAI,CAAC;4BACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACnC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;4BAExC,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,SAAS,EAAE;gCACzE,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;4BAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;gCACtC,UAAU,EAAE,IAAI,CAAC,OAAO;gCACxB,QAAQ,EAAE,EAAE;6BACb,CAAC,CAAC;4BACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BAEpC,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,QAAQ,IAAI,CAAC,IAAI,oCAAoC,UAAU,EAAE;gCAC1E,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,SAAS,EAAE,CAAC;4BACnB,MAAM,YAAY,GAChB,SAAS,YAAY,KAAK;gCACxB,CAAC,CAAC,SAAS,CAAC,OAAO;gCACnB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;4BACxB,UAAU,GAAG,yBAAyB,YAAY,EAAE,CAAC;4BAErD,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,KAAK,YAAY,EAAE;gCAC7D,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,qDAAqD;oBACrD,MAAM,UAAU,GAAsB;wBACpC,IAAI,EAAE,sBAAsB;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE,UAAU;qBACnB,CAAC;oBAEF,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;wBAC1C,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAEzD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,kCAAkC,YAAY,EAAE;wBACzD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,yBAAyB;oBACzB,MAAM,eAAe,GAAsB;wBACzC,IAAI,EAAE,sBAAsB;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE,UAAU,YAAY,EAAE;qBACjC,CAAC;oBAEF,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,2BAA2B,CACjC,IAAsB;QAEtB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAExB,2FAA2F;QAC3F,6FAA6F;QAC7F,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAc;YAC3B,GAAG,MAAM,EAAE,wCAAwC;SACpD,CAAC;IACJ,CAAC;IAEO,2BAA2B,CACjC,IAAsB;QAEtB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExC,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,IAAI;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAGvB;QACC,oCAAoC;QACpC,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;YACzB,OAAO,yBAAyB,OAAO,CAAC,WAAW,EAAE,CAAC;QACxD,CAAC;QAED,2CAA2C;QAC3C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACpD,OAAO,yBAAyB,WAAW,EAAE,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;gBACpD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,MAAM,IAAI,4BAA4B,CACpC,yCAAyC;YACvC,iGAAiG,CACpG,CAAC;IACJ,CAAC;CACF","sourcesContent":["import OpenAI from \"openai\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport {\n  AgentAction,\n  AgentResult,\n  AgentType,\n  AgentExecutionOptions,\n  ResponseInputItem,\n  ResponseItem,\n  ComputerCallItem,\n  FunctionCallItem,\n  SafetyCheck,\n  SafetyConfirmationHandler,\n} from \"../types/public/agent.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport {\n  AgentScreenshotProviderError,\n  StagehandClosedError,\n} from \"../types/public/sdkErrors.js\";\nimport { ToolSet } from \"ai\";\nimport {\n  FlowLogger,\n  extractLlmCuaPromptSummary,\n  extractLlmCuaResponseSummary,\n} from \"../flowlogger/FlowLogger.js\";\nimport { v7 as uuidv7 } from \"uuid\";\n\n/**\n * Client for OpenAI's Computer Use Assistant API\n * This implementation uses the official OpenAI Responses API for Computer Use\n */\nexport class OpenAICUAClient extends AgentClient {\n  private apiKey: string;\n  private organization?: string;\n  private baseURL: string;\n  private client: OpenAI;\n  public lastResponseId?: string;\n  private currentViewport = { width: 1288, height: 711 };\n  private currentUrl?: string;\n  private screenshotProvider?: () => Promise<string>;\n  private actionHandler?: (action: AgentAction) => Promise<void>;\n  private reasoningItems: Map<string, ResponseItem> = new Map();\n  private environment: string = \"browser\"; // \"browser\", \"mac\", \"windows\", or \"ubuntu\"\n  private tools?: ToolSet;\n  private safetyConfirmationHandler?: SafetyConfirmationHandler;\n\n  constructor(\n    type: AgentType,\n    modelName: string,\n    userProvidedInstructions?: string,\n    clientOptions?: ClientOptions,\n    tools?: ToolSet,\n  ) {\n    super(type, modelName, userProvidedInstructions);\n\n    // Process client options\n    this.apiKey =\n      (clientOptions?.apiKey as string) || process.env.OPENAI_API_KEY || \"\";\n    this.baseURL = (clientOptions?.baseURL as string) || undefined;\n    this.organization =\n      (clientOptions?.organization as string) || process.env.OPENAI_ORG;\n\n    // Get environment if specified\n    if (\n      clientOptions?.environment &&\n      typeof clientOptions.environment === \"string\"\n    ) {\n      this.environment = clientOptions.environment;\n    }\n\n    // Store client options for reference\n    this.clientOptions = {\n      apiKey: this.apiKey,\n    };\n\n    if (this.baseURL) {\n      this.clientOptions.baseURL = this.baseURL;\n    }\n\n    // Initialize the OpenAI client\n    this.client = new OpenAI(this.clientOptions);\n\n    this.tools = tools;\n  }\n\n  setViewport(width: number, height: number): void {\n    this.currentViewport = { width, height };\n  }\n\n  setCurrentUrl(url: string): void {\n    this.currentUrl = url;\n  }\n\n  setScreenshotProvider(provider: () => Promise<string>): void {\n    this.screenshotProvider = provider;\n  }\n\n  setActionHandler(handler: (action: AgentAction) => Promise<void>): void {\n    this.actionHandler = handler;\n  }\n\n  setTools(tools: ToolSet): void {\n    this.tools = tools;\n  }\n\n  setSafetyConfirmationHandler(handler?: SafetyConfirmationHandler): void {\n    this.safetyConfirmationHandler = handler;\n  }\n\n  /**\n   * Execute a task with the OpenAI CUA\n   * This is the main entry point for the agent\n   * @implements AgentClient.execute\n   */\n  async execute(executionOptions: AgentExecutionOptions): Promise<AgentResult> {\n    const { options, logger } = executionOptions;\n    const { instruction } = options;\n    const maxSteps = options.maxSteps || 10;\n\n    let currentStep = 0;\n    let completed = false;\n    const actions: AgentAction[] = [];\n    const messageList: string[] = [];\n    let finalMessage = \"\";\n    this.reasoningItems.clear(); // Clear any previous reasoning items\n\n    // Start with the initial instruction\n    let inputItems = this.createInitialInputItems(instruction);\n    let previousResponseId: string | undefined = undefined;\n    let totalInputTokens = 0;\n    let totalOutputTokens = 0;\n    let totalInferenceTime = 0;\n\n    try {\n      // Execute steps until completion or max steps reached\n      while (!completed && currentStep < maxSteps) {\n        logger({\n          category: \"agent\",\n          message: `Executing step ${currentStep + 1}/${maxSteps}`,\n          level: 1,\n        });\n\n        const result = await this.executeStep(\n          inputItems,\n          previousResponseId,\n          logger,\n        );\n        totalInputTokens += result.usage.input_tokens;\n        totalOutputTokens += result.usage.output_tokens;\n        totalInferenceTime += result.usage.inference_time_ms;\n\n        // Add actions to the list\n        actions.push(...result.actions);\n\n        // Update completion status\n        completed = result.completed;\n\n        // Store the previous response ID for the next request\n        previousResponseId = result.responseId;\n\n        // Update the input items for the next step if we're continuing\n        if (!completed) {\n          inputItems = result.nextInputItems;\n        }\n\n        // Record any message for this step\n        if (result.message) {\n          messageList.push(result.message);\n          finalMessage = result.message;\n        }\n\n        // Increment step counter\n        currentStep++;\n      }\n\n      // Return the final result\n      return {\n        success: completed,\n        actions,\n        message: finalMessage,\n        completed,\n        usage: {\n          input_tokens: totalInputTokens,\n          output_tokens: totalOutputTokens,\n          inference_time_ms: totalInferenceTime,\n        },\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      logger({\n        category: \"agent\",\n        message: `Error executing agent task: ${errorMessage}`,\n        level: 0,\n      });\n\n      return {\n        success: false,\n        actions,\n        message: `Failed to execute task: ${errorMessage}`,\n        completed: false,\n        usage: {\n          input_tokens: totalInputTokens,\n          output_tokens: totalOutputTokens,\n          inference_time_ms: totalInferenceTime,\n        },\n      };\n    }\n  }\n\n  /**\n   * Execute a single step of the agent\n   * This coordinates the flow: Request → Get Action → Execute Action\n   */\n  async executeStep(\n    inputItems: ResponseInputItem[],\n    previousResponseId: string | undefined,\n    logger: (message: LogLine) => void,\n  ): Promise<{\n    actions: AgentAction[];\n    message: string;\n    completed: boolean;\n    nextInputItems: ResponseInputItem[];\n    responseId: string;\n    usage: {\n      input_tokens: number;\n      output_tokens: number;\n      inference_time_ms: number;\n    };\n  }> {\n    try {\n      // Get response from the model\n      const result = await this.getAction(inputItems, previousResponseId);\n      const output = result.output;\n      const responseId = result.responseId;\n      const usage = {\n        input_tokens: result.usage.input_tokens,\n        output_tokens: result.usage.output_tokens,\n        inference_time_ms: result.usage.inference_time_ms,\n      };\n\n      // Add any reasoning items to our map\n      for (const item of output) {\n        if (item.type === \"reasoning\") {\n          this.reasoningItems.set(item.id, item);\n          logger({\n            category: \"agent\",\n            message: `Reasoning: ${String(item.content || \"\")}`,\n            level: 1,\n          });\n        }\n      }\n\n      // Extract actions from the output\n      const stepActions: AgentAction[] = [];\n      for (const item of output) {\n        if (item.type === \"computer_call\" && this.isComputerCallItem(item)) {\n          logger({\n            category: \"agent\",\n            message: `Found computer_call: ${item.action.type}, call_id: ${item.call_id}`,\n            level: 2,\n          });\n          const action = this.convertComputerCallToAction(item);\n          if (action) {\n            stepActions.push(action);\n            logger({\n              category: \"agent\",\n              message: `Converted computer_call to action: ${action.type}`,\n              level: 2,\n            });\n          }\n        } else if (\n          item.type === \"function_call\" &&\n          this.isFunctionCallItem(item)\n        ) {\n          logger({\n            category: \"agent\",\n            message: `Found function_call: ${item.name}, call_id: ${item.call_id}`,\n            level: 2,\n          });\n          const action = this.convertFunctionCallToAction(item);\n          if (action) {\n            stepActions.push(action);\n            logger({\n              category: \"agent\",\n              message: `Converted function_call to action: ${action.type}`,\n              level: 2,\n            });\n          }\n        }\n      }\n\n      // Extract message text\n      let message = \"\";\n      for (const item of output) {\n        if (item.type === \"message\") {\n          logger({\n            category: \"agent\",\n            message: `Found message block`,\n            level: 2,\n          });\n          if (item.content && Array.isArray(item.content)) {\n            for (const content of item.content) {\n              if (content.type === \"output_text\" && content.text) {\n                message += content.text + \"\\n\";\n                logger({\n                  category: \"agent\",\n                  message: `Message text: ${String(content.text || \"\")}`,\n                  level: 1,\n                });\n              }\n            }\n          }\n        }\n      }\n\n      // Take actions and get results\n      const nextInputItems = await this.takeAction(output, logger);\n\n      // Check if completed\n      const completed =\n        output.length === 0 ||\n        output.every(\n          (item) => item.type === \"message\" || item.type === \"reasoning\",\n        );\n\n      return {\n        actions: stepActions,\n        message: message.trim(),\n        completed,\n        nextInputItems,\n        responseId,\n        usage: usage,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      logger({\n        category: \"agent\",\n        message: `Error executing step: ${errorMessage}`,\n        level: 0,\n      });\n\n      throw error;\n    }\n  }\n\n  private isComputerCallItem(item: ResponseItem): item is ComputerCallItem {\n    return (\n      item.type === \"computer_call\" &&\n      \"call_id\" in item &&\n      \"action\" in item &&\n      typeof item.action === \"object\"\n    );\n  }\n\n  private async handleSafetyConfirmation(\n    pendingSafetyChecks: SafetyCheck[],\n    logger: (message: LogLine) => void,\n  ): Promise<SafetyCheck[] | undefined> {\n    if (this.safetyConfirmationHandler) {\n      logger({\n        category: \"agent\",\n        message: `Requesting safety confirmation for ${pendingSafetyChecks.length} check(s): ${pendingSafetyChecks.map((c) => c.code).join(\", \")}`,\n        level: 1,\n      });\n\n      const response =\n        await this.safetyConfirmationHandler(pendingSafetyChecks);\n\n      if (response.acknowledged) {\n        logger({\n          category: \"agent\",\n          message: `Safety checks acknowledged by user`,\n          level: 1,\n        });\n        return pendingSafetyChecks;\n      } else {\n        logger({\n          category: \"agent\",\n          message: `Safety checks rejected by user`,\n          level: 1,\n        });\n        return undefined;\n      }\n    }\n\n    logger({\n      category: \"agent\",\n      message: `Auto-acknowledging ${pendingSafetyChecks.length} safety check(s)`,\n      level: 2,\n    });\n    return pendingSafetyChecks;\n  }\n\n  private isFunctionCallItem(item: ResponseItem): item is FunctionCallItem {\n    return (\n      item.type === \"function_call\" &&\n      \"call_id\" in item &&\n      \"name\" in item &&\n      \"arguments\" in item\n    );\n  }\n\n  private createInitialInputItems(instruction: string): ResponseInputItem[] {\n    // For the initial request, we use a simple array with the user's instruction\n    return [\n      {\n        role: \"system\",\n        content: this.userProvidedInstructions,\n      },\n      {\n        role: \"user\",\n        content: instruction,\n      },\n    ];\n  }\n\n  async getAction(\n    inputItems: ResponseInputItem[],\n    previousResponseId?: string,\n  ): Promise<{\n    output: ResponseItem[];\n    responseId: string;\n    usage: Record<string, number>;\n  }> {\n    try {\n      // Create the request parameters\n      const requestParams: Record<string, unknown> = {\n        model: this.modelName,\n        tools: [\n          {\n            type: \"computer_use_preview\",\n            display_width: this.currentViewport.width,\n            display_height: this.currentViewport.height,\n            environment: this.environment,\n          },\n        ],\n        input: inputItems,\n        truncation: \"auto\",\n      };\n\n      // Add custom tools if available\n      if (this.tools && Object.keys(this.tools).length > 0) {\n        const customTools = Object.entries(this.tools).map(([name, tool]) => ({\n          type: \"function\" as const,\n          name,\n          function: {\n            name,\n            description: tool.description,\n            parameters: tool.inputSchema,\n          },\n        }));\n\n        requestParams.tools = [\n          ...(requestParams.tools as Record<string, unknown>[]),\n          ...customTools,\n        ];\n      }\n\n      // Add previous_response_id if available\n      if (previousResponseId) {\n        requestParams.previous_response_id = previousResponseId;\n      }\n\n      // Log LLM request\n      const llmRequestId = uuidv7();\n      FlowLogger.logLlmRequest({\n        requestId: llmRequestId,\n        model: this.modelName,\n        prompt: extractLlmCuaPromptSummary(inputItems),\n      });\n\n      const startTime = Date.now();\n      // Create the response using the OpenAI Responses API\n      // @ts-expect-error - Force type to match what the OpenAI SDK expects\n      const response = await this.client.responses.create(requestParams);\n      const endTime = Date.now();\n      const elapsedMs = endTime - startTime;\n\n      // Extract only the input_tokens and output_tokens\n      const usage = {\n        input_tokens: response.usage.input_tokens,\n        output_tokens: response.usage.output_tokens,\n        inference_time_ms: elapsedMs,\n      };\n\n      // Log LLM response\n      FlowLogger.logLlmResponse({\n        requestId: llmRequestId,\n        model: this.modelName,\n        output: extractLlmCuaResponseSummary(response.output),\n        inputTokens: response.usage.input_tokens,\n        outputTokens: response.usage.output_tokens,\n      });\n\n      // Store the response ID for future use\n      this.lastResponseId = response.id;\n\n      // Return the output and response ID\n      return {\n        output: response.output as unknown as ResponseItem[],\n        responseId: response.id,\n        usage,\n      };\n    } catch (error) {\n      console.error(\"Error getting action from OpenAI:\", error);\n      throw error;\n    }\n  }\n\n  async takeAction(\n    output: ResponseItem[],\n    logger: (message: LogLine) => void,\n  ): Promise<ResponseInputItem[]> {\n    const nextInputItems: ResponseInputItem[] = [];\n\n    // Process each output item\n    for (const item of output) {\n      if (item.type === \"computer_call\" && this.isComputerCallItem(item)) {\n        // Handle computer calls\n        try {\n          const action = this.convertComputerCallToAction(item);\n\n          if (action && this.actionHandler) {\n            logger({\n              category: \"agent\",\n              message: `Executing computer action: ${action.type}`,\n              level: 1,\n            });\n            await this.actionHandler(action);\n          }\n\n          // Capture a screenshot\n          const screenshot = await this.captureScreenshot();\n\n          // Create a computer_call_output for the next request\n          const outputItem = {\n            type: \"computer_call_output\" as const,\n            call_id: item.call_id,\n            output: {\n              type: \"input_image\" as const,\n              image_url: screenshot,\n            },\n          } as ResponseInputItem;\n\n          logger({\n            category: \"agent\",\n            message: `Added computer_call_output for call_id: ${item.call_id}`,\n            level: 2,\n          });\n\n          // Add current URL if available\n          if (this.currentUrl) {\n            const computerCallOutput = outputItem as {\n              type: \"computer_call_output\";\n              call_id: string;\n              output: {\n                type: \"input_image\";\n                image_url: string;\n                current_url?: string;\n              };\n              acknowledged_safety_checks?: SafetyCheck[];\n            };\n            computerCallOutput.output.current_url = this.currentUrl;\n          }\n\n          if (\n            item.pending_safety_checks &&\n            item.pending_safety_checks.length > 0\n          ) {\n            const acknowledgedChecks = await this.handleSafetyConfirmation(\n              item.pending_safety_checks,\n              logger,\n            );\n\n            if (acknowledgedChecks) {\n              const computerCallOutput = outputItem as {\n                type: \"computer_call_output\";\n                call_id: string;\n                output: {\n                  type: \"input_image\";\n                  image_url: string;\n                };\n                acknowledged_safety_checks?: SafetyCheck[];\n              };\n              computerCallOutput.acknowledged_safety_checks =\n                acknowledgedChecks;\n            }\n          }\n\n          nextInputItems.push(outputItem);\n        } catch (error) {\n          if (error instanceof StagehandClosedError) {\n            throw error;\n          }\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n\n          logger({\n            category: \"agent\",\n            message: `Error executing computer call: ${errorMessage}`,\n            level: 0,\n          });\n\n          try {\n            // Capture a screenshot even on error\n            const screenshot = await this.captureScreenshot();\n\n            const errorOutputItem = {\n              type: \"computer_call_output\" as const,\n              call_id: item.call_id,\n              output: {\n                type: \"input_image\" as const,\n                image_url: screenshot,\n                error: errorMessage,\n              },\n            } as ResponseInputItem;\n\n            // Add current URL if available\n            if (this.currentUrl) {\n              const computerCallOutput = errorOutputItem as {\n                type: \"computer_call_output\";\n                call_id: string;\n                output: {\n                  type: \"input_image\";\n                  image_url: string;\n                  current_url?: string;\n                };\n                acknowledged_safety_checks?: SafetyCheck[];\n              };\n              computerCallOutput.output.current_url = this.currentUrl;\n            }\n\n            if (\n              item.pending_safety_checks &&\n              item.pending_safety_checks.length > 0\n            ) {\n              const acknowledgedChecks = await this.handleSafetyConfirmation(\n                item.pending_safety_checks,\n                logger,\n              );\n\n              if (acknowledgedChecks) {\n                const computerCallOutput = errorOutputItem as {\n                  type: \"computer_call_output\";\n                  call_id: string;\n                  output: {\n                    type: \"input_image\";\n                    image_url: string;\n                  };\n                  acknowledged_safety_checks?: SafetyCheck[];\n                };\n                computerCallOutput.acknowledged_safety_checks =\n                  acknowledgedChecks;\n              }\n            }\n\n            nextInputItems.push(errorOutputItem);\n          } catch (screenshotError) {\n            if (screenshotError instanceof StagehandClosedError) {\n              throw screenshotError;\n            }\n            // If we can't capture a screenshot, just send the error\n            logger({\n              category: \"agent\",\n              message: `Error capturing screenshot: ${String(screenshotError)}`,\n              level: 0,\n            });\n\n            // For error cases without a screenshot, we need to use a string output\n            nextInputItems.push({\n              type: \"computer_call_output\",\n              call_id: item.call_id,\n              output: `Error: ${errorMessage}`,\n            } as ResponseInputItem);\n          }\n        }\n      } else if (\n        item.type === \"function_call\" &&\n        this.isFunctionCallItem(item)\n      ) {\n        // Handle function calls (tool calls)\n        try {\n          const action = this.convertFunctionCallToAction(item);\n\n          if (action && this.actionHandler) {\n            await this.actionHandler(action);\n          }\n\n          // Execute the tool if available\n          let toolResult = \"Tool executed successfully\";\n          if (this.tools && item.name in this.tools) {\n            try {\n              const tool = this.tools[item.name];\n              const args = JSON.parse(item.arguments);\n\n              logger({\n                category: \"agent\",\n                message: `Executing tool call: ${item.name} with args: ${item.arguments}`,\n                level: 1,\n              });\n\n              const result = await tool.execute(args, {\n                toolCallId: item.call_id,\n                messages: [],\n              });\n              toolResult = JSON.stringify(result);\n\n              logger({\n                category: \"agent\",\n                message: `Tool ${item.name} completed successfully. Result: ${toolResult}`,\n                level: 1,\n              });\n            } catch (toolError) {\n              const errorMessage =\n                toolError instanceof Error\n                  ? toolError.message\n                  : String(toolError);\n              toolResult = `Error executing tool: ${errorMessage}`;\n\n              logger({\n                category: \"agent\",\n                message: `Error executing tool ${item.name}: ${errorMessage}`,\n                level: 0,\n              });\n            }\n          }\n\n          // Create a function_call_output for the next request\n          const outputItem: ResponseInputItem = {\n            type: \"function_call_output\",\n            call_id: item.call_id,\n            output: toolResult,\n          };\n\n          nextInputItems.push(outputItem);\n        } catch (error) {\n          if (error instanceof StagehandClosedError) {\n            throw error;\n          }\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n\n          logger({\n            category: \"agent\",\n            message: `Error executing function call: ${errorMessage}`,\n            level: 0,\n          });\n\n          // Send error result back\n          const errorOutputItem: ResponseInputItem = {\n            type: \"function_call_output\",\n            call_id: item.call_id,\n            output: `Error: ${errorMessage}`,\n          };\n\n          nextInputItems.push(errorOutputItem);\n        }\n      }\n    }\n\n    return nextInputItems;\n  }\n\n  private convertComputerCallToAction(\n    call: ComputerCallItem,\n  ): AgentAction | null {\n    const { action } = call;\n\n    // Instead of wrapping the action in a params object, spread the action properties directly\n    // This ensures properties like x, y, button, etc. are directly accessible on the AgentAction\n    return {\n      type: action.type as string,\n      ...action, // Spread all properties from the action\n    };\n  }\n\n  private convertFunctionCallToAction(\n    call: FunctionCallItem,\n  ): AgentAction | null {\n    try {\n      const args = JSON.parse(call.arguments);\n\n      return {\n        type: call.name,\n        params: args,\n      };\n    } catch (error) {\n      console.error(\"Error parsing function call arguments:\", error);\n      return null;\n    }\n  }\n\n  async captureScreenshot(options?: {\n    base64Image?: string;\n    currentUrl?: string;\n  }): Promise<string> {\n    // Use provided options if available\n    if (options?.base64Image) {\n      return `data:image/png;base64,${options.base64Image}`;\n    }\n\n    // Use the screenshot provider if available\n    if (this.screenshotProvider) {\n      try {\n        const base64Image = await this.screenshotProvider();\n        return `data:image/png;base64,${base64Image}`;\n      } catch (error) {\n        console.error(\"Error capturing screenshot:\", error);\n        throw error;\n      }\n    }\n\n    throw new AgentScreenshotProviderError(\n      \"`screenshotProvider` has not been set. \" +\n        \"Please call `setScreenshotProvider()` with a valid function that returns a base64-encoded image\",\n    );\n  }\n}\n"]}
+{"version":3,"file":"OpenAICUAClient.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/OpenAICUAClient.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAe5B,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EACL,4BAA4B,EAC5B,oBAAoB,GACrB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,UAAU,EACV,0BAA0B,EAC1B,4BAA4B,GAC7B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAEpC;;;GAGG;AACH,MAAM,oBAAoB,GAAG,sBAAsB,CAAC;AAEpD,MAAM,OAAO,eAAgB,SAAQ,WAAW;IACtC,mBAAmB,GAAa,EAAE,CAAC;IACnC,uBAAuB,GAAG,KAAK,CAAC;IAChC,MAAM,CAAS;IACf,YAAY,CAAU;IACtB,OAAO,CAAS;IAChB,MAAM,CAAS;IAChB,cAAc,CAAU;IACvB,eAAe,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC/C,UAAU,CAAU;IACpB,kBAAkB,CAAyB;IAC3C,aAAa,CAA0C;IACvD,cAAc,GAA8B,IAAI,GAAG,EAAE,CAAC;IACtD,WAAW,GAAW,SAAS,CAAC,CAAC,2CAA2C;IAC5E,KAAK,CAAW;IAChB,yBAAyB,CAA6B;IAE9D,YACE,IAAe,EACf,SAAiB,EACjB,wBAAiC,EACjC,aAA6B,EAC7B,KAAe;QAEf,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,wBAAwB,CAAC,CAAC;QAEjD,yBAAyB;QACzB,IAAI,CAAC,MAAM;YACR,aAAa,EAAE,MAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,OAAO,GAAI,aAAa,EAAE,OAAkB,IAAI,SAAS,CAAC;QAC/D,IAAI,CAAC,YAAY;YACd,aAAa,EAAE,YAAuB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QAEpE,+BAA+B;QAC/B,IACE,aAAa,EAAE,WAAW;YAC1B,OAAO,aAAa,CAAC,WAAW,KAAK,QAAQ,EAC7C,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,aAAa,CAAC,WAAW,CAAC;QAC/C,CAAC;QAED,qCAAqC;QACrC,IAAI,CAAC,aAAa,GAAG;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,aAAa,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5C,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,MAAc;QACvC,IAAI,CAAC,eAAe,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACxB,CAAC;IAED,qBAAqB,CAAC,QAA+B;QACnD,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAED,gBAAgB,CAAC,OAA+C;QAC9D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IAC/B,CAAC;IAED,QAAQ,CAAC,KAAc;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,4BAA4B,CAAC,OAAmC;QAC9D,IAAI,CAAC,yBAAyB,GAAG,OAAO,CAAC;IAC3C,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpC,wEAAwE;QACxE,mEAAmE;QACnE,8DAA8D;QAC9D,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,gBAAuC;QACnD,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC7C,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAExC,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,qCAAqC;QAElE,qCAAqC;QACrC,IAAI,UAAU,GAAG,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,kBAAkB,GAAuB,SAAS,CAAC;QACvD,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,IAAI,CAAC;YACH,sDAAsD;YACtD,OAAO,CAAC,SAAS,IAAI,WAAW,GAAG,QAAQ,EAAE,CAAC;gBAC5C,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBAE3B,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,kBAAkB,WAAW,GAAG,CAAC,IAAI,QAAQ,EAAE;oBACxD,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,UAAU,EACV,kBAAkB,EAClB,MAAM,CACP,CAAC;gBACF,gBAAgB,IAAI,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;gBAC9C,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC;gBAChD,kBAAkB,IAAI,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;gBAErD,0BAA0B;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;gBAEhC,2BAA2B;gBAC3B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAE7B,sDAAsD;gBACtD,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC;gBAEvC,+DAA+D;gBAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC;oBACnC,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC9C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,UAAU,GAAG;4BACX,GAAG,UAAU;4BACb,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gCAC7B,IAAI,EAAE,MAAe;gCACrB,OAAO,EAAE,IAAI;6BACd,CAAC,CAAC;yBACJ,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,mCAAmC;gBACnC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBACjC,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC;gBAChC,CAAC;gBAED,yBAAyB;gBACzB,WAAW,EAAE,CAAC;YAChB,CAAC;YAED,0BAA0B;YAC1B,OAAO;gBACL,OAAO,EAAE,SAAS;gBAClB,OAAO;gBACP,OAAO,EAAE,YAAY;gBACrB,SAAS;gBACT,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,+BAA+B,YAAY,EAAE;gBACtD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO;gBACP,OAAO,EAAE,2BAA2B,YAAY,EAAE;gBAClD,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE;oBACL,YAAY,EAAE,gBAAgB;oBAC9B,aAAa,EAAE,iBAAiB;oBAChC,iBAAiB,EAAE,kBAAkB;iBACtC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,UAA+B,EAC/B,kBAAsC,EACtC,MAAkC;QAalC,IAAI,CAAC;YACH,8BAA8B;YAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACrC,MAAM,KAAK,GAAG;gBACZ,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;gBACvC,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa;gBACzC,iBAAiB,EAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB;aAClD,CAAC;YAEF,qCAAqC;YACrC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBACvC,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE;wBACnD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAkB,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnE,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,MAAM,CAAC,IAAI,cAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE;wBACtH,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBACtD,IAAI,MAAM,EAAE,CAAC;wBACX,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACzB,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,sCAAsC,MAAM,CAAC,IAAI,EAAE;4BAC5D,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;qBAAM,IACL,IAAI,CAAC,IAAI,KAAK,eAAe;oBAC7B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAC7B,CAAC;oBACD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,cAAc,IAAI,CAAC,OAAO,EAAE;wBACtE,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBACtD,IAAI,MAAM,EAAE,CAAC;wBACX,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACzB,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,sCAAsC,MAAM,CAAC,IAAI,EAAE;4BAC5D,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,uBAAuB;YACvB,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC5B,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,qBAAqB;wBAC9B,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;wBAChD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;4BACnC,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gCACnD,OAAO,IAAI,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;gCAC/B,MAAM,CAAC;oCACL,QAAQ,EAAE,OAAO;oCACjB,OAAO,EAAE,iBAAiB,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE;oCACtD,KAAK,EAAE,CAAC;iCACT,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAE7D,qBAAqB;YACrB,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,CAAC;gBACnB,MAAM,CAAC,KAAK,CACV,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,CAC/D,CAAC;YAEJ,OAAO;gBACL,OAAO,EAAE,WAAW;gBACpB,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;gBACvB,SAAS;gBACT,cAAc;gBACd,UAAU;gBACV,KAAK,EAAE,KAAK;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,yBAAyB,YAAY,EAAE;gBAChD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,kBAAkB,CAAC,IAAkB;QAC3C,OAAO,CACL,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,SAAS,IAAI,IAAI;YACjB,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAChC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,mBAAkC,EAClC,MAAkC;QAElC,IAAI,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnC,MAAM,CAAC;gBACL,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,sCAAsC,mBAAmB,CAAC,MAAM,cAAc,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC1I,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YAEH,MAAM,QAAQ,GACZ,MAAM,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;YAE5D,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;gBAC1B,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO,mBAAmB,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC;oBACL,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,gCAAgC;oBACzC,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;QAED,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,sBAAsB,mBAAmB,CAAC,MAAM,kBAAkB;YAC3E,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IAEO,kBAAkB,CAAC,IAAkB;QAC3C,OAAO,CACL,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,SAAS,IAAI,IAAI;YACjB,MAAM,IAAI,IAAI;YACd,WAAW,IAAI,IAAI,CACpB,CAAC;IACJ,CAAC;IAEO,uBAAuB,CAAC,WAAmB;QACjD,6EAA6E;QAC7E,OAAO;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,IAAI,CAAC,wBAAwB;aACvC;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,UAA+B,EAC/B,kBAA2B;QAM3B,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,aAAa,GAA4B;gBAC7C,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,sBAAsB;wBAC5B,aAAa,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK;wBACzC,cAAc,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;wBAC3C,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B;iBACF;gBACD,KAAK,EAAE,UAAU;gBACjB,UAAU,EAAE,MAAM;aACnB,CAAC;YAEF,gCAAgC;YAChC,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;oBACpE,IAAI,EAAE,UAAmB;oBACzB,IAAI;oBACJ,QAAQ,EAAE;wBACR,IAAI;wBACJ,WAAW,EAAE,IAAI,CAAC,WAAW;wBAC7B,UAAU,EAAE,IAAI,CAAC,WAAW;qBAC7B;iBACF,CAAC,CAAC,CAAC;gBAEJ,aAAa,CAAC,KAAK,GAAG;oBACpB,GAAI,aAAa,CAAC,KAAmC;oBACrD,GAAG,WAAW;iBACf,CAAC;YACJ,CAAC;YAED,mEAAmE;YACnE,oEAAoE;YACpE,+DAA+D;YAC/D,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBACjC,aAAa,CAAC,KAAK,GAAG;oBACpB,GAAI,aAAa,CAAC,KAAmC;oBACrD;wBACE,IAAI,EAAE,UAAmB;wBACzB,IAAI,EAAE,oBAAoB;wBAC1B,QAAQ,EAAE;4BACR,IAAI,EAAE,oBAAoB;4BAC1B,WAAW,EACT,qDAAqD;gCACrD,wDAAwD;gCACxD,4CAA4C;4BAC9C,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;yBAC7D;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,wCAAwC;YACxC,IAAI,kBAAkB,EAAE,CAAC;gBACvB,aAAa,CAAC,oBAAoB,GAAG,kBAAkB,CAAC;YAC1D,CAAC;YAED,kBAAkB;YAClB,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC;YAC9B,UAAU,CAAC,aAAa,CAAC;gBACvB,SAAS,EAAE,YAAY;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,MAAM,EAAE,0BAA0B,CAAC,UAAU,CAAC;aAC/C,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,qDAAqD;YACrD,qEAAqE;YACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;YAEtC,kDAAkD;YAClD,MAAM,KAAK,GAAG;gBACZ,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;gBACzC,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;gBAC3C,iBAAiB,EAAE,SAAS;aAC7B,CAAC;YAEF,mBAAmB;YACnB,UAAU,CAAC,cAAc,CAAC;gBACxB,SAAS,EAAE,YAAY;gBACvB,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,MAAM,EAAE,4BAA4B,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACrD,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;gBACxC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;aAC3C,CAAC,CAAC;YAEH,uCAAuC;YACvC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,EAAE,CAAC;YAElC,oCAAoC;YACpC,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAmC;gBACpD,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,KAAK;aACN,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CACd,MAAsB,EACtB,MAAkC;QAElC,MAAM,cAAc,GAAwB,EAAE,CAAC;QAE/C,2BAA2B;QAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnE,wBAAwB;gBACxB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBAEtD,IAAI,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,8BAA8B,MAAM,CAAC,IAAI,EAAE;4BACpD,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;wBACH,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACnC,CAAC;oBAED,uBAAuB;oBACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAElD,qDAAqD;oBACrD,MAAM,UAAU,GAAG;wBACjB,IAAI,EAAE,sBAA+B;wBACrC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE;4BACN,IAAI,EAAE,aAAsB;4BAC5B,SAAS,EAAE,UAAU;yBACtB;qBACmB,CAAC;oBAEvB,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,2CAA2C,IAAI,CAAC,OAAO,EAAE;wBAClE,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,+BAA+B;oBAC/B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,MAAM,kBAAkB,GAAG,UAS1B,CAAC;wBACF,kBAAkB,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;oBAC1D,CAAC;oBAED,IACE,IAAI,CAAC,qBAAqB;wBAC1B,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EACrC,CAAC;wBACD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC5D,IAAI,CAAC,qBAAqB,EAC1B,MAAM,CACP,CAAC;wBAEF,IAAI,kBAAkB,EAAE,CAAC;4BACvB,MAAM,kBAAkB,GAAG,UAQ1B,CAAC;4BACF,kBAAkB,CAAC,0BAA0B;gCAC3C,kBAAkB,CAAC;wBACvB,CAAC;oBACH,CAAC;oBAED,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;wBAC1C,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAEzD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,kCAAkC,YAAY,EAAE;wBACzD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,IAAI,CAAC;wBACH,qCAAqC;wBACrC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;wBAElD,MAAM,eAAe,GAAG;4BACtB,IAAI,EAAE,sBAA+B;4BACrC,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE;gCACN,IAAI,EAAE,aAAsB;gCAC5B,SAAS,EAAE,UAAU;gCACrB,KAAK,EAAE,YAAY;6BACpB;yBACmB,CAAC;wBAEvB,+BAA+B;wBAC/B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;4BACpB,MAAM,kBAAkB,GAAG,eAS1B,CAAC;4BACF,kBAAkB,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;wBAC1D,CAAC;wBAED,IACE,IAAI,CAAC,qBAAqB;4BAC1B,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EACrC,CAAC;4BACD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC5D,IAAI,CAAC,qBAAqB,EAC1B,MAAM,CACP,CAAC;4BAEF,IAAI,kBAAkB,EAAE,CAAC;gCACvB,MAAM,kBAAkB,GAAG,eAQ1B,CAAC;gCACF,kBAAkB,CAAC,0BAA0B;oCAC3C,kBAAkB,CAAC;4BACvB,CAAC;wBACH,CAAC;wBAED,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;oBACvC,CAAC;oBAAC,OAAO,eAAe,EAAE,CAAC;wBACzB,IAAI,eAAe,YAAY,oBAAoB,EAAE,CAAC;4BACpD,MAAM,eAAe,CAAC;wBACxB,CAAC;wBACD,wDAAwD;wBACxD,MAAM,CAAC;4BACL,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,+BAA+B,MAAM,CAAC,eAAe,CAAC,EAAE;4BACjE,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;wBAEH,uEAAuE;wBACvE,cAAc,CAAC,IAAI,CAAC;4BAClB,IAAI,EAAE,sBAAsB;4BAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,UAAU,YAAY,EAAE;yBACZ,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,IACL,IAAI,CAAC,IAAI,KAAK,eAAe;gBAC7B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAC7B,CAAC;gBACD,mEAAmE;gBACnE,gEAAgE;gBAChE,IAAI,IAAI,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;oBACvC,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;oBACrC,cAAc,CAAC,IAAI,CAAC;wBAClB,IAAI,EAAE,sBAAsB;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EACJ,+HAA+H;qBAC7G,CAAC,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,qCAAqC;gBACrC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC;oBAEtD,IAAI,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACnC,CAAC;oBAED,gCAAgC;oBAChC,IAAI,UAAU,GAAG,4BAA4B,CAAC;oBAC9C,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;wBAC1C,IAAI,CAAC;4BACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACnC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;4BAExC,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,SAAS,EAAE;gCACzE,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;4BAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;gCACtC,UAAU,EAAE,IAAI,CAAC,OAAO;gCACxB,QAAQ,EAAE,EAAE;6BACb,CAAC,CAAC;4BACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BAEpC,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,QAAQ,IAAI,CAAC,IAAI,oCAAoC,UAAU,EAAE;gCAC1E,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,SAAS,EAAE,CAAC;4BACnB,MAAM,YAAY,GAChB,SAAS,YAAY,KAAK;gCACxB,CAAC,CAAC,SAAS,CAAC,OAAO;gCACnB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;4BACxB,UAAU,GAAG,yBAAyB,YAAY,EAAE,CAAC;4BAErD,MAAM,CAAC;gCACL,QAAQ,EAAE,OAAO;gCACjB,OAAO,EAAE,wBAAwB,IAAI,CAAC,IAAI,KAAK,YAAY,EAAE;gCAC7D,KAAK,EAAE,CAAC;6BACT,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,qDAAqD;oBACrD,MAAM,UAAU,GAAsB;wBACpC,IAAI,EAAE,sBAAsB;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE,UAAU;qBACnB,CAAC;oBAEF,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;wBAC1C,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAEzD,MAAM,CAAC;wBACL,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,kCAAkC,YAAY,EAAE;wBACzD,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBAEH,yBAAyB;oBACzB,MAAM,eAAe,GAAsB;wBACzC,IAAI,EAAE,sBAAsB;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE,UAAU,YAAY,EAAE;qBACjC,CAAC;oBAEF,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,2BAA2B,CACjC,IAAsB;QAEtB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAExB,2FAA2F;QAC3F,6FAA6F;QAC7F,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAc;YAC3B,GAAG,MAAM,EAAE,wCAAwC;SACpD,CAAC;IACJ,CAAC;IAEO,iBAAiB;QACvB,IAAI,IAAI,CAAC,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC5C,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,2BAA2B,CACjC,IAAsB;QAEtB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExC,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,IAAI;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAGvB;QACC,oCAAoC;QACpC,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;YACzB,OAAO,yBAAyB,OAAO,CAAC,WAAW,EAAE,CAAC;QACxD,CAAC;QAED,2CAA2C;QAC3C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACpD,OAAO,yBAAyB,WAAW,EAAE,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;gBACpD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,MAAM,IAAI,4BAA4B,CACpC,yCAAyC;YACvC,iGAAiG,CACpG,CAAC;IACJ,CAAC;CACF","sourcesContent":["import OpenAI from \"openai\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport {\n  AgentAction,\n  AgentResult,\n  AgentType,\n  AgentExecutionOptions,\n  ResponseInputItem,\n  ResponseItem,\n  ComputerCallItem,\n  FunctionCallItem,\n  SafetyCheck,\n  SafetyConfirmationHandler,\n} from \"../types/public/agent.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport {\n  AgentScreenshotProviderError,\n  StagehandClosedError,\n} from \"../types/public/sdkErrors.js\";\nimport { ToolSet } from \"ai\";\nimport {\n  FlowLogger,\n  extractLlmCuaPromptSummary,\n  extractLlmCuaResponseSummary,\n} from \"../flowlogger/FlowLogger.js\";\nimport { v7 as uuidv7 } from \"uuid\";\n\n/**\n * Client for OpenAI's Computer Use Assistant API\n * This implementation uses the official OpenAI Responses API for Computer Use\n */\nconst CAPTCHA_PROCEED_TOOL = \"captchaSolvedProceed\";\n\nexport class OpenAICUAClient extends AgentClient {\n  private pendingContextNotes: string[] = [];\n  private captchaSolvedToolActive = false;\n  private apiKey: string;\n  private organization?: string;\n  private baseURL: string;\n  private client: OpenAI;\n  public lastResponseId?: string;\n  private currentViewport = { width: 1288, height: 711 };\n  private currentUrl?: string;\n  private screenshotProvider?: () => Promise<string>;\n  private actionHandler?: (action: AgentAction) => Promise<void>;\n  private reasoningItems: Map<string, ResponseItem> = new Map();\n  private environment: string = \"browser\"; // \"browser\", \"mac\", \"windows\", or \"ubuntu\"\n  private tools?: ToolSet;\n  private safetyConfirmationHandler?: SafetyConfirmationHandler;\n\n  constructor(\n    type: AgentType,\n    modelName: string,\n    userProvidedInstructions?: string,\n    clientOptions?: ClientOptions,\n    tools?: ToolSet,\n  ) {\n    super(type, modelName, userProvidedInstructions);\n\n    // Process client options\n    this.apiKey =\n      (clientOptions?.apiKey as string) || process.env.OPENAI_API_KEY || \"\";\n    this.baseURL = (clientOptions?.baseURL as string) || undefined;\n    this.organization =\n      (clientOptions?.organization as string) || process.env.OPENAI_ORG;\n\n    // Get environment if specified\n    if (\n      clientOptions?.environment &&\n      typeof clientOptions.environment === \"string\"\n    ) {\n      this.environment = clientOptions.environment;\n    }\n\n    // Store client options for reference\n    this.clientOptions = {\n      apiKey: this.apiKey,\n    };\n\n    if (this.baseURL) {\n      this.clientOptions.baseURL = this.baseURL;\n    }\n\n    // Initialize the OpenAI client\n    this.client = new OpenAI(this.clientOptions);\n\n    this.tools = tools;\n  }\n\n  setViewport(width: number, height: number): void {\n    this.currentViewport = { width, height };\n  }\n\n  setCurrentUrl(url: string): void {\n    this.currentUrl = url;\n  }\n\n  setScreenshotProvider(provider: () => Promise<string>): void {\n    this.screenshotProvider = provider;\n  }\n\n  setActionHandler(handler: (action: AgentAction) => Promise<void>): void {\n    this.actionHandler = handler;\n  }\n\n  setTools(tools: ToolSet): void {\n    this.tools = tools;\n  }\n\n  setSafetyConfirmationHandler(handler?: SafetyConfirmationHandler): void {\n    this.safetyConfirmationHandler = handler;\n  }\n\n  addContextNote(note: string): void {\n    this.pendingContextNotes.push(note);\n\n    // When a captcha-related note arrives, expose a tool that the model can\n    // call instead of asking the user for confirmation.  This replaces\n    // fragile English-phrase parsing with a structured tool call.\n    if (note.toLowerCase().includes(\"captcha\")) {\n      this.captchaSolvedToolActive = true;\n    }\n  }\n\n  /**\n   * Execute a task with the OpenAI CUA\n   * This is the main entry point for the agent\n   * @implements AgentClient.execute\n   */\n  async execute(executionOptions: AgentExecutionOptions): Promise<AgentResult> {\n    const { options, logger } = executionOptions;\n    const { instruction } = options;\n    const maxSteps = options.maxSteps || 10;\n\n    let currentStep = 0;\n    let completed = false;\n    const actions: AgentAction[] = [];\n    const messageList: string[] = [];\n    let finalMessage = \"\";\n    this.reasoningItems.clear(); // Clear any previous reasoning items\n\n    // Start with the initial instruction\n    let inputItems = this.createInitialInputItems(instruction);\n    let previousResponseId: string | undefined = undefined;\n    let totalInputTokens = 0;\n    let totalOutputTokens = 0;\n    let totalInferenceTime = 0;\n\n    try {\n      // Execute steps until completion or max steps reached\n      while (!completed && currentStep < maxSteps) {\n        await this.preStepHook?.();\n\n        logger({\n          category: \"agent\",\n          message: `Executing step ${currentStep + 1}/${maxSteps}`,\n          level: 1,\n        });\n\n        const result = await this.executeStep(\n          inputItems,\n          previousResponseId,\n          logger,\n        );\n        totalInputTokens += result.usage.input_tokens;\n        totalOutputTokens += result.usage.output_tokens;\n        totalInferenceTime += result.usage.inference_time_ms;\n\n        // Add actions to the list\n        actions.push(...result.actions);\n\n        // Update completion status\n        completed = result.completed;\n\n        // Store the previous response ID for the next request\n        previousResponseId = result.responseId;\n\n        // Update the input items for the next step if we're continuing\n        if (!completed) {\n          inputItems = result.nextInputItems;\n          const contextNotes = this.drainContextNotes();\n          if (contextNotes.length > 0) {\n            inputItems = [\n              ...inputItems,\n              ...contextNotes.map((note) => ({\n                role: \"user\" as const,\n                content: note,\n              })),\n            ];\n          }\n        }\n\n        // Record any message for this step\n        if (result.message) {\n          messageList.push(result.message);\n          finalMessage = result.message;\n        }\n\n        // Increment step counter\n        currentStep++;\n      }\n\n      // Return the final result\n      return {\n        success: completed,\n        actions,\n        message: finalMessage,\n        completed,\n        usage: {\n          input_tokens: totalInputTokens,\n          output_tokens: totalOutputTokens,\n          inference_time_ms: totalInferenceTime,\n        },\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      logger({\n        category: \"agent\",\n        message: `Error executing agent task: ${errorMessage}`,\n        level: 0,\n      });\n\n      return {\n        success: false,\n        actions,\n        message: `Failed to execute task: ${errorMessage}`,\n        completed: false,\n        usage: {\n          input_tokens: totalInputTokens,\n          output_tokens: totalOutputTokens,\n          inference_time_ms: totalInferenceTime,\n        },\n      };\n    }\n  }\n\n  /**\n   * Execute a single step of the agent\n   * This coordinates the flow: Request → Get Action → Execute Action\n   */\n  async executeStep(\n    inputItems: ResponseInputItem[],\n    previousResponseId: string | undefined,\n    logger: (message: LogLine) => void,\n  ): Promise<{\n    actions: AgentAction[];\n    message: string;\n    completed: boolean;\n    nextInputItems: ResponseInputItem[];\n    responseId: string;\n    usage: {\n      input_tokens: number;\n      output_tokens: number;\n      inference_time_ms: number;\n    };\n  }> {\n    try {\n      // Get response from the model\n      const result = await this.getAction(inputItems, previousResponseId);\n      const output = result.output;\n      const responseId = result.responseId;\n      const usage = {\n        input_tokens: result.usage.input_tokens,\n        output_tokens: result.usage.output_tokens,\n        inference_time_ms: result.usage.inference_time_ms,\n      };\n\n      // Add any reasoning items to our map\n      for (const item of output) {\n        if (item.type === \"reasoning\") {\n          this.reasoningItems.set(item.id, item);\n          logger({\n            category: \"agent\",\n            message: `Reasoning: ${String(item.content || \"\")}`,\n            level: 1,\n          });\n        }\n      }\n\n      // Extract actions from the output\n      const stepActions: AgentAction[] = [];\n      for (const item of output) {\n        if (item.type === \"computer_call\" && this.isComputerCallItem(item)) {\n          logger({\n            category: \"agent\",\n            message: `Found computer_call: ${item.action.type}, payload: ${JSON.stringify(item.action)}, call_id: ${item.call_id}`,\n            level: 2,\n          });\n          const action = this.convertComputerCallToAction(item);\n          if (action) {\n            stepActions.push(action);\n            logger({\n              category: \"agent\",\n              message: `Converted computer_call to action: ${action.type}`,\n              level: 2,\n            });\n          }\n        } else if (\n          item.type === \"function_call\" &&\n          this.isFunctionCallItem(item)\n        ) {\n          logger({\n            category: \"agent\",\n            message: `Found function_call: ${item.name}, call_id: ${item.call_id}`,\n            level: 2,\n          });\n          const action = this.convertFunctionCallToAction(item);\n          if (action) {\n            stepActions.push(action);\n            logger({\n              category: \"agent\",\n              message: `Converted function_call to action: ${action.type}`,\n              level: 2,\n            });\n          }\n        }\n      }\n\n      // Extract message text\n      let message = \"\";\n      for (const item of output) {\n        if (item.type === \"message\") {\n          logger({\n            category: \"agent\",\n            message: `Found message block`,\n            level: 2,\n          });\n          if (item.content && Array.isArray(item.content)) {\n            for (const content of item.content) {\n              if (content.type === \"output_text\" && content.text) {\n                message += content.text + \"\\n\";\n                logger({\n                  category: \"agent\",\n                  message: `Message text: ${String(content.text || \"\")}`,\n                  level: 1,\n                });\n              }\n            }\n          }\n        }\n      }\n\n      // Take actions and get results\n      const nextInputItems = await this.takeAction(output, logger);\n\n      // Check if completed\n      const completed =\n        output.length === 0 ||\n        output.every(\n          (item) => item.type === \"message\" || item.type === \"reasoning\",\n        );\n\n      return {\n        actions: stepActions,\n        message: message.trim(),\n        completed,\n        nextInputItems,\n        responseId,\n        usage: usage,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      logger({\n        category: \"agent\",\n        message: `Error executing step: ${errorMessage}`,\n        level: 0,\n      });\n\n      throw error;\n    }\n  }\n\n  private isComputerCallItem(item: ResponseItem): item is ComputerCallItem {\n    return (\n      item.type === \"computer_call\" &&\n      \"call_id\" in item &&\n      \"action\" in item &&\n      typeof item.action === \"object\"\n    );\n  }\n\n  private async handleSafetyConfirmation(\n    pendingSafetyChecks: SafetyCheck[],\n    logger: (message: LogLine) => void,\n  ): Promise<SafetyCheck[] | undefined> {\n    if (this.safetyConfirmationHandler) {\n      logger({\n        category: \"agent\",\n        message: `Requesting safety confirmation for ${pendingSafetyChecks.length} check(s): ${pendingSafetyChecks.map((c) => c.code).join(\", \")}`,\n        level: 1,\n      });\n\n      const response =\n        await this.safetyConfirmationHandler(pendingSafetyChecks);\n\n      if (response.acknowledged) {\n        logger({\n          category: \"agent\",\n          message: `Safety checks acknowledged by user`,\n          level: 1,\n        });\n        return pendingSafetyChecks;\n      } else {\n        logger({\n          category: \"agent\",\n          message: `Safety checks rejected by user`,\n          level: 1,\n        });\n        return undefined;\n      }\n    }\n\n    logger({\n      category: \"agent\",\n      message: `Auto-acknowledging ${pendingSafetyChecks.length} safety check(s)`,\n      level: 2,\n    });\n    return pendingSafetyChecks;\n  }\n\n  private isFunctionCallItem(item: ResponseItem): item is FunctionCallItem {\n    return (\n      item.type === \"function_call\" &&\n      \"call_id\" in item &&\n      \"name\" in item &&\n      \"arguments\" in item\n    );\n  }\n\n  private createInitialInputItems(instruction: string): ResponseInputItem[] {\n    // For the initial request, we use a simple array with the user's instruction\n    return [\n      {\n        role: \"system\",\n        content: this.userProvidedInstructions,\n      },\n      {\n        role: \"user\",\n        content: instruction,\n      },\n    ];\n  }\n\n  async getAction(\n    inputItems: ResponseInputItem[],\n    previousResponseId?: string,\n  ): Promise<{\n    output: ResponseItem[];\n    responseId: string;\n    usage: Record<string, number>;\n  }> {\n    try {\n      // Create the request parameters\n      const requestParams: Record<string, unknown> = {\n        model: this.modelName,\n        tools: [\n          {\n            type: \"computer_use_preview\",\n            display_width: this.currentViewport.width,\n            display_height: this.currentViewport.height,\n            environment: this.environment,\n          },\n        ],\n        input: inputItems,\n        truncation: \"auto\",\n      };\n\n      // Add custom tools if available\n      if (this.tools && Object.keys(this.tools).length > 0) {\n        const customTools = Object.entries(this.tools).map(([name, tool]) => ({\n          type: \"function\" as const,\n          name,\n          function: {\n            name,\n            description: tool.description,\n            parameters: tool.inputSchema,\n          },\n        }));\n\n        requestParams.tools = [\n          ...(requestParams.tools as Record<string, unknown>[]),\n          ...customTools,\n        ];\n      }\n\n      // When a captcha was just solved, expose a tool the model can call\n      // to confirm it should proceed.  This avoids fragile English-phrase\n      // parsing and works regardless of the model's output language.\n      if (this.captchaSolvedToolActive) {\n        requestParams.tools = [\n          ...(requestParams.tools as Record<string, unknown>[]),\n          {\n            type: \"function\" as const,\n            name: CAPTCHA_PROCEED_TOOL,\n            function: {\n              name: CAPTCHA_PROCEED_TOOL,\n              description:\n                \"The captcha on this page was solved automatically. \" +\n                \"Call this tool to confirm and continue with your task \" +\n                \"instead of asking the user for permission.\",\n              parameters: { type: \"object\", properties: {}, required: [] },\n            },\n          },\n        ];\n      }\n\n      // Add previous_response_id if available\n      if (previousResponseId) {\n        requestParams.previous_response_id = previousResponseId;\n      }\n\n      // Log LLM request\n      const llmRequestId = uuidv7();\n      FlowLogger.logLlmRequest({\n        requestId: llmRequestId,\n        model: this.modelName,\n        prompt: extractLlmCuaPromptSummary(inputItems),\n      });\n\n      const startTime = Date.now();\n      // Create the response using the OpenAI Responses API\n      // @ts-expect-error - Force type to match what the OpenAI SDK expects\n      const response = await this.client.responses.create(requestParams);\n      const endTime = Date.now();\n      const elapsedMs = endTime - startTime;\n\n      // Extract only the input_tokens and output_tokens\n      const usage = {\n        input_tokens: response.usage.input_tokens,\n        output_tokens: response.usage.output_tokens,\n        inference_time_ms: elapsedMs,\n      };\n\n      // Log LLM response\n      FlowLogger.logLlmResponse({\n        requestId: llmRequestId,\n        model: this.modelName,\n        output: extractLlmCuaResponseSummary(response.output),\n        inputTokens: response.usage.input_tokens,\n        outputTokens: response.usage.output_tokens,\n      });\n\n      // Store the response ID for future use\n      this.lastResponseId = response.id;\n\n      // Return the output and response ID\n      return {\n        output: response.output as unknown as ResponseItem[],\n        responseId: response.id,\n        usage,\n      };\n    } catch (error) {\n      console.error(\"Error getting action from OpenAI:\", error);\n      throw error;\n    }\n  }\n\n  async takeAction(\n    output: ResponseItem[],\n    logger: (message: LogLine) => void,\n  ): Promise<ResponseInputItem[]> {\n    const nextInputItems: ResponseInputItem[] = [];\n\n    // Process each output item\n    for (const item of output) {\n      if (item.type === \"computer_call\" && this.isComputerCallItem(item)) {\n        // Handle computer calls\n        try {\n          const action = this.convertComputerCallToAction(item);\n\n          if (action && this.actionHandler) {\n            logger({\n              category: \"agent\",\n              message: `Executing computer action: ${action.type}`,\n              level: 1,\n            });\n            await this.actionHandler(action);\n          }\n\n          // Capture a screenshot\n          const screenshot = await this.captureScreenshot();\n\n          // Create a computer_call_output for the next request\n          const outputItem = {\n            type: \"computer_call_output\" as const,\n            call_id: item.call_id,\n            output: {\n              type: \"input_image\" as const,\n              image_url: screenshot,\n            },\n          } as ResponseInputItem;\n\n          logger({\n            category: \"agent\",\n            message: `Added computer_call_output for call_id: ${item.call_id}`,\n            level: 2,\n          });\n\n          // Add current URL if available\n          if (this.currentUrl) {\n            const computerCallOutput = outputItem as {\n              type: \"computer_call_output\";\n              call_id: string;\n              output: {\n                type: \"input_image\";\n                image_url: string;\n                current_url?: string;\n              };\n              acknowledged_safety_checks?: SafetyCheck[];\n            };\n            computerCallOutput.output.current_url = this.currentUrl;\n          }\n\n          if (\n            item.pending_safety_checks &&\n            item.pending_safety_checks.length > 0\n          ) {\n            const acknowledgedChecks = await this.handleSafetyConfirmation(\n              item.pending_safety_checks,\n              logger,\n            );\n\n            if (acknowledgedChecks) {\n              const computerCallOutput = outputItem as {\n                type: \"computer_call_output\";\n                call_id: string;\n                output: {\n                  type: \"input_image\";\n                  image_url: string;\n                };\n                acknowledged_safety_checks?: SafetyCheck[];\n              };\n              computerCallOutput.acknowledged_safety_checks =\n                acknowledgedChecks;\n            }\n          }\n\n          nextInputItems.push(outputItem);\n        } catch (error) {\n          if (error instanceof StagehandClosedError) {\n            throw error;\n          }\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n\n          logger({\n            category: \"agent\",\n            message: `Error executing computer call: ${errorMessage}`,\n            level: 0,\n          });\n\n          try {\n            // Capture a screenshot even on error\n            const screenshot = await this.captureScreenshot();\n\n            const errorOutputItem = {\n              type: \"computer_call_output\" as const,\n              call_id: item.call_id,\n              output: {\n                type: \"input_image\" as const,\n                image_url: screenshot,\n                error: errorMessage,\n              },\n            } as ResponseInputItem;\n\n            // Add current URL if available\n            if (this.currentUrl) {\n              const computerCallOutput = errorOutputItem as {\n                type: \"computer_call_output\";\n                call_id: string;\n                output: {\n                  type: \"input_image\";\n                  image_url: string;\n                  current_url?: string;\n                };\n                acknowledged_safety_checks?: SafetyCheck[];\n              };\n              computerCallOutput.output.current_url = this.currentUrl;\n            }\n\n            if (\n              item.pending_safety_checks &&\n              item.pending_safety_checks.length > 0\n            ) {\n              const acknowledgedChecks = await this.handleSafetyConfirmation(\n                item.pending_safety_checks,\n                logger,\n              );\n\n              if (acknowledgedChecks) {\n                const computerCallOutput = errorOutputItem as {\n                  type: \"computer_call_output\";\n                  call_id: string;\n                  output: {\n                    type: \"input_image\";\n                    image_url: string;\n                  };\n                  acknowledged_safety_checks?: SafetyCheck[];\n                };\n                computerCallOutput.acknowledged_safety_checks =\n                  acknowledgedChecks;\n              }\n            }\n\n            nextInputItems.push(errorOutputItem);\n          } catch (screenshotError) {\n            if (screenshotError instanceof StagehandClosedError) {\n              throw screenshotError;\n            }\n            // If we can't capture a screenshot, just send the error\n            logger({\n              category: \"agent\",\n              message: `Error capturing screenshot: ${String(screenshotError)}`,\n              level: 0,\n            });\n\n            // For error cases without a screenshot, we need to use a string output\n            nextInputItems.push({\n              type: \"computer_call_output\",\n              call_id: item.call_id,\n              output: `Error: ${errorMessage}`,\n            } as ResponseInputItem);\n          }\n        }\n      } else if (\n        item.type === \"function_call\" &&\n        this.isFunctionCallItem(item)\n      ) {\n        // Handle the captcha-proceed tool — just return a confirmation and\n        // deactivate the tool so it doesn't appear on subsequent steps.\n        if (item.name === CAPTCHA_PROCEED_TOOL) {\n          this.captchaSolvedToolActive = false;\n          nextInputItems.push({\n            type: \"function_call_output\",\n            call_id: item.call_id,\n            output:\n              \"Confirmed. The captcha is solved. Continue completing the original task autonomously without asking for further confirmation.\",\n          } as ResponseInputItem);\n          continue;\n        }\n\n        // Handle function calls (tool calls)\n        try {\n          const action = this.convertFunctionCallToAction(item);\n\n          if (action && this.actionHandler) {\n            await this.actionHandler(action);\n          }\n\n          // Execute the tool if available\n          let toolResult = \"Tool executed successfully\";\n          if (this.tools && item.name in this.tools) {\n            try {\n              const tool = this.tools[item.name];\n              const args = JSON.parse(item.arguments);\n\n              logger({\n                category: \"agent\",\n                message: `Executing tool call: ${item.name} with args: ${item.arguments}`,\n                level: 1,\n              });\n\n              const result = await tool.execute(args, {\n                toolCallId: item.call_id,\n                messages: [],\n              });\n              toolResult = JSON.stringify(result);\n\n              logger({\n                category: \"agent\",\n                message: `Tool ${item.name} completed successfully. Result: ${toolResult}`,\n                level: 1,\n              });\n            } catch (toolError) {\n              const errorMessage =\n                toolError instanceof Error\n                  ? toolError.message\n                  : String(toolError);\n              toolResult = `Error executing tool: ${errorMessage}`;\n\n              logger({\n                category: \"agent\",\n                message: `Error executing tool ${item.name}: ${errorMessage}`,\n                level: 0,\n              });\n            }\n          }\n\n          // Create a function_call_output for the next request\n          const outputItem: ResponseInputItem = {\n            type: \"function_call_output\",\n            call_id: item.call_id,\n            output: toolResult,\n          };\n\n          nextInputItems.push(outputItem);\n        } catch (error) {\n          if (error instanceof StagehandClosedError) {\n            throw error;\n          }\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n\n          logger({\n            category: \"agent\",\n            message: `Error executing function call: ${errorMessage}`,\n            level: 0,\n          });\n\n          // Send error result back\n          const errorOutputItem: ResponseInputItem = {\n            type: \"function_call_output\",\n            call_id: item.call_id,\n            output: `Error: ${errorMessage}`,\n          };\n\n          nextInputItems.push(errorOutputItem);\n        }\n      }\n    }\n\n    return nextInputItems;\n  }\n\n  private convertComputerCallToAction(\n    call: ComputerCallItem,\n  ): AgentAction | null {\n    const { action } = call;\n\n    // Instead of wrapping the action in a params object, spread the action properties directly\n    // This ensures properties like x, y, button, etc. are directly accessible on the AgentAction\n    return {\n      type: action.type as string,\n      ...action, // Spread all properties from the action\n    };\n  }\n\n  private drainContextNotes(): string[] {\n    if (this.pendingContextNotes.length === 0) {\n      return [];\n    }\n\n    const notes = [...this.pendingContextNotes];\n    this.pendingContextNotes = [];\n    return notes;\n  }\n\n  private convertFunctionCallToAction(\n    call: FunctionCallItem,\n  ): AgentAction | null {\n    try {\n      const args = JSON.parse(call.arguments);\n\n      return {\n        type: call.name,\n        params: args,\n      };\n    } catch (error) {\n      console.error(\"Error parsing function call arguments:\", error);\n      return null;\n    }\n  }\n\n  async captureScreenshot(options?: {\n    base64Image?: string;\n    currentUrl?: string;\n  }): Promise<string> {\n    // Use provided options if available\n    if (options?.base64Image) {\n      return `data:image/png;base64,${options.base64Image}`;\n    }\n\n    // Use the screenshot provider if available\n    if (this.screenshotProvider) {\n      try {\n        const base64Image = await this.screenshotProvider();\n        return `data:image/png;base64,${base64Image}`;\n      } catch (error) {\n        console.error(\"Error capturing screenshot:\", error);\n        throw error;\n      }\n    }\n\n    throw new AgentScreenshotProviderError(\n      \"`screenshotProvider` has not been set. \" +\n        \"Please call `setScreenshotProvider()` with a valid function that returns a base64-encoded image\",\n    );\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.d.ts

@@ -4,8 +4,8 @@ export interface AgentSystemPromptOptions {
     executionInstruction: string;
     mode: AgentToolMode;
     systemInstructions?: string;
-    /** Whether running on Browserbase (enables captcha solver messaging) */
-    isBrowserbase?: boolean;
+    /** Whether captchas are automatically solved by the browser environment */
+    captchasAutoSolve?: boolean;
     /** Tools to exclude from the system prompt */
     excludeTools?: string[];
     /** Variables available to the agent for use in act/type tools */

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js

@@ -1,3 +1,5 @@
+import { CAPTCHA_SYSTEM_PROMPT_NOTE } from "../utils/captchaSolver.js";
+import { getVariablePromptEntries } from "../utils/variables.js";
 function buildToolsSection(isHybridMode, hasSearch, excludeTools) {
     const excludeSet = new Set(excludeTools ?? []);
     const hybridTools = [
@@ -71,7 +73,7 @@ function buildToolsSection(isHybridMode, hasSearch, excludeTools) {
     return `<tools>\n${toolLines}\n  </tools>`;
 }
 export function buildAgentSystemPrompt(options) {
-    const { url, executionInstruction, mode, systemInstructions, isBrowserbase = false, excludeTools, variables, useSearch = false, } = options;
+    const { url, executionInstruction, mode, systemInstructions, captchasAutoSolve = false, excludeTools, variables, useSearch = false, } = options;
     const localeDate = new Date().toLocaleDateString();
     const isoDate = new Date().toISOString();
     const cdata = (text) => `<![CDATA[${text}]]>`;
@@ -132,11 +134,10 @@ export function buildAgentSystemPrompt(options) {
       </secondary_tool>
     </step_1>
   </page_understanding_protocol>`;
-    // Roadblocks section only shown when running on Browserbase (has captcha solver)
-    const roadblocksSection = isBrowserbase
+    // Roadblocks section only shown when captchas are auto-solved
+    const roadblocksSection = captchasAutoSolve
         ? `<roadblocks>
-    <note>captchas, popups, etc.</note>
-    <captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>
+    <note>${CAPTCHA_SYSTEM_PROMPT_NOTE}</note>
   </roadblocks>`
         : "";
     // Build customInstructions block only if provided
@@ -147,17 +148,15 @@ export function buildAgentSystemPrompt(options) {
     const hasVariables = variables && Object.keys(variables).length > 0;
     const variableToolsNote = isHybridMode
         ? "Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields."
-        : "Use %variableName% syntax in the act or fillForm tool's value/action fields.";
+        : "Use %variableName% syntax in the act or fillForm tool's action fields.";
+    const variableEntries = getVariablePromptEntries(variables);
     const variablesSection = hasVariables
         ? `<variables>
     <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>
     <usage>${variableToolsNote}</usage>
     <example>To type a password, use: type %password% into the password field</example>
-    ${Object.entries(variables)
-            .map(([name, v]) => {
-            const description = typeof v === "object" && v !== null && "value" in v
-                ? v.description
-                : undefined;
+    ${variableEntries
+            .map(({ name, description }) => {
             return description
                 ? `<variable name="${name}">${description}</variable>`
                 : `<variable name="${name}" />`;

package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"agentSystemPrompt.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/prompts/agentSystemPrompt.ts"],"names":[],"mappings":"AA4BA,SAAS,iBAAiB,CACxB,YAAqB,EACrB,SAAkB,EAClB,YAAuB;IAEvB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,WAAW,GAAqB;QACpC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,OAAO;YACb,WAAW,EACT,qFAAqF;SACxF;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,2FAA2F;SAC9F;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EACT,0LAA0L;SAC7L;QACD,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE;QAChE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,8BAA8B,EAAE;QACrE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,mCAAmC;SACjD;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gDAAgD;SAC9D;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExD,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,8IAA8I;SACjJ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,MAAM,SAAS,GAAG,aAAa;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,SAAS,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,YAAY,SAAS,cAAc,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAiC;IAEjC,MAAM,EACJ,GAAG,EACH,oBAAoB,EACpB,IAAI,EACJ,kBAAkB,EAClB,aAAa,GAAG,KAAK,EACrB,YAAY,EACZ,SAAS,EACT,SAAS,GAAG,KAAK,GAClB,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,IAAI,KAAK,CAAC;IAEtD,MAAM,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC;IACvC,MAAM,SAAS,GAAG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAElE,yDAAyD;IACzD,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9E,iCAAiC;IACjC,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC;YACE,uIAAuI;YACvI,4GAA4G;YAC5G,sIAAsI;YACtI,uHAAuH;YACvH,sFAAsF;SACvF;QACH,CAAC,CAAC;YACE,2FAA2F;YAC3F,oJAAoJ;YACpJ,6HAA6H;YAC7H,mIAAmI;YACnI,wHAAwH;SACzH,CAAC;IAEN,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,mBAAmB,GAAG;sNACwL,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU;;;;KAIrQ,CAAC;IAEJ,oDAAoD;IACpD,MAAM,yBAAyB,GAAG,YAAY;QAC5C,CAAC,CAAC;;;;;;;;;;;;;iCAa2B;QAC7B,CAAC,CAAC;;;;;;;;;;;;;iCAa2B,CAAC;IAEhC,iFAAiF;IACjF,MAAM,iBAAiB,GAAG,aAAa;QACrC,CAAC,CAAC;;;gBAGU;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,kBAAkB;QAChD,CAAC,CAAC,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,2BAA2B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,yDAAyD;IACzD,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,gGAAgG;QAClG,CAAC,CAAC,8EAA8E,CAAC;IACnF,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC;;aAEO,iBAAiB;;MAExB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;aACxB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE;YACjB,MAAM,WAAW,GACf,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC;gBACjD,CAAC,CAAC,CAAC,CAAC,WAAW;gBACf,CAAC,CAAC,SAAS,CAAC;YAChB,OAAO,WAAW;gBAChB,CAAC,CAAC,mBAAmB,IAAI,KAAK,WAAW,aAAa;gBACtD,CAAC,CAAC,mBAAmB,IAAI,MAAM,CAAC;QACpC,CAAC,CAAC;aACD,IAAI,CAAC,QAAQ,CAAC;eACN;QACX,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;IAEL,uBAAuB;YACf,KAAK,CAAC,oBAAoB,CAAC;iCACN,OAAO,KAAK,UAAU;;;;2DAII,GAAG;;;;;;;;;;;;IAY1D,yBAAyB;;;MAGvB,SAAS,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE;;IAEtG,YAAY;;MAEV,eAAe;MACf,mBAAmB;;IAErB,iBAAiB;IACjB,gBAAgB;;;;;;;;UAQV,CAAC;AACX,CAAC","sourcesContent":["import type { AgentToolMode, Variables } from \"../../types/public/agent.js\";\n\nexport interface AgentSystemPromptOptions {\n  url: string;\n  executionInstruction: string;\n  mode: AgentToolMode;\n  systemInstructions?: string;\n  /** Whether running on Browserbase (enables captcha solver messaging) */\n  isBrowserbase?: boolean;\n  /** Tools to exclude from the system prompt */\n  excludeTools?: string[];\n  /** Variables available to the agent for use in act/type tools */\n  variables?: Variables;\n  /** Whether the search tool is enabled for this execution */\n  useSearch?: boolean;\n}\n\n/**\n * Builds the system prompt for the agent based on the tool mode.\n *\n * @param options - The prompt configuration options\n * @returns The formatted system prompt string\n */\ninterface ToolDefinition {\n  name: string;\n  description: string;\n}\n\nfunction buildToolsSection(\n  isHybridMode: boolean,\n  hasSearch: boolean,\n  excludeTools?: string[],\n): string {\n  const excludeSet = new Set(excludeTools ?? []);\n\n  const hybridTools: ToolDefinition[] = [\n    {\n      name: \"screenshot\",\n      description: \"Take a compressed JPEG screenshot for quick visual context\",\n    },\n    {\n      name: \"ariaTree\",\n      description:\n        \"Get an accessibility (ARIA) hybrid tree for full page context\",\n    },\n    {\n      name: \"click\",\n      description:\n        \"Click on an element (PREFERRED - more reliable when element is visible in viewport)\",\n    },\n    {\n      name: \"type\",\n      description:\n        \"Type text into an element (PREFERRED - more reliable when element is visible in viewport)\",\n    },\n    {\n      name: \"act\",\n      description:\n        \"Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.\",\n    },\n    { name: \"dragAndDrop\", description: \"Drag and drop an element\" },\n    { name: \"clickAndHold\", description: \"Click and hold on an element\" },\n    { name: \"keys\", description: \"Press a keyboard key\" },\n    {\n      name: \"fillFormVision\",\n      description: \"Fill out a form using coordinates\",\n    },\n    { name: \"think\", description: \"Think about the task\" },\n    { name: \"extract\", description: \"Extract structured data\" },\n    { name: \"goto\", description: \"Navigate to a URL\" },\n    { name: \"wait\", description: \"Wait for a specified time\" },\n    { name: \"navback\", description: \"Navigate back in browser history\" },\n    { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n  ];\n\n  const domTools: ToolDefinition[] = [\n    {\n      name: \"screenshot\",\n      description: \"Take a compressed JPEG screenshot for quick visual context\",\n    },\n    {\n      name: \"ariaTree\",\n      description:\n        \"Get an accessibility (ARIA) hybrid tree for full page context\",\n    },\n    {\n      name: \"act\",\n      description: \"Perform a specific atomic action (click, type)\",\n    },\n    { name: \"keys\", description: \"Press a keyboard key\" },\n    { name: \"fillForm\", description: \"Fill out a form\" },\n    { name: \"think\", description: \"Think about the task\" },\n    { name: \"extract\", description: \"Extract structured data\" },\n    { name: \"goto\", description: \"Navigate to a URL\" },\n    { name: \"wait\", description: \"Wait for a specified time\" },\n    { name: \"navback\", description: \"Navigate back in browser history\" },\n    { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n  ];\n\n  const baseTools = isHybridMode ? hybridTools : domTools;\n\n  if (hasSearch) {\n    baseTools.push({\n      name: \"search\",\n      description:\n        \"Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.\",\n    });\n  }\n\n  const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));\n\n  const toolLines = filteredTools\n    .map((tool) => `    <tool name=\"${tool.name}\">${tool.description}</tool>`)\n    .join(\"\\n\");\n\n  return `<tools>\\n${toolLines}\\n  </tools>`;\n}\n\nexport function buildAgentSystemPrompt(\n  options: AgentSystemPromptOptions,\n): string {\n  const {\n    url,\n    executionInstruction,\n    mode,\n    systemInstructions,\n    isBrowserbase = false,\n    excludeTools,\n    variables,\n    useSearch = false,\n  } = options;\n  const localeDate = new Date().toLocaleDateString();\n  const isoDate = new Date().toISOString();\n  const cdata = (text: string) => `<![CDATA[${text}]]>`;\n\n  const isHybridMode = mode === \"hybrid\";\n  const hasSearch = useSearch || Boolean(process.env.BRAVE_API_KEY);\n\n  // Tools section differs based on mode and excluded tools\n  const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);\n\n  // Strategy differs based on mode\n  const strategyItems = isHybridMode\n    ? [\n        `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,\n        `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,\n        `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,\n        `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,\n        `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,\n      ]\n    : [\n        `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,\n        `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,\n        `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,\n        `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,\n        `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,\n      ];\n\n  const strategySection = strategyItems.join(\"\\n    \");\n\n  const commonStrategyItems = `\n    <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., \"get job listings\", \"extract product details\"). For reading page content or understanding elements, always use ${isHybridMode ? \"screenshot or ariaTree\" : \"ariaTree\"} instead - it's faster and more reliable.</item>\n    <item>Keep actions atomic and verify outcomes before proceeding.</item>\n    <item>For each action, provide clear reasoning about why you're taking that step.</item>\n    <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>\n    `;\n\n  // Page understanding protocol differs based on mode\n  const pageUnderstandingProtocol = isHybridMode\n    ? `<page_understanding_protocol>\n    <step_1>\n      <title>UNDERSTAND THE PAGE</title>\n      <primary_tool>\n        <name>screenshot</name>\n        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n        </primary_tool>\n      <secondary_tool>\n        <name>ariaTree</name>\n        <usage>Get complete page context before taking actions</usage>\n        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n      </secondary_tool>\n    </step_1>\n  </page_understanding_protocol>`\n    : `<page_understanding_protocol>\n    <step_1>\n      <title>UNDERSTAND THE PAGE</title>\n      <primary_tool>\n        <name>ariaTree</name>\n        <usage>Get complete page context before taking actions</usage>\n        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n        </primary_tool>\n      <secondary_tool>\n        <name>screenshot</name>\n        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n      </secondary_tool>\n    </step_1>\n  </page_understanding_protocol>`;\n\n  // Roadblocks section only shown when running on Browserbase (has captcha solver)\n  const roadblocksSection = isBrowserbase\n    ? `<roadblocks>\n    <note>captchas, popups, etc.</note>\n    <captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>\n  </roadblocks>`\n    : \"\";\n\n  // Build customInstructions block only if provided\n  const customInstructionsBlock = systemInstructions\n    ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\\n  `\n    : \"\";\n\n  // Build variables section only if variables are provided\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const variableToolsNote = isHybridMode\n    ? \"Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields.\"\n    : \"Use %variableName% syntax in the act or fillForm tool's value/action fields.\";\n  const variablesSection = hasVariables\n    ? `<variables>\n    <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>\n    <usage>${variableToolsNote}</usage>\n    <example>To type a password, use: type %password% into the password field</example>\n    ${Object.entries(variables)\n      .map(([name, v]) => {\n        const description =\n          typeof v === \"object\" && v !== null && \"value\" in v\n            ? v.description\n            : undefined;\n        return description\n          ? `<variable name=\"${name}\">${description}</variable>`\n          : `<variable name=\"${name}\" />`;\n      })\n      .join(\"\\n    \")}\n  </variables>`\n    : \"\";\n\n  return `<system>\n  <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>\n  ${customInstructionsBlock}<task>\n    <goal>${cdata(executionInstruction)}</goal>\n    <date display=\"local\" iso=\"${isoDate}\">${localeDate}</date>\n    <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>\n  </task>\n  <page>\n    <startingUrl>you are starting your task on this url: ${url}</startingUrl>\n  </page>\n  <mindset>\n    <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>\n    <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>\n    <note>When the task is complete, do not seek more information; you have completed the task.</note>\n  </mindset>\n  <guidelines>\n    <item>Always start by understanding the current page state</item>\n    <item>Use the screenshot tool to verify page state when needed</item>\n    <item>Use appropriate tools for each action</item>\n  </guidelines>\n  ${pageUnderstandingProtocol}\n  <navigation>\n    <rule>If you are confident in the URL, navigate directly to it.</rule>\n    ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}\n  </navigation>\n  ${toolsSection}\n  <strategy>\n    ${strategySection}\n    ${commonStrategyItems}\n  </strategy>\n  ${roadblocksSection}\n  ${variablesSection}\n  <completion>\n    <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>\n    <examples>\n      <example>If you were asked for specific flights, list the flights you found.</example>\n      <example>If you were asked for information about a product, list the product information you were asked for.</example>\n    </examples>\n  </completion>\n</system>`;\n}\n"]}
+{"version":3,"file":"agentSystemPrompt.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/prompts/agentSystemPrompt.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AA4BjE,SAAS,iBAAiB,CACxB,YAAqB,EACrB,SAAkB,EAClB,YAAuB;IAEvB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,WAAW,GAAqB;QACpC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,OAAO;YACb,WAAW,EACT,qFAAqF;SACxF;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,2FAA2F;SAC9F;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EACT,0LAA0L;SAC7L;QACD,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE;QAChE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,8BAA8B,EAAE;QACrE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,mCAAmC;SACjD;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gDAAgD;SAC9D;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExD,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,8IAA8I;SACjJ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,MAAM,SAAS,GAAG,aAAa;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,SAAS,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,YAAY,SAAS,cAAc,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAiC;IAEjC,MAAM,EACJ,GAAG,EACH,oBAAoB,EACpB,IAAI,EACJ,kBAAkB,EAClB,iBAAiB,GAAG,KAAK,EACzB,YAAY,EACZ,SAAS,EACT,SAAS,GAAG,KAAK,GAClB,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,IAAI,KAAK,CAAC;IAEtD,MAAM,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC;IACvC,MAAM,SAAS,GAAG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAElE,yDAAyD;IACzD,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9E,iCAAiC;IACjC,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC;YACE,uIAAuI;YACvI,4GAA4G;YAC5G,sIAAsI;YACtI,uHAAuH;YACvH,sFAAsF;SACvF;QACH,CAAC,CAAC;YACE,2FAA2F;YAC3F,oJAAoJ;YACpJ,6HAA6H;YAC7H,mIAAmI;YACnI,wHAAwH;SACzH,CAAC;IAEN,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,mBAAmB,GAAG;sNACwL,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU;;;;KAIrQ,CAAC;IAEJ,oDAAoD;IACpD,MAAM,yBAAyB,GAAG,YAAY;QAC5C,CAAC,CAAC;;;;;;;;;;;;;iCAa2B;QAC7B,CAAC,CAAC;;;;;;;;;;;;;iCAa2B,CAAC;IAEhC,8DAA8D;IAC9D,MAAM,iBAAiB,GAAG,iBAAiB;QACzC,CAAC,CAAC;YACM,0BAA0B;gBACtB;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,kBAAkB;QAChD,CAAC,CAAC,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,2BAA2B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,yDAAyD;IACzD,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,gGAAgG;QAClG,CAAC,CAAC,wEAAwE,CAAC;IAC7E,MAAM,eAAe,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC5D,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC;;aAEO,iBAAiB;;MAExB,eAAe;aACd,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE;YAC7B,OAAO,WAAW;gBAChB,CAAC,CAAC,mBAAmB,IAAI,KAAK,WAAW,aAAa;gBACtD,CAAC,CAAC,mBAAmB,IAAI,MAAM,CAAC;QACpC,CAAC,CAAC;aACD,IAAI,CAAC,QAAQ,CAAC;eACN;QACX,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;IAEL,uBAAuB;YACf,KAAK,CAAC,oBAAoB,CAAC;iCACN,OAAO,KAAK,UAAU;;;;2DAII,GAAG;;;;;;;;;;;;IAY1D,yBAAyB;;;MAGvB,SAAS,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE;;IAEtG,YAAY;;MAEV,eAAe;MACf,mBAAmB;;IAErB,iBAAiB;IACjB,gBAAgB;;;;;;;;UAQV,CAAC;AACX,CAAC","sourcesContent":["import type { AgentToolMode, Variables } from \"../../types/public/agent.js\";\nimport { CAPTCHA_SYSTEM_PROMPT_NOTE } from \"../utils/captchaSolver.js\";\nimport { getVariablePromptEntries } from \"../utils/variables.js\";\n\nexport interface AgentSystemPromptOptions {\n  url: string;\n  executionInstruction: string;\n  mode: AgentToolMode;\n  systemInstructions?: string;\n  /** Whether captchas are automatically solved by the browser environment */\n  captchasAutoSolve?: boolean;\n  /** Tools to exclude from the system prompt */\n  excludeTools?: string[];\n  /** Variables available to the agent for use in act/type tools */\n  variables?: Variables;\n  /** Whether the search tool is enabled for this execution */\n  useSearch?: boolean;\n}\n\n/**\n * Builds the system prompt for the agent based on the tool mode.\n *\n * @param options - The prompt configuration options\n * @returns The formatted system prompt string\n */\ninterface ToolDefinition {\n  name: string;\n  description: string;\n}\n\nfunction buildToolsSection(\n  isHybridMode: boolean,\n  hasSearch: boolean,\n  excludeTools?: string[],\n): string {\n  const excludeSet = new Set(excludeTools ?? []);\n\n  const hybridTools: ToolDefinition[] = [\n    {\n      name: \"screenshot\",\n      description: \"Take a compressed JPEG screenshot for quick visual context\",\n    },\n    {\n      name: \"ariaTree\",\n      description:\n        \"Get an accessibility (ARIA) hybrid tree for full page context\",\n    },\n    {\n      name: \"click\",\n      description:\n        \"Click on an element (PREFERRED - more reliable when element is visible in viewport)\",\n    },\n    {\n      name: \"type\",\n      description:\n        \"Type text into an element (PREFERRED - more reliable when element is visible in viewport)\",\n    },\n    {\n      name: \"act\",\n      description:\n        \"Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.\",\n    },\n    { name: \"dragAndDrop\", description: \"Drag and drop an element\" },\n    { name: \"clickAndHold\", description: \"Click and hold on an element\" },\n    { name: \"keys\", description: \"Press a keyboard key\" },\n    {\n      name: \"fillFormVision\",\n      description: \"Fill out a form using coordinates\",\n    },\n    { name: \"think\", description: \"Think about the task\" },\n    { name: \"extract\", description: \"Extract structured data\" },\n    { name: \"goto\", description: \"Navigate to a URL\" },\n    { name: \"wait\", description: \"Wait for a specified time\" },\n    { name: \"navback\", description: \"Navigate back in browser history\" },\n    { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n  ];\n\n  const domTools: ToolDefinition[] = [\n    {\n      name: \"screenshot\",\n      description: \"Take a compressed JPEG screenshot for quick visual context\",\n    },\n    {\n      name: \"ariaTree\",\n      description:\n        \"Get an accessibility (ARIA) hybrid tree for full page context\",\n    },\n    {\n      name: \"act\",\n      description: \"Perform a specific atomic action (click, type)\",\n    },\n    { name: \"keys\", description: \"Press a keyboard key\" },\n    { name: \"fillForm\", description: \"Fill out a form\" },\n    { name: \"think\", description: \"Think about the task\" },\n    { name: \"extract\", description: \"Extract structured data\" },\n    { name: \"goto\", description: \"Navigate to a URL\" },\n    { name: \"wait\", description: \"Wait for a specified time\" },\n    { name: \"navback\", description: \"Navigate back in browser history\" },\n    { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n  ];\n\n  const baseTools = isHybridMode ? hybridTools : domTools;\n\n  if (hasSearch) {\n    baseTools.push({\n      name: \"search\",\n      description:\n        \"Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.\",\n    });\n  }\n\n  const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));\n\n  const toolLines = filteredTools\n    .map((tool) => `    <tool name=\"${tool.name}\">${tool.description}</tool>`)\n    .join(\"\\n\");\n\n  return `<tools>\\n${toolLines}\\n  </tools>`;\n}\n\nexport function buildAgentSystemPrompt(\n  options: AgentSystemPromptOptions,\n): string {\n  const {\n    url,\n    executionInstruction,\n    mode,\n    systemInstructions,\n    captchasAutoSolve = false,\n    excludeTools,\n    variables,\n    useSearch = false,\n  } = options;\n  const localeDate = new Date().toLocaleDateString();\n  const isoDate = new Date().toISOString();\n  const cdata = (text: string) => `<![CDATA[${text}]]>`;\n\n  const isHybridMode = mode === \"hybrid\";\n  const hasSearch = useSearch || Boolean(process.env.BRAVE_API_KEY);\n\n  // Tools section differs based on mode and excluded tools\n  const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);\n\n  // Strategy differs based on mode\n  const strategyItems = isHybridMode\n    ? [\n        `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,\n        `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,\n        `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,\n        `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,\n        `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,\n      ]\n    : [\n        `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,\n        `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,\n        `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,\n        `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,\n        `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,\n      ];\n\n  const strategySection = strategyItems.join(\"\\n    \");\n\n  const commonStrategyItems = `\n    <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., \"get job listings\", \"extract product details\"). For reading page content or understanding elements, always use ${isHybridMode ? \"screenshot or ariaTree\" : \"ariaTree\"} instead - it's faster and more reliable.</item>\n    <item>Keep actions atomic and verify outcomes before proceeding.</item>\n    <item>For each action, provide clear reasoning about why you're taking that step.</item>\n    <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>\n    `;\n\n  // Page understanding protocol differs based on mode\n  const pageUnderstandingProtocol = isHybridMode\n    ? `<page_understanding_protocol>\n    <step_1>\n      <title>UNDERSTAND THE PAGE</title>\n      <primary_tool>\n        <name>screenshot</name>\n        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n        </primary_tool>\n      <secondary_tool>\n        <name>ariaTree</name>\n        <usage>Get complete page context before taking actions</usage>\n        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n      </secondary_tool>\n    </step_1>\n  </page_understanding_protocol>`\n    : `<page_understanding_protocol>\n    <step_1>\n      <title>UNDERSTAND THE PAGE</title>\n      <primary_tool>\n        <name>ariaTree</name>\n        <usage>Get complete page context before taking actions</usage>\n        <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n        </primary_tool>\n      <secondary_tool>\n        <name>screenshot</name>\n        <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n      </secondary_tool>\n    </step_1>\n  </page_understanding_protocol>`;\n\n  // Roadblocks section only shown when captchas are auto-solved\n  const roadblocksSection = captchasAutoSolve\n    ? `<roadblocks>\n    <note>${CAPTCHA_SYSTEM_PROMPT_NOTE}</note>\n  </roadblocks>`\n    : \"\";\n\n  // Build customInstructions block only if provided\n  const customInstructionsBlock = systemInstructions\n    ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\\n  `\n    : \"\";\n\n  // Build variables section only if variables are provided\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const variableToolsNote = isHybridMode\n    ? \"Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields.\"\n    : \"Use %variableName% syntax in the act or fillForm tool's action fields.\";\n  const variableEntries = getVariablePromptEntries(variables);\n  const variablesSection = hasVariables\n    ? `<variables>\n    <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>\n    <usage>${variableToolsNote}</usage>\n    <example>To type a password, use: type %password% into the password field</example>\n    ${variableEntries\n      .map(({ name, description }) => {\n        return description\n          ? `<variable name=\"${name}\">${description}</variable>`\n          : `<variable name=\"${name}\" />`;\n      })\n      .join(\"\\n    \")}\n  </variables>`\n    : \"\";\n\n  return `<system>\n  <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>\n  ${customInstructionsBlock}<task>\n    <goal>${cdata(executionInstruction)}</goal>\n    <date display=\"local\" iso=\"${isoDate}\">${localeDate}</date>\n    <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>\n  </task>\n  <page>\n    <startingUrl>you are starting your task on this url: ${url}</startingUrl>\n  </page>\n  <mindset>\n    <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>\n    <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>\n    <note>When the task is complete, do not seek more information; you have completed the task.</note>\n  </mindset>\n  <guidelines>\n    <item>Always start by understanding the current page state</item>\n    <item>Use the screenshot tool to verify page state when needed</item>\n    <item>Use appropriate tools for each action</item>\n  </guidelines>\n  ${pageUnderstandingProtocol}\n  <navigation>\n    <rule>If you are confident in the URL, navigate directly to it.</rule>\n    ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}\n  </navigation>\n  ${toolsSection}\n  <strategy>\n    ${strategySection}\n    ${commonStrategyItems}\n  </strategy>\n  ${roadblocksSection}\n  ${variablesSection}\n  <completion>\n    <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>\n    <examples>\n      <example>If you were asked for specific flights, list the flights you found.</example>\n      <example>If you were asked for information about a product, list the product information you were asked for.</example>\n    </examples>\n  </completion>\n</system>`;\n}\n"]}

package/dist/esm/lib/v3/agent/tools/fillform.d.ts

@@ -4,7 +4,6 @@ import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
 export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
     fields: {
         action: string;
-        value: string;
     }[];
 }, {
     success: boolean;

package/dist/esm/lib/v3/agent/tools/fillform.js

@@ -3,18 +3,15 @@ import { z } from "zod";
 import { TimeoutError } from "../../types/public/sdkErrors.js";
 export const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
     const hasVariables = variables && Object.keys(variables).length > 0;
-    const valueDescription = hasVariables
-        ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
-        : "Text to type into the target";
+    const actionDescription = hasVariables
+        ? `Must follow the pattern: "type <exact value> into the <field name> <fieldType>". Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}. Examples: "type %email% into the email input", "type %password% into the password input"`
+        : 'Must follow the pattern: "type <exact value> into the <field name> <fieldType>". Examples: "type john@example.com into the email input", "type John into the first name input"';
     return tool({
-        description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,
+        description: 'FORM FILL - MULTI-FIELD INPUT TOOL\nFill 2+ form inputs/textareas at once. Each action MUST include the exact text to type and the target field, e.g. "type john@example.com into the email field".',
         inputSchema: z.object({
             fields: z
                 .array(z.object({
-                action: z
-                    .string()
-                    .describe('Description of typing action, e.g. "type foo into the email field"'),
-                value: z.string().describe(valueDescription),
+                action: z.string().describe(actionDescription),
             }))
                 .min(1, "Provide at least one field to fill"),
         }),
@@ -35,8 +32,8 @@ export const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
                     .map((f) => f.action)
                     .join(", ")}`;
                 const observeOptions = executionModel
-                    ? { model: executionModel, timeout: toolTimeout }
-                    : { timeout: toolTimeout };
+                    ? { model: executionModel, variables, timeout: toolTimeout }
+                    : { variables, timeout: toolTimeout };
                 const observeResults = await v3.observe(instruction, observeOptions);
                 const completed = [];
                 const replayableActions = [];

package/dist/esm/lib/v3/agent/tools/fillform.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,+FAA+F,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpI,CAAC,CAAC,8BAA8B,CAAC;IAEnC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE,8GAA8G;QAC3H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,oEAAoE,CACrE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE;oBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n  toolTimeout?: number,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target\";\n\n  return tool({\n    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                'Description of typing action, e.g. \"type foo into the email field\"',\n              ),\n            value: z.string().describe(valueDescription),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillForm`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify(fields),\n              type: \"object\",\n            },\n          },\n        });\n        const instruction = `Return observation results for the following actions: ${fields\n          .map((f) => f.action)\n          .join(\", \")}`;\n\n        const observeOptions = executionModel\n          ? { model: executionModel, timeout: toolTimeout }\n          : { timeout: toolTimeout };\n        const observeResults = await v3.observe(instruction, observeOptions);\n\n        const completed = [] as unknown[];\n        const replayableActions: Action[] = [];\n        for (const res of observeResults) {\n          const actOptions = variables\n            ? { variables, timeout: toolTimeout }\n            : { timeout: toolTimeout };\n          const actResult = await v3.act(res, actOptions);\n          completed.push(actResult);\n          if (Array.isArray(actResult.actions)) {\n            replayableActions.push(...(actResult.actions as Action[]));\n          }\n        }\n        v3.recordAgentReplayStep({\n          type: \"fillForm\",\n          fields,\n          observeResults,\n          actions: replayableActions,\n        });\n        return {\n          success: true,\n          actions: completed,\n          playwrightArguments: replayableActions,\n        };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          throw error;\n        }\n        return {\n          success: false,\n          error: error?.message ?? String(error),\n        };\n      }\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,kJAAkJ,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,4FAA4F;QACjR,CAAC,CAAC,gLAAgL,CAAC;IAErL,OAAO,IAAI,CAAC;QACV,WAAW,EACT,qMAAqM;QACvM,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;oBAC5D,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBACxC,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n  toolTimeout?: number,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const actionDescription = hasVariables\n    ? `Must follow the pattern: \"type <exact value> into the <field name> <fieldType>\". Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}. Examples: \"type %email% into the email input\", \"type %password% into the password input\"`\n    : 'Must follow the pattern: \"type <exact value> into the <field name> <fieldType>\". Examples: \"type john@example.com into the email input\", \"type John into the first name input\"';\n\n  return tool({\n    description:\n      'FORM FILL - MULTI-FIELD INPUT TOOL\\nFill 2+ form inputs/textareas at once. Each action MUST include the exact text to type and the target field, e.g. \"type john@example.com into the email field\".',\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z.string().describe(actionDescription),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillForm`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify(fields),\n              type: \"object\",\n            },\n          },\n        });\n        const instruction = `Return observation results for the following actions: ${fields\n          .map((f) => f.action)\n          .join(\", \")}`;\n\n        const observeOptions = executionModel\n          ? { model: executionModel, variables, timeout: toolTimeout }\n          : { variables, timeout: toolTimeout };\n        const observeResults = await v3.observe(instruction, observeOptions);\n\n        const completed = [] as unknown[];\n        const replayableActions: Action[] = [];\n        for (const res of observeResults) {\n          const actOptions = variables\n            ? { variables, timeout: toolTimeout }\n            : { timeout: toolTimeout };\n          const actResult = await v3.act(res, actOptions);\n          completed.push(actResult);\n          if (Array.isArray(actResult.actions)) {\n            replayableActions.push(...(actResult.actions as Action[]));\n          }\n        }\n        v3.recordAgentReplayStep({\n          type: \"fillForm\",\n          fields,\n          observeResults,\n          actions: replayableActions,\n        });\n        return {\n          success: true,\n          actions: completed,\n          playwrightArguments: replayableActions,\n        };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          throw error;\n        }\n        return {\n          success: false,\n          error: error?.message ?? String(error),\n        };\n      }\n    },\n  });\n};\n"]}