@midscene/web 0.26.6 → 0.26.7-beta-20250815153024.0

package/dist/es/bridge-mode/io-client.mjs

@@ -17,7 +17,7 @@ class BridgeClient {
             this.socket = io(this.endpoint, {
                 reconnection: false,
                 query: {
-                    version: "0.26.6"
+                    version: "0.26.7-beta-20250815153024.0"
                 }
             });
             const timeout = setTimeout(()=>{

package/dist/es/bridge-mode/io-server.mjs

@@ -71,7 +71,7 @@ class BridgeServer {
                     logMsg('one client connected');
                     this.socket = socket;
                     const clientVersion = socket.handshake.query.version;
-                    logMsg(`Bridge connected, cli-side version v0.26.6, browser-side version v${clientVersion}`);
+                    logMsg(`Bridge connected, cli-side version v0.26.7-beta-20250815153024.0, browser-side version v${clientVersion}`);
                     socket.on(BridgeEvent.CallResponse, (params)=>{
                         const id = params.id;
                         const response = params.response;
@@ -99,7 +99,7 @@ class BridgeServer {
                         var _this_onConnect, _this;
                         null == (_this_onConnect = (_this = this).onConnect) || _this_onConnect.call(_this);
                         const payload = {
-                            version: "0.26.6"
+                            version: "0.26.7-beta-20250815153024.0"
                         };
                         socket.emit(BridgeEvent.Connected, payload);
                         Promise.resolve().then(()=>{

package/dist/es/bridge-mode/io-server.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"bridge-mode/io-server.mjs","sources":["webpack://@midscene/web/./src/bridge-mode/io-server.ts"],"sourcesContent":["import { sleep } from '@midscene/core/utils';\nimport { logMsg } from '@midscene/shared/utils';\nimport { Server, type Socket as ServerSocket } from 'socket.io';\nimport { io as ClientIO } from 'socket.io-client';\n\nimport {\n  type BridgeCall,\n  type BridgeCallResponse,\n  BridgeCallTimeout,\n  type BridgeConnectedEventPayload,\n  BridgeErrorCodeNoClientConnected,\n  BridgeEvent,\n  BridgeSignalKill,\n  DefaultBridgeServerPort,\n} from './common';\n\ndeclare const __VERSION__: string;\n\nexport const killRunningServer = async (port?: number) => {\n  try {\n    const client = ClientIO(\n      `ws://localhost:${port || DefaultBridgeServerPort}`,\n      {\n        query: {\n          [BridgeSignalKill]: 1,\n        },\n      },\n    );\n    await sleep(100);\n    await client.close();\n  } catch (e) {\n    // console.error('failed to kill port', e);\n  }\n};\n\n// ws server, this is where the request is sent\nexport class BridgeServer {\n  private callId = 0;\n  private io: Server | null = null;\n  private socket: ServerSocket | null = null;\n  private listeningTimeoutId: NodeJS.Timeout | null = null;\n  private listeningTimerFlag = false;\n  private connectionTipTimer: NodeJS.Timeout | null = null;\n  public calls: Record<string, BridgeCall> = {};\n\n  private connectionLost = false;\n  private connectionLostReason = '';\n\n  constructor(\n    public port: number,\n    public onConnect?: () => void,\n    public onDisconnect?: (reason: string) => void,\n    public closeConflictServer?: boolean,\n  ) {}\n\n  async listen(\n    opts: {\n      timeout?: number | false;\n    } = {},\n  ): Promise<void> {\n    const { timeout = 30000 } = opts;\n\n    if (this.closeConflictServer) {\n      await killRunningServer(this.port);\n    }\n\n    return new Promise((resolve, reject) => {\n      if (this.listeningTimerFlag) {\n        return reject(new Error('already listening'));\n      }\n      this.listeningTimerFlag = true;\n\n      this.listeningTimeoutId = timeout\n        ? setTimeout(() => {\n            reject(\n              new Error(\n                `no extension connected after ${timeout}ms (${BridgeErrorCodeNoClientConnected})`,\n              ),\n            );\n          }, timeout)\n        : null;\n\n      this.connectionTipTimer =\n        !timeout || timeout > 3000\n          ? setTimeout(() => {\n              logMsg('waiting for bridge to connect...');\n            }, 2000)\n          : null;\n      this.io = new Server(this.port, {\n        maxHttpBufferSize: 100 * 1024 * 1024, // 100MB\n      });\n\n      // Listen for the native HTTP server 'listening' event\n      this.io.httpServer.once('listening', () => {\n        resolve();\n      });\n\n      this.io.httpServer.once('error', (err: Error) => {\n        reject(new Error(`Bridge Listening Error: ${err.message}`));\n      });\n\n      this.io.use((socket, next) => {\n        if (this.socket) {\n          next(new Error('server already connected by another client'));\n        }\n        next();\n      });\n\n      this.io.on('connection', (socket) => {\n        // check the connection url\n        const url = socket.handshake.url;\n        if (url.includes(BridgeSignalKill)) {\n          console.warn('kill signal received, closing bridge server');\n          return this.close();\n        }\n\n        this.connectionLost = false;\n        this.connectionLostReason = '';\n        this.listeningTimeoutId && clearTimeout(this.listeningTimeoutId);\n        this.listeningTimeoutId = null;\n        this.connectionTipTimer && clearTimeout(this.connectionTipTimer);\n        this.connectionTipTimer = null;\n        if (this.socket) {\n          socket.emit(BridgeEvent.Refused);\n          // close the socket\n          socket.disconnect();\n\n          return reject(\n            new Error('server already connected by another client'),\n          );\n        }\n\n        try {\n          logMsg('one client connected');\n          this.socket = socket;\n\n          const clientVersion = socket.handshake.query.version;\n          logMsg(\n            `Bridge connected, cli-side version v${__VERSION__}, browser-side version v${clientVersion}`,\n          );\n\n          socket.on(BridgeEvent.CallResponse, (params: BridgeCallResponse) => {\n            const id = params.id;\n            const response = params.response;\n            const error = params.error;\n\n            this.triggerCallResponseCallback(id, error, response);\n          });\n\n          socket.on('disconnect', (reason: string) => {\n            this.connectionLost = true;\n            this.connectionLostReason = reason;\n\n            try {\n              this.io?.close();\n            } catch (e) {\n              // ignore\n            }\n\n            // flush all pending calls as error\n            for (const id in this.calls) {\n              const call = this.calls[id];\n\n              if (!call.responseTime) {\n                const errorMessage = this.connectionLostErrorMsg();\n                this.triggerCallResponseCallback(\n                  id,\n                  new Error(errorMessage),\n                  null,\n                );\n              }\n            }\n\n            this.onDisconnect?.(reason);\n          });\n\n          setTimeout(() => {\n            this.onConnect?.();\n\n            const payload = {\n              version: __VERSION__,\n            } as BridgeConnectedEventPayload;\n            socket.emit(BridgeEvent.Connected, payload);\n            Promise.resolve().then(() => {\n              for (const id in this.calls) {\n                if (this.calls[id].callTime === 0) {\n                  this.emitCall(id);\n                }\n              }\n            });\n          }, 0);\n        } catch (e) {\n          console.error('failed to handle connection event', e);\n          reject(e);\n        }\n      });\n\n      this.io.on('close', () => {\n        this.close();\n      });\n    });\n  }\n\n  private connectionLostErrorMsg = () => {\n    return `Connection lost, reason: ${this.connectionLostReason}`;\n  };\n\n  private async triggerCallResponseCallback(\n    id: string | number,\n    error: Error | null,\n    response: any,\n  ) {\n    const call = this.calls[id];\n    if (!call) {\n      throw new Error(`call ${id} not found`);\n    }\n    call.error = error || undefined;\n    call.response = response;\n    call.responseTime = Date.now();\n\n    call.callback(call.error, response);\n  }\n\n  private async emitCall(id: string) {\n    const call = this.calls[id];\n    if (!call) {\n      throw new Error(`call ${id} not found`);\n    }\n\n    if (this.connectionLost) {\n      const message = `Connection lost, reason: ${this.connectionLostReason}`;\n      call.callback(new Error(message), null);\n      return;\n    }\n\n    if (this.socket) {\n      this.socket.emit(BridgeEvent.Call, {\n        id,\n        method: call.method,\n        args: call.args,\n      });\n      call.callTime = Date.now();\n    }\n  }\n\n  async call<T = any>(\n    method: string,\n    args: any[],\n    timeout = BridgeCallTimeout,\n  ): Promise<T> {\n    const id = `${this.callId++}`;\n\n    return new Promise((resolve, reject) => {\n      const timeoutId = setTimeout(() => {\n        logMsg(`bridge call timeout, id=${id}, method=${method}, args=`, args);\n        this.calls[id].error = new Error(\n          `Bridge call timeout after ${timeout}ms: ${method}`,\n        );\n        reject(this.calls[id].error);\n      }, timeout);\n\n      this.calls[id] = {\n        method,\n        args,\n        response: null,\n        callTime: 0,\n        responseTime: 0,\n        callback: (error: Error | undefined, response: any) => {\n          clearTimeout(timeoutId);\n          if (error) {\n            reject(error);\n          } else {\n            resolve(response);\n          }\n        },\n      };\n\n      this.emitCall(id);\n    });\n  }\n\n  // do NOT restart after close\n  async close() {\n    this.listeningTimeoutId && clearTimeout(this.listeningTimeoutId);\n    this.connectionTipTimer && clearTimeout(this.connectionTipTimer);\n    const closeProcess = this.io?.close();\n    this.io = null;\n\n    return closeProcess;\n  }\n}\n"],"names":["killRunningServer","port","client","ClientIO","DefaultBridgeServerPort","BridgeSignalKill","sleep","e","BridgeServer","opts","timeout","Promise","resolve","reject","Error","setTimeout","BridgeErrorCodeNoClientConnected","logMsg","Server","err","socket","next","url","console","clearTimeout","BridgeEvent","clientVersion","params","id","response","error","reason","_this","_this_io","call","errorMessage","payload","__VERSION__","undefined","Date","message","method","args","BridgeCallTimeout","timeoutId","closeProcess","onConnect","onDisconnect","closeConflictServer"],"mappings":";;;;;;;;;;;;;;;AAkBO,MAAMA,oBAAoB,OAAOC;IACtC,IAAI;QACF,MAAMC,SAASC,GACb,CAAC,eAAe,EAAEF,QAAQG,yBAAyB,EACnD;YACE,OAAO;gBACL,CAACC,iBAAiB,EAAE;YACtB;QACF;QAEF,MAAMC,MAAM;QACZ,MAAMJ,OAAO,KAAK;IACpB,EAAE,OAAOK,GAAG,CAEZ;AACF;AAGO,MAAMC;IAmBX,MAAM,OACJC,OAEI,CAAC,CAAC,EACS;QACf,MAAM,EAAEC,UAAU,KAAK,EAAE,GAAGD;QAE5B,IAAI,IAAI,CAAC,mBAAmB,EAC1B,MAAMT,kBAAkB,IAAI,CAAC,IAAI;QAGnC,OAAO,IAAIW,QAAQ,CAACC,SAASC;YAC3B,IAAI,IAAI,CAAC,kBAAkB,EACzB,OAAOA,OAAO,IAAIC,MAAM;YAE1B,IAAI,CAAC,kBAAkB,GAAG;YAE1B,IAAI,CAAC,kBAAkB,GAAGJ,UACtBK,WAAW;gBACTF,OACE,IAAIC,MACF,CAAC,6BAA6B,EAAEJ,QAAQ,IAAI,EAAEM,iCAAiC,CAAC,CAAC;YAGvF,GAAGN,WACH;YAEJ,IAAI,CAAC,kBAAkB,GACrB,CAACA,WAAWA,UAAU,OAClBK,WAAW;gBACTE,OAAO;YACT,GAAG,QACH;YACN,IAAI,CAAC,EAAE,GAAG,IAAIC,OAAO,IAAI,CAAC,IAAI,EAAE;gBAC9B,mBAAmB;YACrB;YAGA,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa;gBACnCN;YACF;YAEA,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAACO;gBAChCN,OAAO,IAAIC,MAAM,CAAC,wBAAwB,EAAEK,IAAI,OAAO,EAAE;YAC3D;YAEA,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAACC,QAAQC;gBACnB,IAAI,IAAI,CAAC,MAAM,EACbA,KAAK,IAAIP,MAAM;gBAEjBO;YACF;YAEA,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,cAAc,CAACD;gBAExB,MAAME,MAAMF,OAAO,SAAS,CAAC,GAAG;gBAChC,IAAIE,IAAI,QAAQ,CAACjB,mBAAmB;oBAClCkB,QAAQ,IAAI,CAAC;oBACb,OAAO,IAAI,CAAC,KAAK;gBACnB;gBAEA,IAAI,CAAC,cAAc,GAAG;gBACtB,IAAI,CAAC,oBAAoB,GAAG;gBAC5B,IAAI,CAAC,kBAAkB,IAAIC,aAAa,IAAI,CAAC,kBAAkB;gBAC/D,IAAI,CAAC,kBAAkB,GAAG;gBAC1B,IAAI,CAAC,kBAAkB,IAAIA,aAAa,IAAI,CAAC,kBAAkB;gBAC/D,IAAI,CAAC,kBAAkB,GAAG;gBAC1B,IAAI,IAAI,CAAC,MAAM,EAAE;oBACfJ,OAAO,IAAI,CAACK,YAAY,OAAO;oBAE/BL,OAAO,UAAU;oBAEjB,OAAOP,OACL,IAAIC,MAAM;gBAEd;gBAEA,IAAI;oBACFG,OAAO;oBACP,IAAI,CAAC,MAAM,GAAGG;oBAEd,MAAMM,gBAAgBN,OAAO,SAAS,CAAC,KAAK,CAAC,OAAO;oBACpDH,OACE,qEAA6ES,eAAe;oBAG9FN,OAAO,EAAE,CAACK,YAAY,YAAY,EAAE,CAACE;wBACnC,MAAMC,KAAKD,OAAO,EAAE;wBACpB,MAAME,WAAWF,OAAO,QAAQ;wBAChC,MAAMG,QAAQH,OAAO,KAAK;wBAE1B,IAAI,CAAC,2BAA2B,CAACC,IAAIE,OAAOD;oBAC9C;oBAEAT,OAAO,EAAE,CAAC,cAAc,CAACW;4BAwBvBC,oBAAAA;wBAvBA,IAAI,CAAC,cAAc,GAAG;wBACtB,IAAI,CAAC,oBAAoB,GAAGD;wBAE5B,IAAI;gCACFE;oCAAAA,CAAAA,WAAAA,IAAI,CAAC,EAAE,AAAD,KAANA,SAAS,KAAK;wBAChB,EAAE,OAAO1B,GAAG,CAEZ;wBAGA,IAAK,MAAMqB,MAAM,IAAI,CAAC,KAAK,CAAE;4BAC3B,MAAMM,OAAO,IAAI,CAAC,KAAK,CAACN,GAAG;4BAE3B,IAAI,CAACM,KAAK,YAAY,EAAE;gCACtB,MAAMC,eAAe,IAAI,CAAC,sBAAsB;gCAChD,IAAI,CAAC,2BAA2B,CAC9BP,IACA,IAAId,MAAMqB,eACV;4BAEJ;wBACF;gCAEAH,CAAAA,qBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,YAAY,AAAD,KAAhBA,mBAAAA,IAAAA,CAAAA,OAAoBD;oBACtB;oBAEAhB,WAAW;4BACTiB,iBAAAA;gCAAAA,CAAAA,kBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,SAAS,AAAD,KAAbA,gBAAAA,IAAAA,CAAAA;wBAEA,MAAMI,UAAU;4BACd,SAASC;wBACX;wBACAjB,OAAO,IAAI,CAACK,YAAY,SAAS,EAAEW;wBACnCzB,QAAQ,OAAO,GAAG,IAAI,CAAC;4BACrB,IAAK,MAAMiB,MAAM,IAAI,CAAC,KAAK,CACzB,IAAI,AAA4B,MAA5B,IAAI,CAAC,KAAK,CAACA,GAAG,CAAC,QAAQ,EACzB,IAAI,CAAC,QAAQ,CAACA;wBAGpB;oBACF,GAAG;gBACL,EAAE,OAAOrB,GAAG;oBACVgB,QAAQ,KAAK,CAAC,qCAAqChB;oBACnDM,OAAON;gBACT;YACF;YAEA,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS;gBAClB,IAAI,CAAC,KAAK;YACZ;QACF;IACF;IAMA,MAAc,4BACZqB,EAAmB,EACnBE,KAAmB,EACnBD,QAAa,EACb;QACA,MAAMK,OAAO,IAAI,CAAC,KAAK,CAACN,GAAG;QAC3B,IAAI,CAACM,MACH,MAAM,IAAIpB,MAAM,CAAC,KAAK,EAAEc,GAAG,UAAU,CAAC;QAExCM,KAAK,KAAK,GAAGJ,SAASQ;QACtBJ,KAAK,QAAQ,GAAGL;QAChBK,KAAK,YAAY,GAAGK,KAAK,GAAG;QAE5BL,KAAK,QAAQ,CAACA,KAAK,KAAK,EAAEL;IAC5B;IAEA,MAAc,SAASD,EAAU,EAAE;QACjC,MAAMM,OAAO,IAAI,CAAC,KAAK,CAACN,GAAG;QAC3B,IAAI,CAACM,MACH,MAAM,IAAIpB,MAAM,CAAC,KAAK,EAAEc,GAAG,UAAU,CAAC;QAGxC,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAMY,UAAU,CAAC,yBAAyB,EAAE,IAAI,CAAC,oBAAoB,EAAE;YACvEN,KAAK,QAAQ,CAAC,IAAIpB,MAAM0B,UAAU;YAClC;QACF;QAEA,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAACf,YAAY,IAAI,EAAE;gBACjCG;gBACA,QAAQM,KAAK,MAAM;gBACnB,MAAMA,KAAK,IAAI;YACjB;YACAA,KAAK,QAAQ,GAAGK,KAAK,GAAG;QAC1B;IACF;IAEA,MAAM,KACJE,MAAc,EACdC,IAAW,EACXhC,UAAUiC,iBAAiB,EACf;QACZ,MAAMf,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI;QAE7B,OAAO,IAAIjB,QAAQ,CAACC,SAASC;YAC3B,MAAM+B,YAAY7B,WAAW;gBAC3BE,OAAO,CAAC,wBAAwB,EAAEW,GAAG,SAAS,EAAEa,OAAO,OAAO,CAAC,EAAEC;gBACjE,IAAI,CAAC,KAAK,CAACd,GAAG,CAAC,KAAK,GAAG,IAAId,MACzB,CAAC,0BAA0B,EAAEJ,QAAQ,IAAI,EAAE+B,QAAQ;gBAErD5B,OAAO,IAAI,CAAC,KAAK,CAACe,GAAG,CAAC,KAAK;YAC7B,GAAGlB;YAEH,IAAI,CAAC,KAAK,CAACkB,GAAG,GAAG;gBACfa;gBACAC;gBACA,UAAU;gBACV,UAAU;gBACV,cAAc;gBACd,UAAU,CAACZ,OAA0BD;oBACnCL,aAAaoB;oBACb,IAAId,OACFjB,OAAOiB;yBAEPlB,QAAQiB;gBAEZ;YACF;YAEA,IAAI,CAAC,QAAQ,CAACD;QAChB;IACF;IAGA,MAAM,QAAQ;YAGSK;QAFrB,IAAI,CAAC,kBAAkB,IAAIT,aAAa,IAAI,CAAC,kBAAkB;QAC/D,IAAI,CAAC,kBAAkB,IAAIA,aAAa,IAAI,CAAC,kBAAkB;QAC/D,MAAMqB,eAAe,QAAAZ,CAAAA,WAAAA,IAAI,CAAC,EAAE,AAAD,IAANA,KAAAA,IAAAA,SAAS,KAAK;QACnC,IAAI,CAAC,EAAE,GAAG;QAEV,OAAOY;IACT;IAjPA,YACS5C,IAAY,EACZ6C,SAAsB,EACtBC,YAAuC,EACvCC,mBAA6B,CACpC;;;;;QAhBF,uBAAQ,UAAR;QACA,uBAAQ,MAAR;QACA,uBAAQ,UAAR;QACA,uBAAQ,sBAAR;QACA,uBAAQ,sBAAR;QACA,uBAAQ,sBAAR;QACA,uBAAO,SAAP;QAEA,uBAAQ,kBAAR;QACA,uBAAQ,wBAAR;QA6JA,uBAAQ,0BAAR;aA1JS/C,IAAI,GAAJA;aACA6C,SAAS,GAATA;aACAC,YAAY,GAAZA;aACAC,mBAAmB,GAAnBA;aAfD,MAAM,GAAG;aACT,EAAE,GAAkB;aACpB,MAAM,GAAwB;aAC9B,kBAAkB,GAA0B;aAC5C,kBAAkB,GAAG;aACrB,kBAAkB,GAA0B;aAC7C,KAAK,GAA+B,CAAC;aAEpC,cAAc,GAAG;aACjB,oBAAoB,GAAG;aA6JvB,sBAAsB,GAAG,IACxB,CAAC,yBAAyB,EAAE,IAAI,CAAC,oBAAoB,EAAE;IAvJ7D;AA6OL"}
+{"version":3,"file":"bridge-mode/io-server.mjs","sources":["webpack://@midscene/web/./src/bridge-mode/io-server.ts"],"sourcesContent":["import { sleep } from '@midscene/core/utils';\nimport { logMsg } from '@midscene/shared/utils';\nimport { Server, type Socket as ServerSocket } from 'socket.io';\nimport { io as ClientIO } from 'socket.io-client';\n\nimport {\n  type BridgeCall,\n  type BridgeCallResponse,\n  BridgeCallTimeout,\n  type BridgeConnectedEventPayload,\n  BridgeErrorCodeNoClientConnected,\n  BridgeEvent,\n  BridgeSignalKill,\n  DefaultBridgeServerPort,\n} from './common';\n\ndeclare const __VERSION__: string;\n\nexport const killRunningServer = async (port?: number) => {\n  try {\n    const client = ClientIO(\n      `ws://localhost:${port || DefaultBridgeServerPort}`,\n      {\n        query: {\n          [BridgeSignalKill]: 1,\n        },\n      },\n    );\n    await sleep(100);\n    await client.close();\n  } catch (e) {\n    // console.error('failed to kill port', e);\n  }\n};\n\n// ws server, this is where the request is sent\nexport class BridgeServer {\n  private callId = 0;\n  private io: Server | null = null;\n  private socket: ServerSocket | null = null;\n  private listeningTimeoutId: NodeJS.Timeout | null = null;\n  private listeningTimerFlag = false;\n  private connectionTipTimer: NodeJS.Timeout | null = null;\n  public calls: Record<string, BridgeCall> = {};\n\n  private connectionLost = false;\n  private connectionLostReason = '';\n\n  constructor(\n    public port: number,\n    public onConnect?: () => void,\n    public onDisconnect?: (reason: string) => void,\n    public closeConflictServer?: boolean,\n  ) {}\n\n  async listen(\n    opts: {\n      timeout?: number | false;\n    } = {},\n  ): Promise<void> {\n    const { timeout = 30000 } = opts;\n\n    if (this.closeConflictServer) {\n      await killRunningServer(this.port);\n    }\n\n    return new Promise((resolve, reject) => {\n      if (this.listeningTimerFlag) {\n        return reject(new Error('already listening'));\n      }\n      this.listeningTimerFlag = true;\n\n      this.listeningTimeoutId = timeout\n        ? setTimeout(() => {\n            reject(\n              new Error(\n                `no extension connected after ${timeout}ms (${BridgeErrorCodeNoClientConnected})`,\n              ),\n            );\n          }, timeout)\n        : null;\n\n      this.connectionTipTimer =\n        !timeout || timeout > 3000\n          ? setTimeout(() => {\n              logMsg('waiting for bridge to connect...');\n            }, 2000)\n          : null;\n      this.io = new Server(this.port, {\n        maxHttpBufferSize: 100 * 1024 * 1024, // 100MB\n      });\n\n      // Listen for the native HTTP server 'listening' event\n      this.io.httpServer.once('listening', () => {\n        resolve();\n      });\n\n      this.io.httpServer.once('error', (err: Error) => {\n        reject(new Error(`Bridge Listening Error: ${err.message}`));\n      });\n\n      this.io.use((socket, next) => {\n        if (this.socket) {\n          next(new Error('server already connected by another client'));\n        }\n        next();\n      });\n\n      this.io.on('connection', (socket) => {\n        // check the connection url\n        const url = socket.handshake.url;\n        if (url.includes(BridgeSignalKill)) {\n          console.warn('kill signal received, closing bridge server');\n          return this.close();\n        }\n\n        this.connectionLost = false;\n        this.connectionLostReason = '';\n        this.listeningTimeoutId && clearTimeout(this.listeningTimeoutId);\n        this.listeningTimeoutId = null;\n        this.connectionTipTimer && clearTimeout(this.connectionTipTimer);\n        this.connectionTipTimer = null;\n        if (this.socket) {\n          socket.emit(BridgeEvent.Refused);\n          // close the socket\n          socket.disconnect();\n\n          return reject(\n            new Error('server already connected by another client'),\n          );\n        }\n\n        try {\n          logMsg('one client connected');\n          this.socket = socket;\n\n          const clientVersion = socket.handshake.query.version;\n          logMsg(\n            `Bridge connected, cli-side version v${__VERSION__}, browser-side version v${clientVersion}`,\n          );\n\n          socket.on(BridgeEvent.CallResponse, (params: BridgeCallResponse) => {\n            const id = params.id;\n            const response = params.response;\n            const error = params.error;\n\n            this.triggerCallResponseCallback(id, error, response);\n          });\n\n          socket.on('disconnect', (reason: string) => {\n            this.connectionLost = true;\n            this.connectionLostReason = reason;\n\n            try {\n              this.io?.close();\n            } catch (e) {\n              // ignore\n            }\n\n            // flush all pending calls as error\n            for (const id in this.calls) {\n              const call = this.calls[id];\n\n              if (!call.responseTime) {\n                const errorMessage = this.connectionLostErrorMsg();\n                this.triggerCallResponseCallback(\n                  id,\n                  new Error(errorMessage),\n                  null,\n                );\n              }\n            }\n\n            this.onDisconnect?.(reason);\n          });\n\n          setTimeout(() => {\n            this.onConnect?.();\n\n            const payload = {\n              version: __VERSION__,\n            } as BridgeConnectedEventPayload;\n            socket.emit(BridgeEvent.Connected, payload);\n            Promise.resolve().then(() => {\n              for (const id in this.calls) {\n                if (this.calls[id].callTime === 0) {\n                  this.emitCall(id);\n                }\n              }\n            });\n          }, 0);\n        } catch (e) {\n          console.error('failed to handle connection event', e);\n          reject(e);\n        }\n      });\n\n      this.io.on('close', () => {\n        this.close();\n      });\n    });\n  }\n\n  private connectionLostErrorMsg = () => {\n    return `Connection lost, reason: ${this.connectionLostReason}`;\n  };\n\n  private async triggerCallResponseCallback(\n    id: string | number,\n    error: Error | null,\n    response: any,\n  ) {\n    const call = this.calls[id];\n    if (!call) {\n      throw new Error(`call ${id} not found`);\n    }\n    call.error = error || undefined;\n    call.response = response;\n    call.responseTime = Date.now();\n\n    call.callback(call.error, response);\n  }\n\n  private async emitCall(id: string) {\n    const call = this.calls[id];\n    if (!call) {\n      throw new Error(`call ${id} not found`);\n    }\n\n    if (this.connectionLost) {\n      const message = `Connection lost, reason: ${this.connectionLostReason}`;\n      call.callback(new Error(message), null);\n      return;\n    }\n\n    if (this.socket) {\n      this.socket.emit(BridgeEvent.Call, {\n        id,\n        method: call.method,\n        args: call.args,\n      });\n      call.callTime = Date.now();\n    }\n  }\n\n  async call<T = any>(\n    method: string,\n    args: any[],\n    timeout = BridgeCallTimeout,\n  ): Promise<T> {\n    const id = `${this.callId++}`;\n\n    return new Promise((resolve, reject) => {\n      const timeoutId = setTimeout(() => {\n        logMsg(`bridge call timeout, id=${id}, method=${method}, args=`, args);\n        this.calls[id].error = new Error(\n          `Bridge call timeout after ${timeout}ms: ${method}`,\n        );\n        reject(this.calls[id].error);\n      }, timeout);\n\n      this.calls[id] = {\n        method,\n        args,\n        response: null,\n        callTime: 0,\n        responseTime: 0,\n        callback: (error: Error | undefined, response: any) => {\n          clearTimeout(timeoutId);\n          if (error) {\n            reject(error);\n          } else {\n            resolve(response);\n          }\n        },\n      };\n\n      this.emitCall(id);\n    });\n  }\n\n  // do NOT restart after close\n  async close() {\n    this.listeningTimeoutId && clearTimeout(this.listeningTimeoutId);\n    this.connectionTipTimer && clearTimeout(this.connectionTipTimer);\n    const closeProcess = this.io?.close();\n    this.io = null;\n\n    return closeProcess;\n  }\n}\n"],"names":["killRunningServer","port","client","ClientIO","DefaultBridgeServerPort","BridgeSignalKill","sleep","e","BridgeServer","opts","timeout","Promise","resolve","reject","Error","setTimeout","BridgeErrorCodeNoClientConnected","logMsg","Server","err","socket","next","url","console","clearTimeout","BridgeEvent","clientVersion","params","id","response","error","reason","_this","_this_io","call","errorMessage","payload","__VERSION__","undefined","Date","message","method","args","BridgeCallTimeout","timeoutId","closeProcess","onConnect","onDisconnect","closeConflictServer"],"mappings":";;;;;;;;;;;;;;;AAkBO,MAAMA,oBAAoB,OAAOC;IACtC,IAAI;QACF,MAAMC,SAASC,GACb,CAAC,eAAe,EAAEF,QAAQG,yBAAyB,EACnD;YACE,OAAO;gBACL,CAACC,iBAAiB,EAAE;YACtB;QACF;QAEF,MAAMC,MAAM;QACZ,MAAMJ,OAAO,KAAK;IACpB,EAAE,OAAOK,GAAG,CAEZ;AACF;AAGO,MAAMC;IAmBX,MAAM,OACJC,OAEI,CAAC,CAAC,EACS;QACf,MAAM,EAAEC,UAAU,KAAK,EAAE,GAAGD;QAE5B,IAAI,IAAI,CAAC,mBAAmB,EAC1B,MAAMT,kBAAkB,IAAI,CAAC,IAAI;QAGnC,OAAO,IAAIW,QAAQ,CAACC,SAASC;YAC3B,IAAI,IAAI,CAAC,kBAAkB,EACzB,OAAOA,OAAO,IAAIC,MAAM;YAE1B,IAAI,CAAC,kBAAkB,GAAG;YAE1B,IAAI,CAAC,kBAAkB,GAAGJ,UACtBK,WAAW;gBACTF,OACE,IAAIC,MACF,CAAC,6BAA6B,EAAEJ,QAAQ,IAAI,EAAEM,iCAAiC,CAAC,CAAC;YAGvF,GAAGN,WACH;YAEJ,IAAI,CAAC,kBAAkB,GACrB,CAACA,WAAWA,UAAU,OAClBK,WAAW;gBACTE,OAAO;YACT,GAAG,QACH;YACN,IAAI,CAAC,EAAE,GAAG,IAAIC,OAAO,IAAI,CAAC,IAAI,EAAE;gBAC9B,mBAAmB;YACrB;YAGA,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa;gBACnCN;YACF;YAEA,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAACO;gBAChCN,OAAO,IAAIC,MAAM,CAAC,wBAAwB,EAAEK,IAAI,OAAO,EAAE;YAC3D;YAEA,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAACC,QAAQC;gBACnB,IAAI,IAAI,CAAC,MAAM,EACbA,KAAK,IAAIP,MAAM;gBAEjBO;YACF;YAEA,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,cAAc,CAACD;gBAExB,MAAME,MAAMF,OAAO,SAAS,CAAC,GAAG;gBAChC,IAAIE,IAAI,QAAQ,CAACjB,mBAAmB;oBAClCkB,QAAQ,IAAI,CAAC;oBACb,OAAO,IAAI,CAAC,KAAK;gBACnB;gBAEA,IAAI,CAAC,cAAc,GAAG;gBACtB,IAAI,CAAC,oBAAoB,GAAG;gBAC5B,IAAI,CAAC,kBAAkB,IAAIC,aAAa,IAAI,CAAC,kBAAkB;gBAC/D,IAAI,CAAC,kBAAkB,GAAG;gBAC1B,IAAI,CAAC,kBAAkB,IAAIA,aAAa,IAAI,CAAC,kBAAkB;gBAC/D,IAAI,CAAC,kBAAkB,GAAG;gBAC1B,IAAI,IAAI,CAAC,MAAM,EAAE;oBACfJ,OAAO,IAAI,CAACK,YAAY,OAAO;oBAE/BL,OAAO,UAAU;oBAEjB,OAAOP,OACL,IAAIC,MAAM;gBAEd;gBAEA,IAAI;oBACFG,OAAO;oBACP,IAAI,CAAC,MAAM,GAAGG;oBAEd,MAAMM,gBAAgBN,OAAO,SAAS,CAAC,KAAK,CAAC,OAAO;oBACpDH,OACE,2FAA6ES,eAAe;oBAG9FN,OAAO,EAAE,CAACK,YAAY,YAAY,EAAE,CAACE;wBACnC,MAAMC,KAAKD,OAAO,EAAE;wBACpB,MAAME,WAAWF,OAAO,QAAQ;wBAChC,MAAMG,QAAQH,OAAO,KAAK;wBAE1B,IAAI,CAAC,2BAA2B,CAACC,IAAIE,OAAOD;oBAC9C;oBAEAT,OAAO,EAAE,CAAC,cAAc,CAACW;4BAwBvBC,oBAAAA;wBAvBA,IAAI,CAAC,cAAc,GAAG;wBACtB,IAAI,CAAC,oBAAoB,GAAGD;wBAE5B,IAAI;gCACFE;oCAAAA,CAAAA,WAAAA,IAAI,CAAC,EAAE,AAAD,KAANA,SAAS,KAAK;wBAChB,EAAE,OAAO1B,GAAG,CAEZ;wBAGA,IAAK,MAAMqB,MAAM,IAAI,CAAC,KAAK,CAAE;4BAC3B,MAAMM,OAAO,IAAI,CAAC,KAAK,CAACN,GAAG;4BAE3B,IAAI,CAACM,KAAK,YAAY,EAAE;gCACtB,MAAMC,eAAe,IAAI,CAAC,sBAAsB;gCAChD,IAAI,CAAC,2BAA2B,CAC9BP,IACA,IAAId,MAAMqB,eACV;4BAEJ;wBACF;gCAEAH,CAAAA,qBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,YAAY,AAAD,KAAhBA,mBAAAA,IAAAA,CAAAA,OAAoBD;oBACtB;oBAEAhB,WAAW;4BACTiB,iBAAAA;gCAAAA,CAAAA,kBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,SAAS,AAAD,KAAbA,gBAAAA,IAAAA,CAAAA;wBAEA,MAAMI,UAAU;4BACd,SAASC;wBACX;wBACAjB,OAAO,IAAI,CAACK,YAAY,SAAS,EAAEW;wBACnCzB,QAAQ,OAAO,GAAG,IAAI,CAAC;4BACrB,IAAK,MAAMiB,MAAM,IAAI,CAAC,KAAK,CACzB,IAAI,AAA4B,MAA5B,IAAI,CAAC,KAAK,CAACA,GAAG,CAAC,QAAQ,EACzB,IAAI,CAAC,QAAQ,CAACA;wBAGpB;oBACF,GAAG;gBACL,EAAE,OAAOrB,GAAG;oBACVgB,QAAQ,KAAK,CAAC,qCAAqChB;oBACnDM,OAAON;gBACT;YACF;YAEA,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS;gBAClB,IAAI,CAAC,KAAK;YACZ;QACF;IACF;IAMA,MAAc,4BACZqB,EAAmB,EACnBE,KAAmB,EACnBD,QAAa,EACb;QACA,MAAMK,OAAO,IAAI,CAAC,KAAK,CAACN,GAAG;QAC3B,IAAI,CAACM,MACH,MAAM,IAAIpB,MAAM,CAAC,KAAK,EAAEc,GAAG,UAAU,CAAC;QAExCM,KAAK,KAAK,GAAGJ,SAASQ;QACtBJ,KAAK,QAAQ,GAAGL;QAChBK,KAAK,YAAY,GAAGK,KAAK,GAAG;QAE5BL,KAAK,QAAQ,CAACA,KAAK,KAAK,EAAEL;IAC5B;IAEA,MAAc,SAASD,EAAU,EAAE;QACjC,MAAMM,OAAO,IAAI,CAAC,KAAK,CAACN,GAAG;QAC3B,IAAI,CAACM,MACH,MAAM,IAAIpB,MAAM,CAAC,KAAK,EAAEc,GAAG,UAAU,CAAC;QAGxC,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAMY,UAAU,CAAC,yBAAyB,EAAE,IAAI,CAAC,oBAAoB,EAAE;YACvEN,KAAK,QAAQ,CAAC,IAAIpB,MAAM0B,UAAU;YAClC;QACF;QAEA,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAACf,YAAY,IAAI,EAAE;gBACjCG;gBACA,QAAQM,KAAK,MAAM;gBACnB,MAAMA,KAAK,IAAI;YACjB;YACAA,KAAK,QAAQ,GAAGK,KAAK,GAAG;QAC1B;IACF;IAEA,MAAM,KACJE,MAAc,EACdC,IAAW,EACXhC,UAAUiC,iBAAiB,EACf;QACZ,MAAMf,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI;QAE7B,OAAO,IAAIjB,QAAQ,CAACC,SAASC;YAC3B,MAAM+B,YAAY7B,WAAW;gBAC3BE,OAAO,CAAC,wBAAwB,EAAEW,GAAG,SAAS,EAAEa,OAAO,OAAO,CAAC,EAAEC;gBACjE,IAAI,CAAC,KAAK,CAACd,GAAG,CAAC,KAAK,GAAG,IAAId,MACzB,CAAC,0BAA0B,EAAEJ,QAAQ,IAAI,EAAE+B,QAAQ;gBAErD5B,OAAO,IAAI,CAAC,KAAK,CAACe,GAAG,CAAC,KAAK;YAC7B,GAAGlB;YAEH,IAAI,CAAC,KAAK,CAACkB,GAAG,GAAG;gBACfa;gBACAC;gBACA,UAAU;gBACV,UAAU;gBACV,cAAc;gBACd,UAAU,CAACZ,OAA0BD;oBACnCL,aAAaoB;oBACb,IAAId,OACFjB,OAAOiB;yBAEPlB,QAAQiB;gBAEZ;YACF;YAEA,IAAI,CAAC,QAAQ,CAACD;QAChB;IACF;IAGA,MAAM,QAAQ;YAGSK;QAFrB,IAAI,CAAC,kBAAkB,IAAIT,aAAa,IAAI,CAAC,kBAAkB;QAC/D,IAAI,CAAC,kBAAkB,IAAIA,aAAa,IAAI,CAAC,kBAAkB;QAC/D,MAAMqB,eAAe,QAAAZ,CAAAA,WAAAA,IAAI,CAAC,EAAE,AAAD,IAANA,KAAAA,IAAAA,SAAS,KAAK;QACnC,IAAI,CAAC,EAAE,GAAG;QAEV,OAAOY;IACT;IAjPA,YACS5C,IAAY,EACZ6C,SAAsB,EACtBC,YAAuC,EACvCC,mBAA6B,CACpC;;;;;QAhBF,uBAAQ,UAAR;QACA,uBAAQ,MAAR;QACA,uBAAQ,UAAR;QACA,uBAAQ,sBAAR;QACA,uBAAQ,sBAAR;QACA,uBAAQ,sBAAR;QACA,uBAAO,SAAP;QAEA,uBAAQ,kBAAR;QACA,uBAAQ,wBAAR;QA6JA,uBAAQ,0BAAR;aA1JS/C,IAAI,GAAJA;aACA6C,SAAS,GAATA;aACAC,YAAY,GAAZA;aACAC,mBAAmB,GAAnBA;aAfD,MAAM,GAAG;aACT,EAAE,GAAkB;aACpB,MAAM,GAAwB;aAC9B,kBAAkB,GAA0B;aAC5C,kBAAkB,GAAG;aACrB,kBAAkB,GAA0B;aAC7C,KAAK,GAA+B,CAAC;aAEpC,cAAc,GAAG;aACjB,oBAAoB,GAAG;aA6JvB,sBAAsB,GAAG,IACxB,CAAC,yBAAyB,EAAE,IAAI,CAAC,oBAAoB,EAAE;IAvJ7D;AA6OL"}

package/dist/es/bridge-mode/page-browser-side.mjs

@@ -44,7 +44,7 @@ class ExtensionBridgePageBrowserSide extends page {
             }
         }, ()=>this.destroy());
         await this.bridgeClient.connect();
-        this.onLogMessage(`Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v0.26.6`, 'log');
+        this.onLogMessage(`Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v0.26.7-beta-20250815153024.0`, 'log');
     }
     async connect() {
         return await this.setupBridgeClient();

package/dist/es/bridge-mode/page-browser-side.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"bridge-mode/page-browser-side.mjs","sources":["webpack://@midscene/web/./src/bridge-mode/page-browser-side.ts"],"sourcesContent":["import type {\n  ChromePageDestroyOptions,\n  KeyboardAction,\n  MouseAction,\n} from '@/page';\nimport { assert } from '@midscene/shared/utils';\nimport ChromeExtensionProxyPage from '../chrome-extension/page';\nimport {\n  type BridgeConnectTabOptions,\n  BridgeEvent,\n  DefaultBridgeServerPort,\n  KeyboardEvent,\n  MouseEvent,\n} from './common';\nimport { BridgeClient } from './io-client';\n\ndeclare const __VERSION__: string;\n\nexport class ExtensionBridgePageBrowserSide extends ChromeExtensionProxyPage {\n  public bridgeClient: BridgeClient | null = null;\n\n  private destroyOptions?: ChromePageDestroyOptions;\n\n  private newlyCreatedTabIds: number[] = [];\n\n  constructor(\n    public onDisconnect: () => void = () => {},\n    public onLogMessage: (\n      message: string,\n      type: 'log' | 'status',\n    ) => void = () => {},\n    forceSameTabNavigation = true,\n  ) {\n    super(forceSameTabNavigation);\n  }\n\n  private async setupBridgeClient() {\n    this.bridgeClient = new BridgeClient(\n      `ws://localhost:${DefaultBridgeServerPort}`,\n      async (method, args: any[]) => {\n        console.log('bridge call from cli side', method, args);\n        if (method === BridgeEvent.ConnectNewTabWithUrl) {\n          return this.connectNewTabWithUrl.apply(\n            this,\n            args as unknown as [string],\n          );\n        }\n\n        if (method === BridgeEvent.GetBrowserTabList) {\n          return this.getBrowserTabList.apply(this, args as any);\n        }\n\n        if (method === BridgeEvent.SetActiveTabId) {\n          return this.setActiveTabId.apply(this, args as any);\n        }\n\n        if (method === BridgeEvent.ConnectCurrentTab) {\n          return this.connectCurrentTab.apply(this, args as any);\n        }\n\n        if (method === BridgeEvent.UpdateAgentStatus) {\n          return this.onLogMessage(args[0] as string, 'status');\n        }\n\n        const tabId = await this.getActiveTabId();\n        if (!tabId || tabId === 0) {\n          throw new Error('no tab is connected');\n        }\n\n        // this.onLogMessage(`calling method: ${method}`);\n\n        if (method.startsWith(MouseEvent.PREFIX)) {\n          const actionName = method.split('.')[1] as keyof MouseAction;\n          if (actionName === 'drag') {\n            return this.mouse[actionName].apply(this.mouse, args as any);\n          }\n          return this.mouse[actionName].apply(this.mouse, args as any);\n        }\n\n        if (method.startsWith(KeyboardEvent.PREFIX)) {\n          const actionName = method.split('.')[1] as keyof KeyboardAction;\n          if (actionName === 'press') {\n            return this.keyboard[actionName].apply(this.keyboard, args as any);\n          }\n          return this.keyboard[actionName].apply(this.keyboard, args as any);\n        }\n\n        try {\n          // @ts-expect-error\n          const result = await this[method as keyof ChromeExtensionProxyPage](\n            ...args,\n          );\n          return result;\n        } catch (e) {\n          const errorMessage = e instanceof Error ? e.message : 'Unknown error';\n          console.error('error calling method', method, args, e);\n          this.onLogMessage(\n            `Error calling method: ${method}, ${errorMessage}`,\n            'log',\n          );\n          throw new Error(errorMessage, { cause: e });\n        }\n      },\n      // on disconnect\n      () => {\n        return this.destroy();\n      },\n    );\n    await this.bridgeClient.connect();\n    this.onLogMessage(\n      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${__VERSION__}`,\n      'log',\n    );\n  }\n\n  public async connect() {\n    return await this.setupBridgeClient();\n  }\n\n  public async connectNewTabWithUrl(\n    url: string,\n    options: BridgeConnectTabOptions = {\n      forceSameTabNavigation: true,\n    },\n  ) {\n    const tab = await chrome.tabs.create({ url });\n    const tabId = tab.id;\n    assert(tabId, 'failed to get tabId after creating a new tab');\n\n    // new tab\n    this.onLogMessage(`Creating new tab: ${url}`, 'log');\n    this.newlyCreatedTabIds.push(tabId);\n\n    if (options?.forceSameTabNavigation) {\n      this.forceSameTabNavigation = true;\n    }\n\n    await this.setActiveTabId(tabId);\n  }\n\n  public async connectCurrentTab(\n    options: BridgeConnectTabOptions = {\n      forceSameTabNavigation: true,\n    },\n  ) {\n    const tabs = await chrome.tabs.query({ active: true, currentWindow: true });\n    const tabId = tabs[0]?.id;\n    assert(tabId, 'failed to get tabId');\n\n    this.onLogMessage(`Connected to current tab: ${tabs[0]?.url}`, 'log');\n\n    if (options?.forceSameTabNavigation) {\n      this.forceSameTabNavigation = true;\n    }\n\n    await this.setActiveTabId(tabId);\n  }\n\n  public async setDestroyOptions(options: ChromePageDestroyOptions) {\n    this.destroyOptions = options;\n  }\n\n  async destroy() {\n    if (this.destroyOptions?.closeTab && this.newlyCreatedTabIds.length > 0) {\n      this.onLogMessage('Closing all newly created tabs by bridge...', 'log');\n      for (const tabId of this.newlyCreatedTabIds) {\n        await chrome.tabs.remove(tabId);\n      }\n      this.newlyCreatedTabIds = [];\n    }\n\n    await super.destroy();\n\n    if (this.bridgeClient) {\n      this.bridgeClient.disconnect();\n      this.bridgeClient = null;\n      this.onDisconnect();\n    }\n  }\n}\n"],"names":["ExtensionBridgePageBrowserSide","ChromeExtensionProxyPage","BridgeClient","DefaultBridgeServerPort","method","args","console","BridgeEvent","tabId","Error","MouseEvent","actionName","KeyboardEvent","result","e","errorMessage","url","options","tab","chrome","assert","_tabs_","_tabs_1","tabs","_this_destroyOptions","onDisconnect","onLogMessage","forceSameTabNavigation"],"mappings":";;;;;;;;;;;;;;AAkBO,MAAMA,uCAAuCC;IAkBlD,MAAc,oBAAoB;QAChC,IAAI,CAAC,YAAY,GAAG,IAAIC,aACtB,CAAC,eAAe,EAAEC,yBAAyB,EAC3C,OAAOC,QAAQC;YACbC,QAAQ,GAAG,CAAC,6BAA6BF,QAAQC;YACjD,IAAID,WAAWG,YAAY,oBAAoB,EAC7C,OAAO,IAAI,CAAC,oBAAoB,CAAC,KAAK,CACpC,IAAI,EACJF;YAIJ,IAAID,WAAWG,YAAY,iBAAiB,EAC1C,OAAO,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAEF;YAG5C,IAAID,WAAWG,YAAY,cAAc,EACvC,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAEF;YAGzC,IAAID,WAAWG,YAAY,iBAAiB,EAC1C,OAAO,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAEF;YAG5C,IAAID,WAAWG,YAAY,iBAAiB,EAC1C,OAAO,IAAI,CAAC,YAAY,CAACF,IAAI,CAAC,EAAE,EAAY;YAG9C,MAAMG,QAAQ,MAAM,IAAI,CAAC,cAAc;YACvC,IAAI,CAACA,SAASA,AAAU,MAAVA,OACZ,MAAM,IAAIC,MAAM;YAKlB,IAAIL,OAAO,UAAU,CAACM,WAAW,MAAM,GAAG;gBACxC,MAAMC,aAAaP,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE;gBAIvC,OAAO,IAAI,CAAC,KAAK,CAACO,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAEN;YAClD;YAEA,IAAID,OAAO,UAAU,CAACQ,cAAc,MAAM,GAAG;gBAC3C,MAAMD,aAAaP,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE;gBAIvC,OAAO,IAAI,CAAC,QAAQ,CAACO,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAEN;YACxD;YAEA,IAAI;gBAEF,MAAMQ,SAAS,MAAM,IAAI,CAACT,OAAyC,IAC9DC;gBAEL,OAAOQ;YACT,EAAE,OAAOC,GAAG;gBACV,MAAMC,eAAeD,aAAaL,QAAQK,EAAE,OAAO,GAAG;gBACtDR,QAAQ,KAAK,CAAC,wBAAwBF,QAAQC,MAAMS;gBACpD,IAAI,CAAC,YAAY,CACf,CAAC,sBAAsB,EAAEV,OAAO,EAAE,EAAEW,cAAc,EAClD;gBAEF,MAAM,IAAIN,MAAMM,cAAc;oBAAE,OAAOD;gBAAE;YAC3C;QACF,GAEA,IACS,IAAI,CAAC,OAAO;QAGvB,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO;QAC/B,IAAI,CAAC,YAAY,CACf,uCAAuC,IAAI,CAAC,YAAY,CAAC,aAAa,gCAAwC,EAC9G;IAEJ;IAEA,MAAa,UAAU;QACrB,OAAO,MAAM,IAAI,CAAC,iBAAiB;IACrC;IAEA,MAAa,qBACXE,GAAW,EACXC,UAAmC;QACjC,wBAAwB;IAC1B,CAAC,EACD;QACA,MAAMC,MAAM,MAAMC,OAAO,IAAI,CAAC,MAAM,CAAC;YAAEH;QAAI;QAC3C,MAAMR,QAAQU,IAAI,EAAE;QACpBE,OAAOZ,OAAO;QAGd,IAAI,CAAC,YAAY,CAAC,CAAC,kBAAkB,EAAEQ,KAAK,EAAE;QAC9C,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAACR;QAE7B,IAAIS,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,EACjC,IAAI,CAAC,sBAAsB,GAAG;QAGhC,MAAM,IAAI,CAAC,cAAc,CAACT;IAC5B;IAEA,MAAa,kBACXS,UAAmC;QACjC,wBAAwB;IAC1B,CAAC,EACD;YAEcI,QAGiCC;QAJ/C,MAAMC,OAAO,MAAMJ,OAAO,IAAI,CAAC,KAAK,CAAC;YAAE,QAAQ;YAAM,eAAe;QAAK;QACzE,MAAMX,QAAQ,QAAAa,CAAAA,SAAAA,IAAI,CAAC,EAAE,AAAD,IAANA,KAAAA,IAAAA,OAAS,EAAE;QACzBD,OAAOZ,OAAO;QAEd,IAAI,CAAC,YAAY,CAAC,CAAC,0BAA0B,EAAE,QAAAc,CAAAA,UAAAA,IAAI,CAAC,EAAE,AAAD,IAANA,KAAAA,IAAAA,QAAS,GAAG,EAAE,EAAE;QAE/D,IAAIL,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,EACjC,IAAI,CAAC,sBAAsB,GAAG;QAGhC,MAAM,IAAI,CAAC,cAAc,CAACT;IAC5B;IAEA,MAAa,kBAAkBS,OAAiC,EAAE;QAChE,IAAI,CAAC,cAAc,GAAGA;IACxB;IAEA,MAAM,UAAU;YACVO;QAAJ,IAAIA,AAAAA,SAAAA,CAAAA,uBAAAA,IAAI,CAAC,cAAc,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,QAAQ,AAAD,KAAK,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,GAAG;YACvE,IAAI,CAAC,YAAY,CAAC,+CAA+C;YACjE,KAAK,MAAMhB,SAAS,IAAI,CAAC,kBAAkB,CACzC,MAAMW,OAAO,IAAI,CAAC,MAAM,CAACX;YAE3B,IAAI,CAAC,kBAAkB,GAAG,EAAE;QAC9B;QAEA,MAAM,KAAK,CAAC;QAEZ,IAAI,IAAI,CAAC,YAAY,EAAE;YACrB,IAAI,CAAC,YAAY,CAAC,UAAU;YAC5B,IAAI,CAAC,YAAY,GAAG;YACpB,IAAI,CAAC,YAAY;QACnB;IACF;IAzJA,YACSiB,eAA2B,KAAO,CAAC,EACnCC,eAGK,KAAO,CAAC,EACpBC,yBAAyB,IAAI,CAC7B;QACA,KAAK,CAACA,yBAAAA,iBAAAA,IAAAA,EAAAA,gBAAAA,KAAAA,IAAAA,iBAAAA,IAAAA,EAAAA,gBAAAA,KAAAA,IAdR,uBAAO,gBAAP,SAEA,uBAAQ,kBAAR,SAEA,uBAAQ,sBAAR,cAGSF,YAAY,GAAZA,cAAAA,IAAAA,CACAC,YAAY,GAAZA,cAAAA,IAAAA,CARF,YAAY,GAAwB,WAInC,kBAAkB,GAAa,EAAE;IAWzC;AAiJF"}
+{"version":3,"file":"bridge-mode/page-browser-side.mjs","sources":["webpack://@midscene/web/./src/bridge-mode/page-browser-side.ts"],"sourcesContent":["import type {\n  ChromePageDestroyOptions,\n  KeyboardAction,\n  MouseAction,\n} from '@/page';\nimport { assert } from '@midscene/shared/utils';\nimport ChromeExtensionProxyPage from '../chrome-extension/page';\nimport {\n  type BridgeConnectTabOptions,\n  BridgeEvent,\n  DefaultBridgeServerPort,\n  KeyboardEvent,\n  MouseEvent,\n} from './common';\nimport { BridgeClient } from './io-client';\n\ndeclare const __VERSION__: string;\n\nexport class ExtensionBridgePageBrowserSide extends ChromeExtensionProxyPage {\n  public bridgeClient: BridgeClient | null = null;\n\n  private destroyOptions?: ChromePageDestroyOptions;\n\n  private newlyCreatedTabIds: number[] = [];\n\n  constructor(\n    public onDisconnect: () => void = () => {},\n    public onLogMessage: (\n      message: string,\n      type: 'log' | 'status',\n    ) => void = () => {},\n    forceSameTabNavigation = true,\n  ) {\n    super(forceSameTabNavigation);\n  }\n\n  private async setupBridgeClient() {\n    this.bridgeClient = new BridgeClient(\n      `ws://localhost:${DefaultBridgeServerPort}`,\n      async (method, args: any[]) => {\n        console.log('bridge call from cli side', method, args);\n        if (method === BridgeEvent.ConnectNewTabWithUrl) {\n          return this.connectNewTabWithUrl.apply(\n            this,\n            args as unknown as [string],\n          );\n        }\n\n        if (method === BridgeEvent.GetBrowserTabList) {\n          return this.getBrowserTabList.apply(this, args as any);\n        }\n\n        if (method === BridgeEvent.SetActiveTabId) {\n          return this.setActiveTabId.apply(this, args as any);\n        }\n\n        if (method === BridgeEvent.ConnectCurrentTab) {\n          return this.connectCurrentTab.apply(this, args as any);\n        }\n\n        if (method === BridgeEvent.UpdateAgentStatus) {\n          return this.onLogMessage(args[0] as string, 'status');\n        }\n\n        const tabId = await this.getActiveTabId();\n        if (!tabId || tabId === 0) {\n          throw new Error('no tab is connected');\n        }\n\n        // this.onLogMessage(`calling method: ${method}`);\n\n        if (method.startsWith(MouseEvent.PREFIX)) {\n          const actionName = method.split('.')[1] as keyof MouseAction;\n          if (actionName === 'drag') {\n            return this.mouse[actionName].apply(this.mouse, args as any);\n          }\n          return this.mouse[actionName].apply(this.mouse, args as any);\n        }\n\n        if (method.startsWith(KeyboardEvent.PREFIX)) {\n          const actionName = method.split('.')[1] as keyof KeyboardAction;\n          if (actionName === 'press') {\n            return this.keyboard[actionName].apply(this.keyboard, args as any);\n          }\n          return this.keyboard[actionName].apply(this.keyboard, args as any);\n        }\n\n        try {\n          // @ts-expect-error\n          const result = await this[method as keyof ChromeExtensionProxyPage](\n            ...args,\n          );\n          return result;\n        } catch (e) {\n          const errorMessage = e instanceof Error ? e.message : 'Unknown error';\n          console.error('error calling method', method, args, e);\n          this.onLogMessage(\n            `Error calling method: ${method}, ${errorMessage}`,\n            'log',\n          );\n          throw new Error(errorMessage, { cause: e });\n        }\n      },\n      // on disconnect\n      () => {\n        return this.destroy();\n      },\n    );\n    await this.bridgeClient.connect();\n    this.onLogMessage(\n      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${__VERSION__}`,\n      'log',\n    );\n  }\n\n  public async connect() {\n    return await this.setupBridgeClient();\n  }\n\n  public async connectNewTabWithUrl(\n    url: string,\n    options: BridgeConnectTabOptions = {\n      forceSameTabNavigation: true,\n    },\n  ) {\n    const tab = await chrome.tabs.create({ url });\n    const tabId = tab.id;\n    assert(tabId, 'failed to get tabId after creating a new tab');\n\n    // new tab\n    this.onLogMessage(`Creating new tab: ${url}`, 'log');\n    this.newlyCreatedTabIds.push(tabId);\n\n    if (options?.forceSameTabNavigation) {\n      this.forceSameTabNavigation = true;\n    }\n\n    await this.setActiveTabId(tabId);\n  }\n\n  public async connectCurrentTab(\n    options: BridgeConnectTabOptions = {\n      forceSameTabNavigation: true,\n    },\n  ) {\n    const tabs = await chrome.tabs.query({ active: true, currentWindow: true });\n    const tabId = tabs[0]?.id;\n    assert(tabId, 'failed to get tabId');\n\n    this.onLogMessage(`Connected to current tab: ${tabs[0]?.url}`, 'log');\n\n    if (options?.forceSameTabNavigation) {\n      this.forceSameTabNavigation = true;\n    }\n\n    await this.setActiveTabId(tabId);\n  }\n\n  public async setDestroyOptions(options: ChromePageDestroyOptions) {\n    this.destroyOptions = options;\n  }\n\n  async destroy() {\n    if (this.destroyOptions?.closeTab && this.newlyCreatedTabIds.length > 0) {\n      this.onLogMessage('Closing all newly created tabs by bridge...', 'log');\n      for (const tabId of this.newlyCreatedTabIds) {\n        await chrome.tabs.remove(tabId);\n      }\n      this.newlyCreatedTabIds = [];\n    }\n\n    await super.destroy();\n\n    if (this.bridgeClient) {\n      this.bridgeClient.disconnect();\n      this.bridgeClient = null;\n      this.onDisconnect();\n    }\n  }\n}\n"],"names":["ExtensionBridgePageBrowserSide","ChromeExtensionProxyPage","BridgeClient","DefaultBridgeServerPort","method","args","console","BridgeEvent","tabId","Error","MouseEvent","actionName","KeyboardEvent","result","e","errorMessage","url","options","tab","chrome","assert","_tabs_","_tabs_1","tabs","_this_destroyOptions","onDisconnect","onLogMessage","forceSameTabNavigation"],"mappings":";;;;;;;;;;;;;;AAkBO,MAAMA,uCAAuCC;IAkBlD,MAAc,oBAAoB;QAChC,IAAI,CAAC,YAAY,GAAG,IAAIC,aACtB,CAAC,eAAe,EAAEC,yBAAyB,EAC3C,OAAOC,QAAQC;YACbC,QAAQ,GAAG,CAAC,6BAA6BF,QAAQC;YACjD,IAAID,WAAWG,YAAY,oBAAoB,EAC7C,OAAO,IAAI,CAAC,oBAAoB,CAAC,KAAK,CACpC,IAAI,EACJF;YAIJ,IAAID,WAAWG,YAAY,iBAAiB,EAC1C,OAAO,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAEF;YAG5C,IAAID,WAAWG,YAAY,cAAc,EACvC,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAEF;YAGzC,IAAID,WAAWG,YAAY,iBAAiB,EAC1C,OAAO,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAEF;YAG5C,IAAID,WAAWG,YAAY,iBAAiB,EAC1C,OAAO,IAAI,CAAC,YAAY,CAACF,IAAI,CAAC,EAAE,EAAY;YAG9C,MAAMG,QAAQ,MAAM,IAAI,CAAC,cAAc;YACvC,IAAI,CAACA,SAASA,AAAU,MAAVA,OACZ,MAAM,IAAIC,MAAM;YAKlB,IAAIL,OAAO,UAAU,CAACM,WAAW,MAAM,GAAG;gBACxC,MAAMC,aAAaP,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE;gBAIvC,OAAO,IAAI,CAAC,KAAK,CAACO,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAEN;YAClD;YAEA,IAAID,OAAO,UAAU,CAACQ,cAAc,MAAM,GAAG;gBAC3C,MAAMD,aAAaP,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE;gBAIvC,OAAO,IAAI,CAAC,QAAQ,CAACO,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAEN;YACxD;YAEA,IAAI;gBAEF,MAAMQ,SAAS,MAAM,IAAI,CAACT,OAAyC,IAC9DC;gBAEL,OAAOQ;YACT,EAAE,OAAOC,GAAG;gBACV,MAAMC,eAAeD,aAAaL,QAAQK,EAAE,OAAO,GAAG;gBACtDR,QAAQ,KAAK,CAAC,wBAAwBF,QAAQC,MAAMS;gBACpD,IAAI,CAAC,YAAY,CACf,CAAC,sBAAsB,EAAEV,OAAO,EAAE,EAAEW,cAAc,EAClD;gBAEF,MAAM,IAAIN,MAAMM,cAAc;oBAAE,OAAOD;gBAAE;YAC3C;QACF,GAEA,IACS,IAAI,CAAC,OAAO;QAGvB,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO;QAC/B,IAAI,CAAC,YAAY,CACf,uCAAuC,IAAI,CAAC,YAAY,CAAC,aAAa,sDAAwC,EAC9G;IAEJ;IAEA,MAAa,UAAU;QACrB,OAAO,MAAM,IAAI,CAAC,iBAAiB;IACrC;IAEA,MAAa,qBACXE,GAAW,EACXC,UAAmC;QACjC,wBAAwB;IAC1B,CAAC,EACD;QACA,MAAMC,MAAM,MAAMC,OAAO,IAAI,CAAC,MAAM,CAAC;YAAEH;QAAI;QAC3C,MAAMR,QAAQU,IAAI,EAAE;QACpBE,OAAOZ,OAAO;QAGd,IAAI,CAAC,YAAY,CAAC,CAAC,kBAAkB,EAAEQ,KAAK,EAAE;QAC9C,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAACR;QAE7B,IAAIS,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,EACjC,IAAI,CAAC,sBAAsB,GAAG;QAGhC,MAAM,IAAI,CAAC,cAAc,CAACT;IAC5B;IAEA,MAAa,kBACXS,UAAmC;QACjC,wBAAwB;IAC1B,CAAC,EACD;YAEcI,QAGiCC;QAJ/C,MAAMC,OAAO,MAAMJ,OAAO,IAAI,CAAC,KAAK,CAAC;YAAE,QAAQ;YAAM,eAAe;QAAK;QACzE,MAAMX,QAAQ,QAAAa,CAAAA,SAAAA,IAAI,CAAC,EAAE,AAAD,IAANA,KAAAA,IAAAA,OAAS,EAAE;QACzBD,OAAOZ,OAAO;QAEd,IAAI,CAAC,YAAY,CAAC,CAAC,0BAA0B,EAAE,QAAAc,CAAAA,UAAAA,IAAI,CAAC,EAAE,AAAD,IAANA,KAAAA,IAAAA,QAAS,GAAG,EAAE,EAAE;QAE/D,IAAIL,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,EACjC,IAAI,CAAC,sBAAsB,GAAG;QAGhC,MAAM,IAAI,CAAC,cAAc,CAACT;IAC5B;IAEA,MAAa,kBAAkBS,OAAiC,EAAE;QAChE,IAAI,CAAC,cAAc,GAAGA;IACxB;IAEA,MAAM,UAAU;YACVO;QAAJ,IAAIA,AAAAA,SAAAA,CAAAA,uBAAAA,IAAI,CAAC,cAAc,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,QAAQ,AAAD,KAAK,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,GAAG;YACvE,IAAI,CAAC,YAAY,CAAC,+CAA+C;YACjE,KAAK,MAAMhB,SAAS,IAAI,CAAC,kBAAkB,CACzC,MAAMW,OAAO,IAAI,CAAC,MAAM,CAACX;YAE3B,IAAI,CAAC,kBAAkB,GAAG,EAAE;QAC9B;QAEA,MAAM,KAAK,CAAC;QAEZ,IAAI,IAAI,CAAC,YAAY,EAAE;YACrB,IAAI,CAAC,YAAY,CAAC,UAAU;YAC5B,IAAI,CAAC,YAAY,GAAG;YACpB,IAAI,CAAC,YAAY;QACnB;IACF;IAzJA,YACSiB,eAA2B,KAAO,CAAC,EACnCC,eAGK,KAAO,CAAC,EACpBC,yBAAyB,IAAI,CAC7B;QACA,KAAK,CAACA,yBAAAA,iBAAAA,IAAAA,EAAAA,gBAAAA,KAAAA,IAAAA,iBAAAA,IAAAA,EAAAA,gBAAAA,KAAAA,IAdR,uBAAO,gBAAP,SAEA,uBAAQ,kBAAR,SAEA,uBAAQ,sBAAR,cAGSF,YAAY,GAAZA,cAAAA,IAAAA,CACAC,YAAY,GAAZA,cAAAA,IAAAA,CARF,YAAY,GAAwB,WAInC,kBAAkB,GAAa,EAAE;IAWzC;AAiJF"}

package/dist/es/chrome-extension/page.mjs

@@ -341,7 +341,7 @@ class ChromeExtensionProxyPage {
     constructor(forceSameTabNavigation){
         _define_property(this, "pageType", 'chrome-extension-proxy');
         _define_property(this, "forceSameTabNavigation", void 0);
-        _define_property(this, "version", "0.26.6");
+        _define_property(this, "version", "0.26.7-beta-20250815153024.0");
         _define_property(this, "viewportSize", void 0);
         _define_property(this, "activeTabId", null);
         _define_property(this, "tabIdOfDebuggerAttached", null);

package/dist/es/common/agent.mjs

@@ -7,9 +7,8 @@ import { getAIConfigInBoolean, vlLocateMode } from "@midscene/shared/env";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { PageTaskExecutor } from "./tasks.mjs";
-import { buildPlans } from "./plan-builder.mjs";
 import { TaskCache } from "./task-cache.mjs";
-import { locateParamStr, paramStr, scrollParamStr, taskTitleStr, typeStr } from "./ui-utils.mjs";
+import { locateParamStr, paramStr, taskTitleStr, typeStr } from "./ui-utils.mjs";
 import { getReportFileName, parseContextFromWebPage, printReportMsg, trimContextByViewport } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -21,7 +20,7 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const debug = getDebug('web-integration');
+const debug = getDebug('agent');
 const distanceOfTwoPoints = (p1, p2)=>{
     const [x1, y1] = p1;
     const [x2, y2] = p2;
@@ -37,6 +36,9 @@ const defaultInsightExtractOption = {
     screenshotIncluded: true
 };
 class PageAgent {
+    async getActionSpace() {
+        return this.page.actionSpace();
+    }
     async getUIContext(action) {
         if (this.frozenPageContext) {
             debug('Using frozen page context for action:', action);
@@ -128,68 +130,99 @@ class PageAgent {
             prompt: locatePrompt
         };
     }
-    async aiTap(locatePrompt, opt) {
-        const detailedLocateParam = this.buildDetailedLocateParam(locatePrompt, opt);
-        const plans = buildPlans('Tap', detailedLocateParam);
-        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('Tap', locateParamStr(detailedLocateParam)), plans, {
+    locateTaskForLocate(prompt, opt) {
+        const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
+        const locatePlan = {
+            type: 'Locate',
+            locate: detailedLocateParam,
+            param: detailedLocateParam,
+            thought: ''
+        };
+        return locatePlan;
+    }
+    async callActionInActionSpace(type, locatePrompt, opt) {
+        debug('callActionInActionSpace', type, ',', locatePrompt, ',', opt);
+        const locatePlan = locatePrompt ? this.locateTaskForLocate(locatePrompt, opt) : null;
+        const actionPlan = {
+            type: type,
+            param: opt,
+            thought: '',
+            locate: null == locatePlan ? void 0 : locatePlan.locate
+        };
+        const plans = [
+            locatePlan,
+            actionPlan
+        ].filter(Boolean);
+        const title = taskTitleStr(type, locateParamStr((null == locatePlan ? void 0 : locatePlan.locate) || void 0));
+        const { executor } = await this.taskExecutor.runPlans(title, plans, {
             cacheable: null == opt ? void 0 : opt.cacheable
         });
         await this.afterTaskRunning(executor);
-        return output;
+    }
+    async aiTap(locatePrompt, opt) {
+        return this.callActionInActionSpace('Tap', locatePrompt, opt);
     }
     async aiRightClick(locatePrompt, opt) {
-        const detailedLocateParam = this.buildDetailedLocateParam(locatePrompt, opt);
-        const plans = buildPlans('RightClick', detailedLocateParam);
-        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('RightClick', locateParamStr(detailedLocateParam)), plans, {
-            cacheable: null == opt ? void 0 : opt.cacheable
-        });
-        await this.afterTaskRunning(executor);
-        return output;
+        return this.callActionInActionSpace('RightClick', locatePrompt, opt);
     }
     async aiHover(locatePrompt, opt) {
-        const detailedLocateParam = this.buildDetailedLocateParam(locatePrompt, opt);
-        const plans = buildPlans('Hover', detailedLocateParam);
-        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('Hover', locateParamStr(detailedLocateParam)), plans, {
-            cacheable: null == opt ? void 0 : opt.cacheable
-        });
-        await this.afterTaskRunning(executor);
-        return output;
-    }
-    async aiInput(value, locatePrompt, opt) {
+        return this.callActionInActionSpace('Hover', locatePrompt, opt);
+    }
+    async aiInput(locatePromptOrValue, locatePromptOrOpt, optOrUndefined) {
+        let value;
+        let locatePrompt;
+        let opt;
+        if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'value' in locatePromptOrOpt) {
+            locatePrompt = locatePromptOrValue;
+            const optWithValue = locatePromptOrOpt;
+            value = optWithValue.value;
+            opt = optWithValue;
+        } else {
+            value = locatePromptOrValue;
+            locatePrompt = locatePromptOrOpt;
+            opt = {
+                ...optOrUndefined,
+                value
+            };
+        }
         assert('string' == typeof value, 'input value must be a string, use empty string if you want to clear the input');
         assert(locatePrompt, 'missing locate prompt for input');
-        const detailedLocateParam = this.buildDetailedLocateParam(locatePrompt, opt);
-        const plans = buildPlans('Input', detailedLocateParam, {
-            value,
-            autoDismissKeyboard: null == opt ? void 0 : opt.autoDismissKeyboard
-        });
-        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('Input', locateParamStr(detailedLocateParam)), plans, {
-            cacheable: null == opt ? void 0 : opt.cacheable
-        });
-        await this.afterTaskRunning(executor);
-        return output;
-    }
-    async aiKeyboardPress(keyName, locatePrompt, opt) {
-        assert(keyName, 'missing keyName for keyboard press');
-        const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
-        const plans = buildPlans('KeyboardPress', detailedLocateParam, {
-            value: keyName
-        });
-        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('KeyboardPress', locateParamStr(detailedLocateParam)), plans, {
-            cacheable: null == opt ? void 0 : opt.cacheable
-        });
-        await this.afterTaskRunning(executor);
-        return output;
-    }
-    async aiScroll(scrollParam, locatePrompt, opt) {
-        const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
-        const plans = buildPlans('Scroll', detailedLocateParam, scrollParam);
-        const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
-        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('Scroll', paramInTitle), plans, {
-            cacheable: null == opt ? void 0 : opt.cacheable
-        });
-        await this.afterTaskRunning(executor);
-        return output;
+        return this.callActionInActionSpace('Input', locatePrompt, opt);
+    }
+    async aiKeyboardPress(locatePromptOrKeyName, locatePromptOrOpt, optOrUndefined) {
+        let keyName;
+        let locatePrompt;
+        let opt;
+        if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'keyName' in locatePromptOrOpt) {
+            locatePrompt = locatePromptOrKeyName;
+            opt = locatePromptOrOpt;
+        } else {
+            keyName = locatePromptOrKeyName;
+            locatePrompt = locatePromptOrOpt;
+            opt = {
+                ...optOrUndefined || {},
+                keyName
+            };
+        }
+        assert(null == opt ? void 0 : opt.keyName, 'missing keyName for keyboard press');
+        return this.callActionInActionSpace('KeyboardPress', locatePrompt, opt);
+    }
+    async aiScroll(locatePromptOrScrollParam, locatePromptOrOpt, optOrUndefined) {
+        let scrollParam;
+        let locatePrompt;
+        let opt;
+        if ('object' == typeof locatePromptOrOpt && ('direction' in locatePromptOrOpt || 'scrollType' in locatePromptOrOpt || 'distance' in locatePromptOrOpt)) {
+            locatePrompt = locatePromptOrScrollParam;
+            opt = locatePromptOrOpt;
+        } else {
+            scrollParam = locatePromptOrScrollParam;
+            locatePrompt = locatePromptOrOpt;
+            opt = {
+                ...optOrUndefined || {},
+                ...scrollParam || {}
+            };
+        }
+        return this.callActionInActionSpace('Scroll', locatePrompt, opt);
     }
     async aiAction(taskPrompt, opt) {
         var _this_taskCache, _this_taskCache1;
@@ -295,9 +328,11 @@ class PageAgent {
         return verifyResult;
     }
     async aiLocate(prompt, opt) {
-        const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
-        const plans = buildPlans('Locate', detailedLocateParam);
-        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('Locate', locateParamStr(detailedLocateParam)), plans, {
+        const locatePlan = this.locateTaskForLocate(prompt, opt);
+        const plans = [
+            locatePlan
+        ];
+        const { executor, output } = await this.taskExecutor.runPlans(taskTitleStr('Locate', locateParamStr((null == locatePlan ? void 0 : locatePlan.locate) || void 0)), plans, {
             cacheable: null == opt ? void 0 : opt.cacheable
         });
         await this.afterTaskRunning(executor);
@@ -346,6 +381,7 @@ class PageAgent {
     }
     async runYaml(yamlScriptContent) {
         const script = parseYamlScript(yamlScriptContent, 'yaml', true);
+        await this.page.actionSpace();
         const player = new ScriptPlayer(script, async (target)=>({
                 agent: this,
                 freeFn: []

package/dist/es/common/agent.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"common/agent.mjs","sources":["webpack://@midscene/web/./src/common/agent.ts"],"sourcesContent":["import type { WebPage } from '@/common/page';\nimport {\n  type AgentAssertOpt,\n  type AgentDescribeElementAtPointResult,\n  type AgentWaitForOpt,\n  type DetailedLocateParam,\n  type ExecutionDump,\n  type ExecutionRecorderItem,\n  type ExecutionTask,\n  type ExecutionTaskLog,\n  type Executor,\n  type GroupedActionDump,\n  Insight,\n  type InsightAction,\n  type InsightExtractOption,\n  type InsightExtractParam,\n  type LocateOption,\n  type LocateResultElement,\n  type LocateValidatorResult,\n  type LocatorValidatorOption,\n  type MidsceneYamlScript,\n  type OnTaskStartTip,\n  type Rect,\n  type ScrollParam,\n  type TUserPrompt,\n} from '@midscene/core';\n\nimport yaml from 'js-yaml';\n\nimport { ScriptPlayer, parseYamlScript } from '@/yaml/index';\nimport {\n  groupedActionDumpFileExt,\n  reportHTMLContent,\n  stringifyDumpData,\n  writeLogFile,\n} from '@midscene/core/utils';\nimport {\n  DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,\n  DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT,\n} from '@midscene/shared/constants';\nimport { getAIConfigInBoolean, vlLocateMode } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { PageTaskExecutor } from '../common/tasks';\nimport type { PlaywrightWebPage } from '../playwright';\nimport type { PuppeteerWebPage } from '../puppeteer';\nimport type { WebElementInfo, WebUIContext } from '../web-element';\nimport type { AndroidDeviceInputOpt } from './page';\nimport { buildPlans } from './plan-builder';\nimport { TaskCache } from './task-cache';\nimport {\n  locateParamStr,\n  paramStr,\n  scrollParamStr,\n  taskTitleStr,\n  typeStr,\n} from './ui-utils';\nimport { getReportFileName, printReportMsg } from './utils';\nimport { parseContextFromWebPage } from './utils';\nimport { trimContextByViewport } from './utils';\n\nconst debug = getDebug('web-integration');\n\nconst distanceOfTwoPoints = (p1: [number, number], p2: [number, number]) => {\n  const [x1, y1] = p1;\n  const [x2, y2] = p2;\n  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));\n};\n\nconst includedInRect = (point: [number, number], rect: Rect) => {\n  const [x, y] = point;\n  const { left, top, width, height } = rect;\n  return x >= left && x <= left + width && y >= top && y <= top + height;\n};\n\nconst defaultInsightExtractOption: InsightExtractOption = {\n  domIncluded: false,\n  screenshotIncluded: true,\n};\n\nexport interface PageAgentOpt {\n  forceSameTabNavigation?: boolean /* if limit the new tab to the current page, default true */;\n  testId?: string;\n  cacheId?: string;\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n}\n\nexport type WebPageAgentOpt = PageAgentOpt & WebPageOpt;\nexport type WebPageOpt = {\n  waitForNavigationTimeout?: number;\n  waitForNetworkIdleTimeout?: number;\n};\n\nexport class PageAgent<PageType extends WebPage = WebPage> {\n  page: PageType;\n\n  insight: Insight<WebElementInfo, WebUIContext>;\n\n  dump: GroupedActionDump;\n\n  reportFile?: string | null;\n\n  reportFileName?: string;\n\n  taskExecutor: PageTaskExecutor;\n\n  opts: PageAgentOpt;\n\n  /**\n   * If true, the agent will not perform any actions\n   */\n  dryMode = false;\n\n  onTaskStartTip?: OnTaskStartTip;\n\n  taskCache?: TaskCache;\n\n  onDumpUpdate?: (dump: string) => void;\n\n  destroyed = false;\n\n  /**\n   * Frozen page context for consistent AI operations\n   */\n  private frozenPageContext?: WebUIContext;\n\n  constructor(page: PageType, opts?: PageAgentOpt) {\n    this.page = page;\n    this.opts = Object.assign(\n      {\n        generateReport: true,\n        autoPrintReportMsg: true,\n        groupName: 'Midscene Report',\n        groupDescription: '',\n      },\n      opts || {},\n    );\n\n    if (\n      this.page.pageType === 'puppeteer' ||\n      this.page.pageType === 'playwright'\n    ) {\n      (\n        this.page as PuppeteerWebPage | PlaywrightWebPage\n      ).waitForNavigationTimeout =\n        (this.opts as WebPageAgentOpt).waitForNavigationTimeout ??\n        DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;\n      (\n        this.page as PuppeteerWebPage | PlaywrightWebPage\n      ).waitForNetworkIdleTimeout =\n        (this.opts as WebPageAgentOpt).waitForNetworkIdleTimeout ??\n        DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;\n    }\n\n    this.onTaskStartTip = this.opts.onTaskStartTip;\n    // get the parent browser of the puppeteer page\n    // const browser = (this.page as PuppeteerWebPage).browser();\n\n    this.insight = new Insight<WebElementInfo, WebUIContext>(\n      async (action: InsightAction) => {\n        return this.getUIContext(action);\n      },\n    );\n\n    if (opts?.cacheId && this.page.pageType !== 'android') {\n      this.taskCache = new TaskCache(\n        opts.cacheId,\n        getAIConfigInBoolean('MIDSCENE_CACHE'), // if we should use cache to match the element\n      );\n    }\n\n    this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {\n      taskCache: this.taskCache,\n      onTaskStart: this.callbackOnTaskStartTip.bind(this),\n    });\n    this.dump = this.resetDump();\n    this.reportFileName =\n      opts?.reportFileName ||\n      getReportFileName(opts?.testId || this.page.pageType || 'web');\n  }\n\n  async getUIContext(action?: InsightAction): Promise<WebUIContext> {\n    // If page context is frozen, return the frozen context for all actions\n    if (this.frozenPageContext) {\n      debug('Using frozen page context for action:', action);\n      return this.frozenPageContext;\n    }\n\n    // Otherwise, get fresh context based on the action type\n    if (action && (action === 'extract' || action === 'assert')) {\n      return await parseContextFromWebPage(this.page, {\n        ignoreMarker: true,\n      });\n    }\n    return await parseContextFromWebPage(this.page, {\n      ignoreMarker: !!vlLocateMode(),\n    });\n  }\n\n  async _snapshotContext(): Promise<WebUIContext> {\n    return await this.getUIContext('locate');\n  }\n\n  async setAIActionContext(prompt: string) {\n    this.opts.aiActionContext = prompt;\n  }\n\n  resetDump() {\n    this.dump = {\n      groupName: this.opts.groupName!,\n      groupDescription: this.opts.groupDescription,\n      executions: [],\n    };\n\n    return this.dump;\n  }\n\n  appendExecutionDump(execution: ExecutionDump) {\n    // use trimContextByViewport to process execution\n    const trimmedExecution = trimContextByViewport(execution);\n    const currentDump = this.dump;\n    currentDump.executions.push(trimmedExecution);\n  }\n\n  dumpDataString() {\n    // update dump info\n    this.dump.groupName = this.opts.groupName!;\n    this.dump.groupDescription = this.opts.groupDescription;\n    return stringifyDumpData(this.dump);\n  }\n\n  reportHTMLString() {\n    return reportHTMLContent(this.dumpDataString());\n  }\n\n  writeOutActionDumps() {\n    if (this.destroyed) {\n      throw new Error(\n        'PageAgent has been destroyed. Cannot update report file.',\n      );\n    }\n    const { generateReport, autoPrintReportMsg } = this.opts;\n    this.reportFile = writeLogFile({\n      fileName: this.reportFileName!,\n      fileExt: groupedActionDumpFileExt,\n      fileContent: this.dumpDataString(),\n      type: 'dump',\n      generateReport,\n    });\n    debug('writeOutActionDumps', this.reportFile);\n    if (generateReport && autoPrintReportMsg && this.reportFile) {\n      printReportMsg(this.reportFile);\n    }\n  }\n\n  private async callbackOnTaskStartTip(task: ExecutionTask) {\n    const param = paramStr(task);\n    const tip = param ? `${typeStr(task)} - ${param}` : typeStr(task);\n\n    if (this.onTaskStartTip) {\n      await this.onTaskStartTip(tip);\n    }\n  }\n\n  private async afterTaskRunning(executor: Executor, doNotThrowError = false) {\n    this.appendExecutionDump(executor.dump());\n\n    try {\n      await this.onDumpUpdate?.(this.dumpDataString());\n    } catch (error) {\n      console.error('Error in onDumpUpdate', error);\n    }\n\n    this.writeOutActionDumps();\n\n    if (executor.isInErrorState() && !doNotThrowError) {\n      const errorTask = executor.latestErrorTask();\n      throw new Error(`${errorTask?.errorMessage}\\n${errorTask?.errorStack}`, {\n        cause: errorTask?.error,\n      });\n    }\n  }\n\n  private buildDetailedLocateParam(\n    locatePrompt: TUserPrompt,\n    opt?: LocateOption,\n  ): DetailedLocateParam {\n    assert(locatePrompt, 'missing locate prompt');\n\n    if (typeof opt === 'object' && opt !== null) {\n      const prompt = locatePrompt;\n      const deepThink = opt.deepThink ?? false;\n      const cacheable = opt.cacheable ?? true;\n      const xpath = opt.xpath;\n\n      return {\n        prompt,\n        deepThink,\n        cacheable,\n        xpath,\n      };\n    }\n\n    return {\n      prompt: locatePrompt,\n    };\n  }\n\n  async aiTap(locatePrompt: TUserPrompt, opt?: LocateOption) {\n    const detailedLocateParam = this.buildDetailedLocateParam(\n      locatePrompt,\n      opt,\n    );\n    const plans = buildPlans('Tap', detailedLocateParam);\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('Tap', locateParamStr(detailedLocateParam)),\n      plans,\n      { cacheable: opt?.cacheable },\n    );\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption) {\n    const detailedLocateParam = this.buildDetailedLocateParam(\n      locatePrompt,\n      opt,\n    );\n    const plans = buildPlans('RightClick', detailedLocateParam);\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('RightClick', locateParamStr(detailedLocateParam)),\n      plans,\n      { cacheable: opt?.cacheable },\n    );\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiHover(locatePrompt: TUserPrompt, opt?: LocateOption) {\n    const detailedLocateParam = this.buildDetailedLocateParam(\n      locatePrompt,\n      opt,\n    );\n    const plans = buildPlans('Hover', detailedLocateParam);\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('Hover', locateParamStr(detailedLocateParam)),\n      plans,\n      { cacheable: opt?.cacheable },\n    );\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiInput(\n    value: string,\n    locatePrompt: TUserPrompt,\n    opt?: AndroidDeviceInputOpt & LocateOption,\n  ) {\n    assert(\n      typeof value === 'string',\n      'input value must be a string, use empty string if you want to clear the input',\n    );\n    assert(locatePrompt, 'missing locate prompt for input');\n    const detailedLocateParam = this.buildDetailedLocateParam(\n      locatePrompt,\n      opt,\n    );\n    const plans = buildPlans('Input', detailedLocateParam, {\n      value,\n      autoDismissKeyboard: opt?.autoDismissKeyboard,\n    });\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('Input', locateParamStr(detailedLocateParam)),\n      plans,\n      {\n        cacheable: opt?.cacheable,\n      },\n    );\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiKeyboardPress(\n    keyName: string,\n    locatePrompt?: TUserPrompt,\n    opt?: LocateOption,\n  ) {\n    assert(keyName, 'missing keyName for keyboard press');\n    const detailedLocateParam = locatePrompt\n      ? this.buildDetailedLocateParam(locatePrompt, opt)\n      : undefined;\n    const plans = buildPlans('KeyboardPress', detailedLocateParam, {\n      value: keyName,\n    });\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('KeyboardPress', locateParamStr(detailedLocateParam)),\n      plans,\n      { cacheable: opt?.cacheable },\n    );\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiScroll(\n    scrollParam: ScrollParam,\n    locatePrompt?: TUserPrompt,\n    opt?: LocateOption,\n  ) {\n    const detailedLocateParam = locatePrompt\n      ? this.buildDetailedLocateParam(locatePrompt, opt)\n      : undefined;\n    const plans = buildPlans('Scroll', detailedLocateParam, scrollParam);\n    const paramInTitle = locatePrompt\n      ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}`\n      : scrollParamStr(scrollParam);\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('Scroll', paramInTitle),\n      plans,\n      { cacheable: opt?.cacheable },\n    );\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiAction(\n    taskPrompt: string,\n    opt?: {\n      cacheable?: boolean;\n    },\n  ) {\n    const cacheable = opt?.cacheable;\n    // if vlm-ui-tars, plan cache is not used\n    const isVlmUiTars = vlLocateMode() === 'vlm-ui-tars';\n    const matchedCache =\n      isVlmUiTars || cacheable === false\n        ? undefined\n        : this.taskCache?.matchPlanCache(taskPrompt);\n    if (matchedCache && this.taskCache?.isCacheResultUsed) {\n      // log into report file\n      const { executor } = await this.taskExecutor.loadYamlFlowAsPlanning(\n        taskPrompt,\n        matchedCache.cacheContent?.yamlWorkflow,\n      );\n\n      await await this.afterTaskRunning(executor);\n\n      debug('matched cache, will call .runYaml to run the action');\n      const yaml = matchedCache.cacheContent?.yamlWorkflow;\n      return this.runYaml(yaml);\n    }\n\n    const { output, executor } = await (isVlmUiTars\n      ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable })\n      : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {\n          cacheable,\n        }));\n\n    // update cache\n    if (this.taskCache && output?.yamlFlow && cacheable !== false) {\n      const yamlContent: MidsceneYamlScript = {\n        tasks: [\n          {\n            name: taskPrompt,\n            flow: output.yamlFlow,\n          },\n        ],\n      };\n      const yamlFlowStr = yaml.dump(yamlContent);\n      this.taskCache.updateOrAppendCacheRecord(\n        {\n          type: 'plan',\n          prompt: taskPrompt,\n          yamlWorkflow: yamlFlowStr,\n        },\n        matchedCache,\n      );\n    }\n\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiQuery(\n    demand: InsightExtractParam,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.query(demand, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiBoolean(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.boolean(prompt, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiNumber(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.number(prompt, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiString(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.string(prompt, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiAsk(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    return this.aiString(prompt, opt);\n  }\n\n  async describeElementAtPoint(\n    center: [number, number],\n    opt?: {\n      verifyPrompt?: boolean;\n      retryLimit?: number;\n      deepThink?: boolean;\n    } & LocatorValidatorOption,\n  ): Promise<AgentDescribeElementAtPointResult> {\n    const { verifyPrompt = true, retryLimit = 3 } = opt || {};\n\n    let success = false;\n    let retryCount = 0;\n    let resultPrompt = '';\n    let deepThink = opt?.deepThink || false;\n    let verifyResult: LocateValidatorResult | undefined;\n\n    while (!success && retryCount < retryLimit) {\n      if (retryCount >= 2) {\n        deepThink = true;\n      }\n      debug(\n        'aiDescribe',\n        center,\n        'verifyPrompt',\n        verifyPrompt,\n        'retryCount',\n        retryCount,\n        'deepThink',\n        deepThink,\n      );\n      const text = await this.insight.describe(center, { deepThink });\n      debug('aiDescribe text', text);\n      assert(text.description, `failed to describe element at [${center}]`);\n      resultPrompt = text.description;\n\n      verifyResult = await this.verifyLocator(\n        resultPrompt,\n        deepThink ? { deepThink: true } : undefined,\n        center,\n        opt,\n      );\n      if (verifyResult.pass) {\n        success = true;\n      } else {\n        retryCount++;\n      }\n    }\n\n    return {\n      prompt: resultPrompt,\n      deepThink,\n      verifyResult,\n    };\n  }\n\n  async verifyLocator(\n    prompt: string,\n    locateOpt: LocateOption | undefined,\n    expectCenter: [number, number],\n    verifyLocateOption?: LocatorValidatorOption,\n  ): Promise<LocateValidatorResult> {\n    debug('verifyLocator', prompt, locateOpt, expectCenter, verifyLocateOption);\n\n    const { center: verifyCenter, rect: verifyRect } = await this.aiLocate(\n      prompt,\n      locateOpt,\n    );\n    const distance = distanceOfTwoPoints(expectCenter, verifyCenter);\n    const included = includedInRect(expectCenter, verifyRect);\n    const pass =\n      distance <= (verifyLocateOption?.centerDistanceThreshold || 20) ||\n      included;\n    const verifyResult = {\n      pass,\n      rect: verifyRect,\n      center: verifyCenter,\n      centerDistance: distance,\n    };\n    debug('aiDescribe verifyResult', verifyResult);\n    return verifyResult;\n  }\n\n  async aiLocate(prompt: TUserPrompt, opt?: LocateOption) {\n    const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);\n    const plans = buildPlans('Locate', detailedLocateParam);\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('Locate', locateParamStr(detailedLocateParam)),\n      plans,\n      { cacheable: opt?.cacheable },\n    );\n    await this.afterTaskRunning(executor);\n\n    const { element } = output;\n\n    return {\n      rect: element?.rect,\n      center: element?.center,\n      scale: (await this.page.size()).dpr,\n    } as Pick<LocateResultElement, 'rect' | 'center'> & {\n      scale: number;\n    };\n  }\n\n  async aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt) {\n    const { output, executor, thought } = await this.taskExecutor.assert(\n      assertion,\n      {\n        returnThought: true,\n      },\n    );\n    await this.afterTaskRunning(executor, true);\n\n    if (opt?.keepRawResponse) {\n      return {\n        pass: output,\n        thought,\n      };\n    }\n\n    if (!output) {\n      const errMsg = msg || `Assertion failed: ${assertion}`;\n      const reasonMsg = `Reason: ${\n        thought || executor.latestErrorTask()?.error || '(no_reason)'\n      }`;\n      throw new Error(`${errMsg}\\n${reasonMsg}`);\n    }\n  }\n\n  async aiWaitFor(assertion: string, opt?: AgentWaitForOpt) {\n    const { executor } = await this.taskExecutor.waitFor(assertion, {\n      timeoutMs: opt?.timeoutMs || 15 * 1000,\n      checkIntervalMs: opt?.checkIntervalMs || 3 * 1000,\n      assertion,\n    });\n    await this.afterTaskRunning(executor, true);\n\n    if (executor.isInErrorState()) {\n      const errorTask = executor.latestErrorTask();\n      throw new Error(`${errorTask?.error}\\n${errorTask?.errorStack}`);\n    }\n  }\n\n  async ai(taskPrompt: string, type = 'action') {\n    if (type === 'action') {\n      return this.aiAction(taskPrompt);\n    }\n    if (type === 'query') {\n      return this.aiQuery(taskPrompt);\n    }\n\n    if (type === 'assert') {\n      return this.aiAssert(taskPrompt);\n    }\n\n    if (type === 'tap') {\n      return this.aiTap(taskPrompt);\n    }\n\n    if (type === 'rightClick') {\n      return this.aiRightClick(taskPrompt);\n    }\n\n    throw new Error(\n      `Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'rightClick'`,\n    );\n  }\n\n  async runYaml(yamlScriptContent: string): Promise<{\n    result: Record<string, any>;\n  }> {\n    const script = parseYamlScript(yamlScriptContent, 'yaml', true);\n    const player = new ScriptPlayer(script, async (target) => {\n      return { agent: this, freeFn: [] };\n    });\n    await player.run();\n\n    if (player.status === 'error') {\n      const errors = player.taskStatusList\n        .filter((task) => task.status === 'error')\n        .map((task) => {\n          return `task - ${task.name}: ${task.error?.message}`;\n        })\n        .join('\\n');\n      throw new Error(`Error(s) occurred in running yaml script:\\n${errors}`);\n    }\n\n    return {\n      result: player.result,\n    };\n  }\n\n  async evaluateJavaScript(script: string) {\n    assert(\n      this.page.evaluateJavaScript,\n      'evaluateJavaScript is not supported in current agent',\n    );\n    return this.page.evaluateJavaScript(script);\n  }\n\n  async destroy() {\n    await this.page.destroy();\n    this.resetDump(); // reset dump to release memory\n    this.destroyed = true;\n  }\n\n  async logScreenshot(\n    title?: string,\n    opt?: {\n      content: string;\n    },\n  ) {\n    // 1. screenshot\n    const base64 = await this.page.screenshotBase64();\n    const now = Date.now();\n    // 2. build recorder\n    const recorder: ExecutionRecorderItem[] = [\n      {\n        type: 'screenshot',\n        ts: now,\n        screenshot: base64,\n      },\n    ];\n    // 3. build ExecutionTaskLog\n    const task: ExecutionTaskLog = {\n      type: 'Log',\n      subType: 'Screenshot',\n      status: 'finished',\n      recorder,\n      timing: {\n        start: now,\n        end: now,\n        cost: 0,\n      },\n      param: {\n        content: opt?.content || '',\n      },\n      executor: async () => {},\n    };\n    // 4. build ExecutionDump\n    const executionDump: ExecutionDump = {\n      sdkVersion: '',\n      logTime: now,\n      model_name: '',\n      model_description: '',\n      name: `Log - ${title || 'untitled'}`,\n      description: opt?.content || '',\n      tasks: [task],\n    };\n    // 5. append to execution dump\n    this.appendExecutionDump(executionDump);\n\n    try {\n      this.onDumpUpdate?.(this.dumpDataString());\n    } catch (error) {\n      console.error('Failed to update dump', error);\n    }\n\n    this.writeOutActionDumps();\n  }\n\n  _unstableLogContent() {\n    const { groupName, groupDescription, executions } = this.dump;\n    const newExecutions = Array.isArray(executions)\n      ? executions.map((execution: any) => {\n          const { tasks, ...restExecution } = execution;\n          let newTasks = tasks;\n          if (Array.isArray(tasks)) {\n            newTasks = tasks.map((task: any) => {\n              // only remove pageContext and log from task\n              const { pageContext, log, ...restTask } = task;\n              return restTask;\n            });\n          }\n          return { ...restExecution, ...(newTasks ? { tasks: newTasks } : {}) };\n        })\n      : [];\n    return {\n      groupName,\n      groupDescription,\n      executions: newExecutions,\n    };\n  }\n\n  /**\n   * Freezes the current page context to be reused in subsequent AI operations\n   * This avoids recalculating page context for each operation\n   */\n  async freezePageContext(): Promise<void> {\n    debug('Freezing page context');\n    const context = await this._snapshotContext();\n    // Mark the context as frozen\n    context._isFrozen = true;\n    this.frozenPageContext = context;\n    debug('Page context frozen successfully');\n  }\n\n  /**\n   * Unfreezes the page context, allowing AI operations to calculate context dynamically\n   */\n  async unfreezePageContext(): Promise<void> {\n    debug('Unfreezing page context');\n    this.frozenPageContext = undefined;\n    debug('Page context unfrozen successfully');\n  }\n}\n"],"names":["debug","getDebug","distanceOfTwoPoints","p1","p2","x1","y1","x2","y2","Math","includedInRect","point","rect","x","y","left","top","width","height","defaultInsightExtractOption","PageAgent","action","parseContextFromWebPage","vlLocateMode","prompt","execution","trimmedExecution","trimContextByViewport","currentDump","stringifyDumpData","reportHTMLContent","Error","generateReport","autoPrintReportMsg","writeLogFile","groupedActionDumpFileExt","printReportMsg","task","param","paramStr","tip","typeStr","executor","doNotThrowError","_this","error","console","errorTask","locatePrompt","opt","assert","deepThink","cacheable","xpath","detailedLocateParam","plans","buildPlans","output","taskTitleStr","locateParamStr","value","keyName","undefined","scrollParam","paramInTitle","scrollParamStr","taskPrompt","_this_taskCache","_this_taskCache1","isVlmUiTars","matchedCache","_matchedCache_cacheContent","_matchedCache_cacheContent1","yaml","yamlContent","yamlFlowStr","demand","center","verifyPrompt","retryLimit","success","retryCount","resultPrompt","verifyResult","text","locateOpt","expectCenter","verifyLocateOption","verifyCenter","verifyRect","distance","included","pass","element","assertion","msg","thought","_executor_latestErrorTask","errMsg","reasonMsg","type","yamlScriptContent","script","parseYamlScript","player","ScriptPlayer","target","errors","_task_error","title","base64","now","Date","recorder","executionDump","groupName","groupDescription","executions","newExecutions","Array","tasks","restExecution","newTasks","pageContext","log","restTask","context","page","opts","Object","DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT","DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT","Insight","TaskCache","getAIConfigInBoolean","PageTaskExecutor","getReportFileName"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AA6DA,MAAMA,QAAQC,SAAS;AAEvB,MAAMC,sBAAsB,CAACC,IAAsBC;IACjD,MAAM,CAACC,IAAIC,GAAG,GAAGH;IACjB,MAAM,CAACI,IAAIC,GAAG,GAAGJ;IACjB,OAAOK,KAAK,KAAK,CAACA,KAAK,IAAI,CAAEJ,AAAAA,CAAAA,KAAKE,EAAC,KAAM,IAAKD,AAAAA,CAAAA,KAAKE,EAAC,KAAM;AAC5D;AAEA,MAAME,iBAAiB,CAACC,OAAyBC;IAC/C,MAAM,CAACC,GAAGC,EAAE,GAAGH;IACf,MAAM,EAAEI,IAAI,EAAEC,GAAG,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGN;IACrC,OAAOC,KAAKE,QAAQF,KAAKE,OAAOE,SAASH,KAAKE,OAAOF,KAAKE,MAAME;AAClE;AAEA,MAAMC,8BAAoD;IACxD,aAAa;IACb,oBAAoB;AACtB;AAwBO,MAAMC;IAwFX,MAAM,aAAaC,MAAsB,EAAyB;QAEhE,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1BrB,MAAM,yCAAyCqB;YAC/C,OAAO,IAAI,CAAC,iBAAiB;QAC/B;QAGA,IAAIA,UAAWA,CAAAA,AAAW,cAAXA,UAAwBA,AAAW,aAAXA,MAAkB,GACvD,OAAO,MAAMC,wBAAwB,IAAI,CAAC,IAAI,EAAE;YAC9C,cAAc;QAChB;QAEF,OAAO,MAAMA,wBAAwB,IAAI,CAAC,IAAI,EAAE;YAC9C,cAAc,CAAC,CAACC;QAClB;IACF;IAEA,MAAM,mBAA0C;QAC9C,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC;IACjC;IAEA,MAAM,mBAAmBC,MAAc,EAAE;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,GAAGA;IAC9B;IAEA,YAAY;QACV,IAAI,CAAC,IAAI,GAAG;YACV,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,kBAAkB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAC5C,YAAY,EAAE;QAChB;QAEA,OAAO,IAAI,CAAC,IAAI;IAClB;IAEA,oBAAoBC,SAAwB,EAAE;QAE5C,MAAMC,mBAAmBC,sBAAsBF;QAC/C,MAAMG,cAAc,IAAI,CAAC,IAAI;QAC7BA,YAAY,UAAU,CAAC,IAAI,CAACF;IAC9B;IAEA,iBAAiB;QAEf,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;QACvD,OAAOG,kBAAkB,IAAI,CAAC,IAAI;IACpC;IAEA,mBAAmB;QACjB,OAAOC,kBAAkB,IAAI,CAAC,cAAc;IAC9C;IAEA,sBAAsB;QACpB,IAAI,IAAI,CAAC,SAAS,EAChB,MAAM,IAAIC,MACR;QAGJ,MAAM,EAAEC,cAAc,EAAEC,kBAAkB,EAAE,GAAG,IAAI,CAAC,IAAI;QACxD,IAAI,CAAC,UAAU,GAAGC,aAAa;YAC7B,UAAU,IAAI,CAAC,cAAc;YAC7B,SAASC;YACT,aAAa,IAAI,CAAC,cAAc;YAChC,MAAM;YACNH;QACF;QACAhC,MAAM,uBAAuB,IAAI,CAAC,UAAU;QAC5C,IAAIgC,kBAAkBC,sBAAsB,IAAI,CAAC,UAAU,EACzDG,eAAe,IAAI,CAAC,UAAU;IAElC;IAEA,MAAc,uBAAuBC,IAAmB,EAAE;QACxD,MAAMC,QAAQC,SAASF;QACvB,MAAMG,MAAMF,QAAQ,GAAGG,QAAQJ,MAAM,GAAG,EAAEC,OAAO,GAAGG,QAAQJ;QAE5D,IAAI,IAAI,CAAC,cAAc,EACrB,MAAM,IAAI,CAAC,cAAc,CAACG;IAE9B;IAEA,MAAc,iBAAiBE,QAAkB,EAAEC,kBAAkB,KAAK,EAAE;QAC1E,IAAI,CAAC,mBAAmB,CAACD,SAAS,IAAI;QAEtC,IAAI;gBACIE,oBAAAA;YAAN,eAAMA,CAAAA,qBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,YAAY,AAAD,IAAhBA,KAAAA,IAAAA,mBAAAA,IAAAA,CAAAA,OAAoB,IAAI,CAAC,cAAc,GAAE;QACjD,EAAE,OAAOC,OAAO;YACdC,QAAQ,KAAK,CAAC,yBAAyBD;QACzC;QAEA,IAAI,CAAC,mBAAmB;QAExB,IAAIH,SAAS,cAAc,MAAM,CAACC,iBAAiB;YACjD,MAAMI,YAAYL,SAAS,eAAe;YAC1C,MAAM,IAAIX,MAAM,GAAGgB,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,YAAY,CAAC,EAAE,EAAEA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,UAAU,EAAE,EAAE;gBACtE,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK;YACzB;QACF;IACF;IAEQ,yBACNC,YAAyB,EACzBC,GAAkB,EACG;QACrBC,OAAOF,cAAc;QAErB,IAAI,AAAe,YAAf,OAAOC,OAAoBA,AAAQ,SAARA,KAAc;YAC3C,MAAMzB,SAASwB;YACf,MAAMG,YAAYF,IAAI,SAAS,IAAI;YACnC,MAAMG,YAAYH,IAAI,SAAS,IAAI;YACnC,MAAMI,QAAQJ,IAAI,KAAK;YAEvB,OAAO;gBACLzB;gBACA2B;gBACAC;gBACAC;YACF;QACF;QAEA,OAAO;YACL,QAAQL;QACV;IACF;IAEA,MAAM,MAAMA,YAAyB,EAAEC,GAAkB,EAAE;QACzD,MAAMK,sBAAsB,IAAI,CAAC,wBAAwB,CACvDN,cACAC;QAEF,MAAMM,QAAQC,WAAW,OAAOF;QAChC,MAAM,EAAEZ,QAAQ,EAAEe,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DC,aAAa,OAAOC,eAAeL,uBACnCC,OACA;YAAE,WAAWN,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,aAAaT,YAAyB,EAAEC,GAAkB,EAAE;QAChE,MAAMK,sBAAsB,IAAI,CAAC,wBAAwB,CACvDN,cACAC;QAEF,MAAMM,QAAQC,WAAW,cAAcF;QACvC,MAAM,EAAEZ,QAAQ,EAAEe,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DC,aAAa,cAAcC,eAAeL,uBAC1CC,OACA;YAAE,WAAWN,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,QAAQT,YAAyB,EAAEC,GAAkB,EAAE;QAC3D,MAAMK,sBAAsB,IAAI,CAAC,wBAAwB,CACvDN,cACAC;QAEF,MAAMM,QAAQC,WAAW,SAASF;QAClC,MAAM,EAAEZ,QAAQ,EAAEe,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DC,aAAa,SAASC,eAAeL,uBACrCC,OACA;YAAE,WAAWN,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,QACJG,KAAa,EACbZ,YAAyB,EACzBC,GAA0C,EAC1C;QACAC,OACE,AAAiB,YAAjB,OAAOU,OACP;QAEFV,OAAOF,cAAc;QACrB,MAAMM,sBAAsB,IAAI,CAAC,wBAAwB,CACvDN,cACAC;QAEF,MAAMM,QAAQC,WAAW,SAASF,qBAAqB;YACrDM;YACA,qBAAqBX,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,mBAAmB;QAC/C;QACA,MAAM,EAAEP,QAAQ,EAAEe,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DC,aAAa,SAASC,eAAeL,uBACrCC,OACA;YACE,WAAWN,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAC3B;QAEF,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,gBACJI,OAAe,EACfb,YAA0B,EAC1BC,GAAkB,EAClB;QACAC,OAAOW,SAAS;QAChB,MAAMP,sBAAsBN,eACxB,IAAI,CAAC,wBAAwB,CAACA,cAAcC,OAC5Ca;QACJ,MAAMP,QAAQC,WAAW,iBAAiBF,qBAAqB;YAC7D,OAAOO;QACT;QACA,MAAM,EAAEnB,QAAQ,EAAEe,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DC,aAAa,iBAAiBC,eAAeL,uBAC7CC,OACA;YAAE,WAAWN,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,SACJM,WAAwB,EACxBf,YAA0B,EAC1BC,GAAkB,EAClB;QACA,MAAMK,sBAAsBN,eACxB,IAAI,CAAC,wBAAwB,CAACA,cAAcC,OAC5Ca;QACJ,MAAMP,QAAQC,WAAW,UAAUF,qBAAqBS;QACxD,MAAMC,eAAehB,eACjB,GAAGW,eAAeL,qBAAqB,GAAG,EAAEW,eAAeF,cAAc,GACzEE,eAAeF;QACnB,MAAM,EAAErB,QAAQ,EAAEe,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DC,aAAa,UAAUM,eACvBT,OACA;YAAE,WAAWN,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,SACJS,UAAkB,EAClBjB,GAEC,EACD;YAOMkB,iBACcC;QAPpB,MAAMhB,YAAYH,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAEhC,MAAMoB,cAAc9C,AAAmB,kBAAnBA;QACpB,MAAM+C,eACJD,eAAejB,AAAc,UAAdA,YACXU,SAAAA,QACAK,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,cAAc,CAACD;QACrC,IAAII,gBAAAA,SAAgBF,CAAAA,mBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,iBAAgB,iBAAiB,AAAD,GAAG;gBAInDG,4BAMWC;YARb,MAAM,EAAE9B,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,sBAAsB,CACjEwB,YAAAA,QACAK,CAAAA,6BAAAA,aAAa,YAAY,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,YAAY;YAGzC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC7B;YAElC1C,MAAM;YACN,MAAMyE,OAAO,QAAAD,CAAAA,8BAAAA,aAAa,YAAY,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,YAAY;YACpD,OAAO,IAAI,CAAC,OAAO,CAACC;QACtB;QAEA,MAAM,EAAEhB,MAAM,EAAEf,QAAQ,EAAE,GAAG,MAAO2B,CAAAA,cAChC,IAAI,CAAC,YAAY,CAAC,YAAY,CAACH,YAAY;YAAEd;QAAU,KACvD,IAAI,CAAC,YAAY,CAAC,MAAM,CAACc,YAAY,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YAC9Dd;QACF,EAAC;QAGL,IAAI,IAAI,CAAC,SAAS,IAAIK,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,QAAQ,AAAD,KAAKL,AAAc,UAAdA,WAAqB;YAC7D,MAAMsB,cAAkC;gBACtC,OAAO;oBACL;wBACE,MAAMR;wBACN,MAAMT,OAAO,QAAQ;oBACvB;iBACD;YACH;YACA,MAAMkB,cAAcF,QAAAA,IAAS,CAACC;YAC9B,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gBACE,MAAM;gBACN,QAAQR;gBACR,cAAcS;YAChB,GACAL;QAEJ;QAEA,MAAM,IAAI,CAAC,gBAAgB,CAAC5B;QAC5B,OAAOe;IACT;IAEA,MAAM,QACJmB,MAA2B,EAC3B3B,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAEsC,MAAM,EAAEf,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAACkC,QAAQ3B;QACnE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,UACJjC,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAEsC,MAAM,EAAEf,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAClB,QAAQyB;QACrE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,SACJjC,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAEsC,MAAM,EAAEf,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAClB,QAAQyB;QACpE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,SACJjC,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAEsC,MAAM,EAAEf,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAClB,QAAQyB;QACpE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOe;IACT;IAEA,MAAM,MACJjC,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,OAAO,IAAI,CAAC,QAAQ,CAACK,QAAQyB;IAC/B;IAEA,MAAM,uBACJ4B,MAAwB,EACxB5B,GAI0B,EACkB;QAC5C,MAAM,EAAE6B,eAAe,IAAI,EAAEC,aAAa,CAAC,EAAE,GAAG9B,OAAO,CAAC;QAExD,IAAI+B,UAAU;QACd,IAAIC,aAAa;QACjB,IAAIC,eAAe;QACnB,IAAI/B,YAAYF,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,AAAD,KAAK;QAClC,IAAIkC;QAEJ,MAAO,CAACH,WAAWC,aAAaF,WAAY;YAC1C,IAAIE,cAAc,GAChB9B,YAAY;YAEdnD,MACE,cACA6E,QACA,gBACAC,cACA,cACAG,YACA,aACA9B;YAEF,MAAMiC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACP,QAAQ;gBAAE1B;YAAU;YAC7DnD,MAAM,mBAAmBoF;YACzBlC,OAAOkC,KAAK,WAAW,EAAE,CAAC,+BAA+B,EAAEP,OAAO,CAAC,CAAC;YACpEK,eAAeE,KAAK,WAAW;YAE/BD,eAAe,MAAM,IAAI,CAAC,aAAa,CACrCD,cACA/B,YAAY;gBAAE,WAAW;YAAK,IAAIW,QAClCe,QACA5B;YAEF,IAAIkC,aAAa,IAAI,EACnBH,UAAU;iBAEVC;QAEJ;QAEA,OAAO;YACL,QAAQC;YACR/B;YACAgC;QACF;IACF;IAEA,MAAM,cACJ3D,MAAc,EACd6D,SAAmC,EACnCC,YAA8B,EAC9BC,kBAA2C,EACX;QAChCvF,MAAM,iBAAiBwB,QAAQ6D,WAAWC,cAAcC;QAExD,MAAM,EAAE,QAAQC,YAAY,EAAE,MAAMC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CACpEjE,QACA6D;QAEF,MAAMK,WAAWxF,oBAAoBoF,cAAcE;QACnD,MAAMG,WAAWjF,eAAe4E,cAAcG;QAC9C,MAAMG,OACJF,YAAaH,CAAAA,CAAAA,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,uBAAuB,AAAD,KAAK,EAAC,KAC7DI;QACF,MAAMR,eAAe;YACnBS;YACA,MAAMH;YACN,QAAQD;YACR,gBAAgBE;QAClB;QACA1F,MAAM,2BAA2BmF;QACjC,OAAOA;IACT;IAEA,MAAM,SAAS3D,MAAmB,EAAEyB,GAAkB,EAAE;QACtD,MAAMK,sBAAsB,IAAI,CAAC,wBAAwB,CAAC9B,QAAQyB;QAClE,MAAMM,QAAQC,WAAW,UAAUF;QACnC,MAAM,EAAEZ,QAAQ,EAAEe,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DC,aAAa,UAAUC,eAAeL,uBACtCC,OACA;YAAE,WAAWN,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAE5B,MAAM,EAAEmD,OAAO,EAAE,GAAGpC;QAEpB,OAAO;YACL,MAAMoC,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,IAAI;YACnB,QAAQA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM;YACvB,OAAQ,OAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAC,EAAG,GAAG;QACrC;IAGF;IAEA,MAAM,SAASC,SAAsB,EAAEC,GAAY,EAAE9C,GAAoB,EAAE;QACzE,MAAM,EAAEQ,MAAM,EAAEf,QAAQ,EAAEsD,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAClEF,WACA;YACE,eAAe;QACjB;QAEF,MAAM,IAAI,CAAC,gBAAgB,CAACpD,UAAU;QAEtC,IAAIO,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,EACtB,OAAO;YACL,MAAMQ;YACNuC;QACF;QAGF,IAAI,CAACvC,QAAQ;gBAGEwC;YAFb,MAAMC,SAASH,OAAO,CAAC,kBAAkB,EAAED,WAAW;YACtD,MAAMK,YAAY,CAAC,QAAQ,EACzBH,WAAAA,SAAWC,CAAAA,4BAAAA,SAAS,eAAe,EAAC,IAAzBA,KAAAA,IAAAA,0BAA4B,KAAK,AAAD,KAAK,eAChD;YACF,MAAM,IAAIlE,MAAM,GAAGmE,OAAO,EAAE,EAAEC,WAAW;QAC3C;IACF;IAEA,MAAM,UAAUL,SAAiB,EAAE7C,GAAqB,EAAE;QACxD,MAAM,EAAEP,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAACoD,WAAW;YAC9D,WAAW7C,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,AAAD,KAAK;YAC7B,iBAAiBA,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,KAAK;YACzC6C;QACF;QACA,MAAM,IAAI,CAAC,gBAAgB,CAACpD,UAAU;QAEtC,IAAIA,SAAS,cAAc,IAAI;YAC7B,MAAMK,YAAYL,SAAS,eAAe;YAC1C,MAAM,IAAIX,MAAM,GAAGgB,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK,CAAC,EAAE,EAAEA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,UAAU,EAAE;QACjE;IACF;IAEA,MAAM,GAAGmB,UAAkB,EAAEkC,OAAO,QAAQ,EAAE;QAC5C,IAAIA,AAAS,aAATA,MACF,OAAO,IAAI,CAAC,QAAQ,CAAClC;QAEvB,IAAIkC,AAAS,YAATA,MACF,OAAO,IAAI,CAAC,OAAO,CAAClC;QAGtB,IAAIkC,AAAS,aAATA,MACF,OAAO,IAAI,CAAC,QAAQ,CAAClC;QAGvB,IAAIkC,AAAS,UAATA,MACF,OAAO,IAAI,CAAC,KAAK,CAAClC;QAGpB,IAAIkC,AAAS,iBAATA,MACF,OAAO,IAAI,CAAC,YAAY,CAAClC;QAG3B,MAAM,IAAInC,MACR,CAAC,cAAc,EAAEqE,KAAK,+DAA+D,CAAC;IAE1F;IAEA,MAAM,QAAQC,iBAAyB,EAEpC;QACD,MAAMC,SAASC,gBAAgBF,mBAAmB,QAAQ;QAC1D,MAAMG,SAAS,IAAIC,aAAaH,QAAQ,OAAOI,SACtC;gBAAE,OAAO,IAAI;gBAAE,QAAQ,EAAE;YAAC;QAEnC,MAAMF,OAAO,GAAG;QAEhB,IAAIA,AAAkB,YAAlBA,OAAO,MAAM,EAAc;YAC7B,MAAMG,SAASH,OAAO,cAAc,CACjC,MAAM,CAAC,CAACnE,OAASA,AAAgB,YAAhBA,KAAK,MAAM,EAC5B,GAAG,CAAC,CAACA;oBAC2BuE;gBAA/B,OAAO,CAAC,OAAO,EAAEvE,KAAK,IAAI,CAAC,EAAE,EAAE,QAAAuE,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,EAAE;YACtD,GACC,IAAI,CAAC;YACR,MAAM,IAAI7E,MAAM,CAAC,2CAA2C,EAAE4E,QAAQ;QACxE;QAEA,OAAO;YACL,QAAQH,OAAO,MAAM;QACvB;IACF;IAEA,MAAM,mBAAmBF,MAAc,EAAE;QACvCpD,OACE,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B;QAEF,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAACoD;IACtC;IAEA,MAAM,UAAU;QACd,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO;QACvB,IAAI,CAAC,SAAS;QACd,IAAI,CAAC,SAAS,GAAG;IACnB;IAEA,MAAM,cACJO,KAAc,EACd5D,GAEC,EACD;QAEA,MAAM6D,SAAS,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAC/C,MAAMC,MAAMC,KAAK,GAAG;QAEpB,MAAMC,WAAoC;YACxC;gBACE,MAAM;gBACN,IAAIF;gBACJ,YAAYD;YACd;SACD;QAED,MAAMzE,OAAyB;YAC7B,MAAM;YACN,SAAS;YACT,QAAQ;YACR4E;YACA,QAAQ;gBACN,OAAOF;gBACP,KAAKA;gBACL,MAAM;YACR;YACA,OAAO;gBACL,SAAS9D,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAK;YAC3B;YACA,UAAU,WAAa;QACzB;QAEA,MAAMiE,gBAA+B;YACnC,YAAY;YACZ,SAASH;YACT,YAAY;YACZ,mBAAmB;YACnB,MAAM,CAAC,MAAM,EAAEF,SAAS,YAAY;YACpC,aAAa5D,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAK;YAC7B,OAAO;gBAACZ;aAAK;QACf;QAEA,IAAI,CAAC,mBAAmB,CAAC6E;QAEzB,IAAI;gBACFtE,oBAAAA;oBAAAA,CAAAA,qBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,YAAY,AAAD,KAAhBA,mBAAAA,IAAAA,CAAAA,OAAoB,IAAI,CAAC,cAAc;QACzC,EAAE,OAAOC,OAAO;YACdC,QAAQ,KAAK,CAAC,yBAAyBD;QACzC;QAEA,IAAI,CAAC,mBAAmB;IAC1B;IAEA,sBAAsB;QACpB,MAAM,EAAEsE,SAAS,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAG,IAAI,CAAC,IAAI;QAC7D,MAAMC,gBAAgBC,MAAM,OAAO,CAACF,cAChCA,WAAW,GAAG,CAAC,CAAC5F;YACd,MAAM,EAAE+F,KAAK,EAAE,GAAGC,eAAe,GAAGhG;YACpC,IAAIiG,WAAWF;YACf,IAAID,MAAM,OAAO,CAACC,QAChBE,WAAWF,MAAM,GAAG,CAAC,CAACnF;gBAEpB,MAAM,EAAEsF,WAAW,EAAEC,GAAG,EAAE,GAAGC,UAAU,GAAGxF;gBAC1C,OAAOwF;YACT;YAEF,OAAO;gBAAE,GAAGJ,aAAa;gBAAE,GAAIC,WAAW;oBAAE,OAAOA;gBAAS,IAAI,CAAC,CAAC;YAAE;QACtE,KACA,EAAE;QACN,OAAO;YACLP;YACAC;YACA,YAAYE;QACd;IACF;IAMA,MAAM,oBAAmC;QACvCtH,MAAM;QACN,MAAM8H,UAAU,MAAM,IAAI,CAAC,gBAAgB;QAE3CA,QAAQ,SAAS,GAAG;QACpB,IAAI,CAAC,iBAAiB,GAAGA;QACzB9H,MAAM;IACR;IAKA,MAAM,sBAAqC;QACzCA,MAAM;QACN,IAAI,CAAC,iBAAiB,GAAG8D;QACzB9D,MAAM;IACR;IA9rBA,YAAY+H,IAAc,EAAEC,IAAmB,CAAE;QAhCjD;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA;QAKA,kCAAU;QAEV;QAEA;QAEA;QAEA,oCAAY;QAKZ,uBAAQ,qBAAR;QAGE,IAAI,CAAC,IAAI,GAAGD;QACZ,IAAI,CAAC,IAAI,GAAGE,OAAO,MAAM,CACvB;YACE,gBAAgB;YAChB,oBAAoB;YACpB,WAAW;YACX,kBAAkB;QACpB,GACAD,QAAQ,CAAC;QAGX,IACE,AAAuB,gBAAvB,IAAI,CAAC,IAAI,CAAC,QAAQ,IAClB,AAAuB,iBAAvB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAClB;YAEE,IAAI,CAAC,IAAI,CACT,wBAAwB,GACvB,IAAI,CAAC,IAAI,CAAqB,wBAAwB,IACvDE;YAEA,IAAI,CAAC,IAAI,CACT,yBAAyB,GACxB,IAAI,CAAC,IAAI,CAAqB,yBAAyB,IACxDC;QACJ;QAEA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc;QAI9C,IAAI,CAAC,OAAO,GAAG,IAAIC,QACjB,OAAO/G,SACE,IAAI,CAAC,YAAY,CAACA;QAI7B,IAAI2G,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAK,AAAuB,cAAvB,IAAI,CAAC,IAAI,CAAC,QAAQ,EACrC,IAAI,CAAC,SAAS,GAAG,IAAIK,UACnBL,KAAK,OAAO,EACZM,qBAAqB;QAIzB,IAAI,CAAC,YAAY,GAAG,IAAIC,iBAAiB,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE;YAChE,WAAW,IAAI,CAAC,SAAS;YACzB,aAAa,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI;QACpD;QACA,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS;QAC1B,IAAI,CAAC,cAAc,GACjBP,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,cAAc,AAAD,KACnBQ,kBAAkBR,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI;IAC5D;AA0oBF"}
+{"version":3,"file":"common/agent.mjs","sources":["webpack://@midscene/web/./src/common/agent.ts"],"sourcesContent":["import type { WebPage } from '@/common/page';\nimport {\n  type AgentAssertOpt,\n  type AgentDescribeElementAtPointResult,\n  type AgentWaitForOpt,\n  type DetailedLocateParam,\n  type DeviceAction,\n  type ExecutionDump,\n  type ExecutionRecorderItem,\n  type ExecutionTask,\n  type ExecutionTaskLog,\n  type Executor,\n  type GroupedActionDump,\n  Insight,\n  type InsightAction,\n  type InsightExtractOption,\n  type InsightExtractParam,\n  type LocateOption,\n  type LocateResultElement,\n  type LocateValidatorResult,\n  type LocatorValidatorOption,\n  type MidsceneYamlScript,\n  type OnTaskStartTip,\n  type PlanningAction,\n  type PlanningLocateParam,\n  type Rect,\n  type ScrollParam,\n  type TUserPrompt,\n} from '@midscene/core';\n\nimport yaml from 'js-yaml';\n\nimport { ScriptPlayer, parseYamlScript } from '@/yaml/index';\nimport {\n  groupedActionDumpFileExt,\n  reportHTMLContent,\n  stringifyDumpData,\n  writeLogFile,\n} from '@midscene/core/utils';\nimport {\n  DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,\n  DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT,\n} from '@midscene/shared/constants';\nimport { getAIConfigInBoolean, vlLocateMode } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { PageTaskExecutor } from '../common/tasks';\nimport type { PlaywrightWebPage } from '../playwright';\nimport type { PuppeteerWebPage } from '../puppeteer';\nimport type { WebElementInfo, WebUIContext } from '../web-element';\nimport type { AndroidDeviceInputOpt } from './page';\nimport { TaskCache } from './task-cache';\nimport { locateParamStr, paramStr, taskTitleStr, typeStr } from './ui-utils';\nimport { getReportFileName, printReportMsg } from './utils';\nimport { parseContextFromWebPage } from './utils';\nimport { trimContextByViewport } from './utils';\n\nconst debug = getDebug('agent');\n\nconst distanceOfTwoPoints = (p1: [number, number], p2: [number, number]) => {\n  const [x1, y1] = p1;\n  const [x2, y2] = p2;\n  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));\n};\n\nconst includedInRect = (point: [number, number], rect: Rect) => {\n  const [x, y] = point;\n  const { left, top, width, height } = rect;\n  return x >= left && x <= left + width && y >= top && y <= top + height;\n};\n\nconst defaultInsightExtractOption: InsightExtractOption = {\n  domIncluded: false,\n  screenshotIncluded: true,\n};\n\nexport interface PageAgentOpt {\n  forceSameTabNavigation?: boolean /* if limit the new tab to the current page, default true */;\n  testId?: string;\n  cacheId?: string;\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n}\n\nexport type WebPageAgentOpt = PageAgentOpt & WebPageOpt;\nexport type WebPageOpt = {\n  waitForNavigationTimeout?: number;\n  waitForNetworkIdleTimeout?: number;\n};\n\nexport class PageAgent<PageType extends WebPage = WebPage> {\n  page: PageType;\n\n  insight: Insight<WebElementInfo, WebUIContext>;\n\n  dump: GroupedActionDump;\n\n  reportFile?: string | null;\n\n  reportFileName?: string;\n\n  taskExecutor: PageTaskExecutor;\n\n  opts: PageAgentOpt;\n\n  /**\n   * If true, the agent will not perform any actions\n   */\n  dryMode = false;\n\n  onTaskStartTip?: OnTaskStartTip;\n\n  taskCache?: TaskCache;\n\n  onDumpUpdate?: (dump: string) => void;\n\n  destroyed = false;\n\n  /**\n   * Frozen page context for consistent AI operations\n   */\n  private frozenPageContext?: WebUIContext;\n\n  constructor(page: PageType, opts?: PageAgentOpt) {\n    this.page = page;\n    this.opts = Object.assign(\n      {\n        generateReport: true,\n        autoPrintReportMsg: true,\n        groupName: 'Midscene Report',\n        groupDescription: '',\n      },\n      opts || {},\n    );\n\n    if (\n      this.page.pageType === 'puppeteer' ||\n      this.page.pageType === 'playwright'\n    ) {\n      (\n        this.page as PuppeteerWebPage | PlaywrightWebPage\n      ).waitForNavigationTimeout =\n        (this.opts as WebPageAgentOpt).waitForNavigationTimeout ??\n        DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;\n      (\n        this.page as PuppeteerWebPage | PlaywrightWebPage\n      ).waitForNetworkIdleTimeout =\n        (this.opts as WebPageAgentOpt).waitForNetworkIdleTimeout ??\n        DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;\n    }\n\n    this.onTaskStartTip = this.opts.onTaskStartTip;\n    // get the parent browser of the puppeteer page\n    // const browser = (this.page as PuppeteerWebPage).browser();\n\n    this.insight = new Insight<WebElementInfo, WebUIContext>(\n      async (action: InsightAction) => {\n        return this.getUIContext(action);\n      },\n    );\n\n    if (opts?.cacheId && this.page.pageType !== 'android') {\n      this.taskCache = new TaskCache(\n        opts.cacheId,\n        getAIConfigInBoolean('MIDSCENE_CACHE'), // if we should use cache to match the element\n      );\n    }\n\n    this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {\n      taskCache: this.taskCache,\n      onTaskStart: this.callbackOnTaskStartTip.bind(this),\n    });\n    this.dump = this.resetDump();\n    this.reportFileName =\n      opts?.reportFileName ||\n      getReportFileName(opts?.testId || this.page.pageType || 'web');\n  }\n\n  async getActionSpace(): Promise<DeviceAction[]> {\n    return this.page.actionSpace();\n  }\n\n  async getUIContext(action?: InsightAction): Promise<WebUIContext> {\n    // If page context is frozen, return the frozen context for all actions\n    if (this.frozenPageContext) {\n      debug('Using frozen page context for action:', action);\n      return this.frozenPageContext;\n    }\n\n    // Otherwise, get fresh context based on the action type\n    if (action && (action === 'extract' || action === 'assert')) {\n      return await parseContextFromWebPage(this.page, {\n        ignoreMarker: true,\n      });\n    }\n    return await parseContextFromWebPage(this.page, {\n      ignoreMarker: !!vlLocateMode(),\n    });\n  }\n\n  async _snapshotContext(): Promise<WebUIContext> {\n    return await this.getUIContext('locate');\n  }\n\n  async setAIActionContext(prompt: string) {\n    this.opts.aiActionContext = prompt;\n  }\n\n  resetDump() {\n    this.dump = {\n      groupName: this.opts.groupName!,\n      groupDescription: this.opts.groupDescription,\n      executions: [],\n    };\n\n    return this.dump;\n  }\n\n  appendExecutionDump(execution: ExecutionDump) {\n    // use trimContextByViewport to process execution\n    const trimmedExecution = trimContextByViewport(execution);\n    const currentDump = this.dump;\n    currentDump.executions.push(trimmedExecution);\n  }\n\n  dumpDataString() {\n    // update dump info\n    this.dump.groupName = this.opts.groupName!;\n    this.dump.groupDescription = this.opts.groupDescription;\n    return stringifyDumpData(this.dump);\n  }\n\n  reportHTMLString() {\n    return reportHTMLContent(this.dumpDataString());\n  }\n\n  writeOutActionDumps() {\n    if (this.destroyed) {\n      throw new Error(\n        'PageAgent has been destroyed. Cannot update report file.',\n      );\n    }\n    const { generateReport, autoPrintReportMsg } = this.opts;\n    this.reportFile = writeLogFile({\n      fileName: this.reportFileName!,\n      fileExt: groupedActionDumpFileExt,\n      fileContent: this.dumpDataString(),\n      type: 'dump',\n      generateReport,\n    });\n    debug('writeOutActionDumps', this.reportFile);\n    if (generateReport && autoPrintReportMsg && this.reportFile) {\n      printReportMsg(this.reportFile);\n    }\n  }\n\n  private async callbackOnTaskStartTip(task: ExecutionTask) {\n    const param = paramStr(task);\n    const tip = param ? `${typeStr(task)} - ${param}` : typeStr(task);\n\n    if (this.onTaskStartTip) {\n      await this.onTaskStartTip(tip);\n    }\n  }\n\n  private async afterTaskRunning(executor: Executor, doNotThrowError = false) {\n    this.appendExecutionDump(executor.dump());\n\n    try {\n      await this.onDumpUpdate?.(this.dumpDataString());\n    } catch (error) {\n      console.error('Error in onDumpUpdate', error);\n    }\n\n    this.writeOutActionDumps();\n\n    if (executor.isInErrorState() && !doNotThrowError) {\n      const errorTask = executor.latestErrorTask();\n      throw new Error(`${errorTask?.errorMessage}\\n${errorTask?.errorStack}`, {\n        cause: errorTask?.error,\n      });\n    }\n  }\n\n  private buildDetailedLocateParam(\n    locatePrompt: TUserPrompt,\n    opt?: LocateOption,\n  ): DetailedLocateParam {\n    assert(locatePrompt, 'missing locate prompt');\n\n    if (typeof opt === 'object' && opt !== null) {\n      const prompt = locatePrompt;\n      const deepThink = opt.deepThink ?? false;\n      const cacheable = opt.cacheable ?? true;\n      const xpath = opt.xpath;\n\n      return {\n        prompt,\n        deepThink,\n        cacheable,\n        xpath,\n      };\n    }\n\n    return {\n      prompt: locatePrompt,\n    };\n  }\n\n  private locateTaskForLocate(prompt: TUserPrompt, opt?: LocateOption) {\n    const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);\n    const locatePlan: PlanningAction<PlanningLocateParam> = {\n      type: 'Locate',\n      locate: detailedLocateParam,\n      param: detailedLocateParam,\n      thought: '',\n    };\n    return locatePlan;\n  }\n\n  async callActionInActionSpace<T = any>(\n    type: string,\n    locatePrompt?: TUserPrompt,\n    opt?: LocateOption, // and all other action params\n  ) {\n    debug('callActionInActionSpace', type, ',', locatePrompt, ',', opt);\n    const locatePlan = locatePrompt\n      ? this.locateTaskForLocate(locatePrompt, opt)\n      : null;\n\n    const actionPlan: PlanningAction<T> = {\n      type: type as any,\n      param: opt as any,\n      thought: '',\n      locate: locatePlan?.locate,\n    };\n\n    const plans: PlanningAction[] = [locatePlan, actionPlan].filter(\n      Boolean,\n    ) as PlanningAction[];\n\n    const title = taskTitleStr(\n      type as any,\n      locateParamStr(locatePlan?.locate || undefined),\n    );\n\n    const { executor } = await this.taskExecutor.runPlans(title, plans, {\n      cacheable: opt?.cacheable,\n    });\n    await this.afterTaskRunning(executor);\n  }\n\n  async aiTap(locatePrompt: TUserPrompt, opt?: LocateOption) {\n    return this.callActionInActionSpace('Tap', locatePrompt, opt);\n  }\n\n  async aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption) {\n    return this.callActionInActionSpace('RightClick', locatePrompt, opt);\n  }\n\n  async aiHover(locatePrompt: TUserPrompt, opt?: LocateOption) {\n    return this.callActionInActionSpace('Hover', locatePrompt, opt);\n  }\n\n  // New signature, always use locatePrompt as the first param\n  async aiInput(\n    locatePrompt: TUserPrompt,\n    opt: AndroidDeviceInputOpt & LocateOption & { value: string },\n  ): Promise<any>;\n\n  // Legacy signature - deprecated\n  /**\n   * @deprecated Use aiInput(locatePrompt, opt) instead where opt contains the value\n   */\n  async aiInput(\n    value: string,\n    locatePrompt: TUserPrompt,\n    opt?: AndroidDeviceInputOpt & LocateOption,\n  ): Promise<any>;\n\n  // Implementation\n  async aiInput(\n    locatePromptOrValue: TUserPrompt | string,\n    locatePromptOrOpt:\n      | TUserPrompt\n      | (AndroidDeviceInputOpt & LocateOption & { value: string })\n      | undefined,\n    optOrUndefined?: AndroidDeviceInputOpt & LocateOption,\n  ) {\n    let value: string;\n    let locatePrompt: TUserPrompt;\n    let opt:\n      | (AndroidDeviceInputOpt & LocateOption & { value: string })\n      | undefined;\n\n    // Check if using new signature (first param is locatePrompt, second has value)\n    if (\n      typeof locatePromptOrOpt === 'object' &&\n      locatePromptOrOpt !== null &&\n      'value' in locatePromptOrOpt\n    ) {\n      // New signature: aiInput(locatePrompt, opt)\n      locatePrompt = locatePromptOrValue as TUserPrompt;\n      const optWithValue = locatePromptOrOpt as AndroidDeviceInputOpt &\n        LocateOption & { value: string };\n      value = optWithValue.value;\n      opt = optWithValue;\n    } else {\n      // Legacy signature: aiInput(value, locatePrompt, opt)\n      value = locatePromptOrValue as string;\n      locatePrompt = locatePromptOrOpt as TUserPrompt;\n      opt = {\n        ...optOrUndefined,\n        value,\n      };\n    }\n\n    assert(\n      typeof value === 'string',\n      'input value must be a string, use empty string if you want to clear the input',\n    );\n    assert(locatePrompt, 'missing locate prompt for input');\n    return this.callActionInActionSpace('Input', locatePrompt, opt);\n  }\n\n  // New signature\n  async aiKeyboardPress(\n    locatePrompt: TUserPrompt,\n    opt: LocateOption & { keyName: string },\n  ): Promise<any>;\n\n  // Legacy signature - deprecated\n  /**\n   * @deprecated Use aiKeyboardPress(locatePrompt, opt) instead where opt contains the keyName\n   */\n  async aiKeyboardPress(\n    keyName: string,\n    locatePrompt?: TUserPrompt,\n    opt?: LocateOption,\n  ): Promise<any>;\n\n  // Implementation\n  async aiKeyboardPress(\n    locatePromptOrKeyName: TUserPrompt | string,\n    locatePromptOrOpt:\n      | TUserPrompt\n      | (LocateOption & { keyName: string })\n      | undefined,\n    optOrUndefined?: LocateOption,\n  ) {\n    let keyName: string;\n    let locatePrompt: TUserPrompt | undefined;\n    let opt: (LocateOption & { keyName: string }) | undefined;\n\n    // Check if using new signature (first param is locatePrompt, second has keyName)\n    if (\n      typeof locatePromptOrOpt === 'object' &&\n      locatePromptOrOpt !== null &&\n      'keyName' in locatePromptOrOpt\n    ) {\n      // New signature: aiKeyboardPress(locatePrompt, opt)\n      locatePrompt = locatePromptOrKeyName as TUserPrompt;\n      opt = locatePromptOrOpt as LocateOption & {\n        keyName: string;\n      };\n    } else {\n      // Legacy signature: aiKeyboardPress(keyName, locatePrompt, opt)\n      keyName = locatePromptOrKeyName as string;\n      locatePrompt = locatePromptOrOpt as TUserPrompt | undefined;\n      opt = {\n        ...(optOrUndefined || {}),\n        keyName,\n      };\n    }\n\n    assert(opt?.keyName, 'missing keyName for keyboard press');\n    return this.callActionInActionSpace('KeyboardPress', locatePrompt, opt);\n  }\n\n  // New signature\n  async aiScroll(\n    locatePrompt: TUserPrompt | undefined,\n    opt: LocateOption & ScrollParam,\n  ): Promise<any>;\n\n  // Legacy signature - deprecated\n  /**\n   * @deprecated Use aiScroll(locatePrompt, opt) instead where opt contains the scroll parameters\n   */\n  async aiScroll(\n    scrollParam: ScrollParam,\n    locatePrompt?: TUserPrompt,\n    opt?: LocateOption,\n  ): Promise<any>;\n\n  // Implementation\n  async aiScroll(\n    locatePromptOrScrollParam: TUserPrompt | ScrollParam | undefined,\n    locatePromptOrOpt: TUserPrompt | (LocateOption & ScrollParam) | undefined,\n    optOrUndefined?: LocateOption,\n  ) {\n    let scrollParam: ScrollParam;\n    let locatePrompt: TUserPrompt | undefined;\n    let opt: LocateOption | undefined;\n\n    // Check if using new signature (first param is locatePrompt, second has scroll params)\n    if (\n      typeof locatePromptOrOpt === 'object' &&\n      ('direction' in locatePromptOrOpt ||\n        'scrollType' in locatePromptOrOpt ||\n        'distance' in locatePromptOrOpt)\n    ) {\n      // New signature: aiScroll(locatePrompt, opt)\n      locatePrompt = locatePromptOrScrollParam as TUserPrompt;\n      opt = locatePromptOrOpt as LocateOption & ScrollParam;\n    } else {\n      // Legacy signature: aiScroll(scrollParam, locatePrompt, opt)\n      scrollParam = locatePromptOrScrollParam as ScrollParam;\n      locatePrompt = locatePromptOrOpt as TUserPrompt | undefined;\n      opt = {\n        ...(optOrUndefined || {}),\n        ...(scrollParam || {}),\n      };\n    }\n\n    return this.callActionInActionSpace('Scroll', locatePrompt, opt);\n  }\n\n  async aiAction(\n    taskPrompt: string,\n    opt?: {\n      cacheable?: boolean;\n    },\n  ) {\n    const cacheable = opt?.cacheable;\n    // if vlm-ui-tars, plan cache is not used\n    const isVlmUiTars = vlLocateMode() === 'vlm-ui-tars';\n    const matchedCache =\n      isVlmUiTars || cacheable === false\n        ? undefined\n        : this.taskCache?.matchPlanCache(taskPrompt);\n    if (matchedCache && this.taskCache?.isCacheResultUsed) {\n      // log into report file\n      const { executor } = await this.taskExecutor.loadYamlFlowAsPlanning(\n        taskPrompt,\n        matchedCache.cacheContent?.yamlWorkflow,\n      );\n\n      await await this.afterTaskRunning(executor);\n\n      debug('matched cache, will call .runYaml to run the action');\n      const yaml = matchedCache.cacheContent?.yamlWorkflow;\n      return this.runYaml(yaml);\n    }\n\n    const { output, executor } = await (isVlmUiTars\n      ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable })\n      : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {\n          cacheable,\n        }));\n\n    // update cache\n    if (this.taskCache && output?.yamlFlow && cacheable !== false) {\n      const yamlContent: MidsceneYamlScript = {\n        tasks: [\n          {\n            name: taskPrompt,\n            flow: output.yamlFlow,\n          },\n        ],\n      };\n      const yamlFlowStr = yaml.dump(yamlContent);\n      this.taskCache.updateOrAppendCacheRecord(\n        {\n          type: 'plan',\n          prompt: taskPrompt,\n          yamlWorkflow: yamlFlowStr,\n        },\n        matchedCache,\n      );\n    }\n\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiQuery(\n    demand: InsightExtractParam,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.query(demand, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiBoolean(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.boolean(prompt, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiNumber(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.number(prompt, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiString(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    const { output, executor } = await this.taskExecutor.string(prompt, opt);\n    await this.afterTaskRunning(executor);\n    return output;\n  }\n\n  async aiAsk(\n    prompt: TUserPrompt,\n    opt: InsightExtractOption = defaultInsightExtractOption,\n  ) {\n    return this.aiString(prompt, opt);\n  }\n\n  async describeElementAtPoint(\n    center: [number, number],\n    opt?: {\n      verifyPrompt?: boolean;\n      retryLimit?: number;\n      deepThink?: boolean;\n    } & LocatorValidatorOption,\n  ): Promise<AgentDescribeElementAtPointResult> {\n    const { verifyPrompt = true, retryLimit = 3 } = opt || {};\n\n    let success = false;\n    let retryCount = 0;\n    let resultPrompt = '';\n    let deepThink = opt?.deepThink || false;\n    let verifyResult: LocateValidatorResult | undefined;\n\n    while (!success && retryCount < retryLimit) {\n      if (retryCount >= 2) {\n        deepThink = true;\n      }\n      debug(\n        'aiDescribe',\n        center,\n        'verifyPrompt',\n        verifyPrompt,\n        'retryCount',\n        retryCount,\n        'deepThink',\n        deepThink,\n      );\n      const text = await this.insight.describe(center, { deepThink });\n      debug('aiDescribe text', text);\n      assert(text.description, `failed to describe element at [${center}]`);\n      resultPrompt = text.description;\n\n      verifyResult = await this.verifyLocator(\n        resultPrompt,\n        deepThink ? { deepThink: true } : undefined,\n        center,\n        opt,\n      );\n      if (verifyResult.pass) {\n        success = true;\n      } else {\n        retryCount++;\n      }\n    }\n\n    return {\n      prompt: resultPrompt,\n      deepThink,\n      verifyResult,\n    };\n  }\n\n  async verifyLocator(\n    prompt: string,\n    locateOpt: LocateOption | undefined,\n    expectCenter: [number, number],\n    verifyLocateOption?: LocatorValidatorOption,\n  ): Promise<LocateValidatorResult> {\n    debug('verifyLocator', prompt, locateOpt, expectCenter, verifyLocateOption);\n\n    const { center: verifyCenter, rect: verifyRect } = await this.aiLocate(\n      prompt,\n      locateOpt,\n    );\n    const distance = distanceOfTwoPoints(expectCenter, verifyCenter);\n    const included = includedInRect(expectCenter, verifyRect);\n    const pass =\n      distance <= (verifyLocateOption?.centerDistanceThreshold || 20) ||\n      included;\n    const verifyResult = {\n      pass,\n      rect: verifyRect,\n      center: verifyCenter,\n      centerDistance: distance,\n    };\n    debug('aiDescribe verifyResult', verifyResult);\n    return verifyResult;\n  }\n\n  async aiLocate(prompt: TUserPrompt, opt?: LocateOption) {\n    const locatePlan = this.locateTaskForLocate(prompt, opt);\n    const plans = [locatePlan];\n    const { executor, output } = await this.taskExecutor.runPlans(\n      taskTitleStr('Locate', locateParamStr(locatePlan?.locate || undefined)),\n      plans,\n      { cacheable: opt?.cacheable },\n    );\n    await this.afterTaskRunning(executor);\n\n    const { element } = output;\n\n    return {\n      rect: element?.rect,\n      center: element?.center,\n      scale: (await this.page.size()).dpr,\n    } as Pick<LocateResultElement, 'rect' | 'center'> & {\n      scale: number;\n    };\n  }\n\n  async aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt) {\n    const { output, executor, thought } = await this.taskExecutor.assert(\n      assertion,\n      {\n        returnThought: true,\n      },\n    );\n    await this.afterTaskRunning(executor, true);\n\n    if (opt?.keepRawResponse) {\n      return {\n        pass: output,\n        thought,\n      };\n    }\n\n    if (!output) {\n      const errMsg = msg || `Assertion failed: ${assertion}`;\n      const reasonMsg = `Reason: ${\n        thought || executor.latestErrorTask()?.error || '(no_reason)'\n      }`;\n      throw new Error(`${errMsg}\\n${reasonMsg}`);\n    }\n  }\n\n  async aiWaitFor(assertion: string, opt?: AgentWaitForOpt) {\n    const { executor } = await this.taskExecutor.waitFor(assertion, {\n      timeoutMs: opt?.timeoutMs || 15 * 1000,\n      checkIntervalMs: opt?.checkIntervalMs || 3 * 1000,\n      assertion,\n    });\n    await this.afterTaskRunning(executor, true);\n\n    if (executor.isInErrorState()) {\n      const errorTask = executor.latestErrorTask();\n      throw new Error(`${errorTask?.error}\\n${errorTask?.errorStack}`);\n    }\n  }\n\n  async ai(taskPrompt: string, type = 'action') {\n    if (type === 'action') {\n      return this.aiAction(taskPrompt);\n    }\n    if (type === 'query') {\n      return this.aiQuery(taskPrompt);\n    }\n\n    if (type === 'assert') {\n      return this.aiAssert(taskPrompt);\n    }\n\n    if (type === 'tap') {\n      return this.aiTap(taskPrompt);\n    }\n\n    if (type === 'rightClick') {\n      return this.aiRightClick(taskPrompt);\n    }\n\n    throw new Error(\n      `Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'rightClick'`,\n    );\n  }\n\n  async runYaml(yamlScriptContent: string): Promise<{\n    result: Record<string, any>;\n  }> {\n    const script = parseYamlScript(yamlScriptContent, 'yaml', true);\n    const actionSpace = await this.page.actionSpace();\n    const player = new ScriptPlayer(script, async (target) => {\n      return { agent: this, freeFn: [] };\n    });\n    await player.run();\n\n    if (player.status === 'error') {\n      const errors = player.taskStatusList\n        .filter((task) => task.status === 'error')\n        .map((task) => {\n          return `task - ${task.name}: ${task.error?.message}`;\n        })\n        .join('\\n');\n      throw new Error(`Error(s) occurred in running yaml script:\\n${errors}`);\n    }\n\n    return {\n      result: player.result,\n    };\n  }\n\n  async evaluateJavaScript(script: string) {\n    assert(\n      this.page.evaluateJavaScript,\n      'evaluateJavaScript is not supported in current agent',\n    );\n    return this.page.evaluateJavaScript(script);\n  }\n\n  async destroy() {\n    await this.page.destroy();\n    this.resetDump(); // reset dump to release memory\n    this.destroyed = true;\n  }\n\n  async logScreenshot(\n    title?: string,\n    opt?: {\n      content: string;\n    },\n  ) {\n    // 1. screenshot\n    const base64 = await this.page.screenshotBase64();\n    const now = Date.now();\n    // 2. build recorder\n    const recorder: ExecutionRecorderItem[] = [\n      {\n        type: 'screenshot',\n        ts: now,\n        screenshot: base64,\n      },\n    ];\n    // 3. build ExecutionTaskLog\n    const task: ExecutionTaskLog = {\n      type: 'Log',\n      subType: 'Screenshot',\n      status: 'finished',\n      recorder,\n      timing: {\n        start: now,\n        end: now,\n        cost: 0,\n      },\n      param: {\n        content: opt?.content || '',\n      },\n      executor: async () => {},\n    };\n    // 4. build ExecutionDump\n    const executionDump: ExecutionDump = {\n      sdkVersion: '',\n      logTime: now,\n      model_name: '',\n      model_description: '',\n      name: `Log - ${title || 'untitled'}`,\n      description: opt?.content || '',\n      tasks: [task],\n    };\n    // 5. append to execution dump\n    this.appendExecutionDump(executionDump);\n\n    try {\n      this.onDumpUpdate?.(this.dumpDataString());\n    } catch (error) {\n      console.error('Failed to update dump', error);\n    }\n\n    this.writeOutActionDumps();\n  }\n\n  _unstableLogContent() {\n    const { groupName, groupDescription, executions } = this.dump;\n    const newExecutions = Array.isArray(executions)\n      ? executions.map((execution: any) => {\n          const { tasks, ...restExecution } = execution;\n          let newTasks = tasks;\n          if (Array.isArray(tasks)) {\n            newTasks = tasks.map((task: any) => {\n              // only remove pageContext and log from task\n              const { pageContext, log, ...restTask } = task;\n              return restTask;\n            });\n          }\n          return { ...restExecution, ...(newTasks ? { tasks: newTasks } : {}) };\n        })\n      : [];\n    return {\n      groupName,\n      groupDescription,\n      executions: newExecutions,\n    };\n  }\n\n  /**\n   * Freezes the current page context to be reused in subsequent AI operations\n   * This avoids recalculating page context for each operation\n   */\n  async freezePageContext(): Promise<void> {\n    debug('Freezing page context');\n    const context = await this._snapshotContext();\n    // Mark the context as frozen\n    context._isFrozen = true;\n    this.frozenPageContext = context;\n    debug('Page context frozen successfully');\n  }\n\n  /**\n   * Unfreezes the page context, allowing AI operations to calculate context dynamically\n   */\n  async unfreezePageContext(): Promise<void> {\n    debug('Unfreezing page context');\n    this.frozenPageContext = undefined;\n    debug('Page context unfrozen successfully');\n  }\n}\n"],"names":["debug","getDebug","distanceOfTwoPoints","p1","p2","x1","y1","x2","y2","Math","includedInRect","point","rect","x","y","left","top","width","height","defaultInsightExtractOption","PageAgent","action","parseContextFromWebPage","vlLocateMode","prompt","execution","trimmedExecution","trimContextByViewport","currentDump","stringifyDumpData","reportHTMLContent","Error","generateReport","autoPrintReportMsg","writeLogFile","groupedActionDumpFileExt","printReportMsg","task","param","paramStr","tip","typeStr","executor","doNotThrowError","_this","error","console","errorTask","locatePrompt","opt","assert","deepThink","cacheable","xpath","detailedLocateParam","locatePlan","type","actionPlan","plans","Boolean","title","taskTitleStr","locateParamStr","undefined","locatePromptOrValue","locatePromptOrOpt","optOrUndefined","value","optWithValue","locatePromptOrKeyName","keyName","locatePromptOrScrollParam","scrollParam","taskPrompt","_this_taskCache","_this_taskCache1","isVlmUiTars","matchedCache","_matchedCache_cacheContent","_matchedCache_cacheContent1","yaml","output","yamlContent","yamlFlowStr","demand","center","verifyPrompt","retryLimit","success","retryCount","resultPrompt","verifyResult","text","locateOpt","expectCenter","verifyLocateOption","verifyCenter","verifyRect","distance","included","pass","element","assertion","msg","thought","_executor_latestErrorTask","errMsg","reasonMsg","yamlScriptContent","script","parseYamlScript","player","ScriptPlayer","target","errors","_task_error","base64","now","Date","recorder","executionDump","groupName","groupDescription","executions","newExecutions","Array","tasks","restExecution","newTasks","pageContext","log","restTask","context","page","opts","Object","DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT","DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT","Insight","TaskCache","getAIConfigInBoolean","PageTaskExecutor","getReportFileName"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAyDA,MAAMA,QAAQC,SAAS;AAEvB,MAAMC,sBAAsB,CAACC,IAAsBC;IACjD,MAAM,CAACC,IAAIC,GAAG,GAAGH;IACjB,MAAM,CAACI,IAAIC,GAAG,GAAGJ;IACjB,OAAOK,KAAK,KAAK,CAACA,KAAK,IAAI,CAAEJ,AAAAA,CAAAA,KAAKE,EAAC,KAAM,IAAKD,AAAAA,CAAAA,KAAKE,EAAC,KAAM;AAC5D;AAEA,MAAME,iBAAiB,CAACC,OAAyBC;IAC/C,MAAM,CAACC,GAAGC,EAAE,GAAGH;IACf,MAAM,EAAEI,IAAI,EAAEC,GAAG,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGN;IACrC,OAAOC,KAAKE,QAAQF,KAAKE,OAAOE,SAASH,KAAKE,OAAOF,KAAKE,MAAME;AAClE;AAEA,MAAMC,8BAAoD;IACxD,aAAa;IACb,oBAAoB;AACtB;AAwBO,MAAMC;IAwFX,MAAM,iBAA0C;QAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW;IAC9B;IAEA,MAAM,aAAaC,MAAsB,EAAyB;QAEhE,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1BrB,MAAM,yCAAyCqB;YAC/C,OAAO,IAAI,CAAC,iBAAiB;QAC/B;QAGA,IAAIA,UAAWA,CAAAA,AAAW,cAAXA,UAAwBA,AAAW,aAAXA,MAAkB,GACvD,OAAO,MAAMC,wBAAwB,IAAI,CAAC,IAAI,EAAE;YAC9C,cAAc;QAChB;QAEF,OAAO,MAAMA,wBAAwB,IAAI,CAAC,IAAI,EAAE;YAC9C,cAAc,CAAC,CAACC;QAClB;IACF;IAEA,MAAM,mBAA0C;QAC9C,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC;IACjC;IAEA,MAAM,mBAAmBC,MAAc,EAAE;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,GAAGA;IAC9B;IAEA,YAAY;QACV,IAAI,CAAC,IAAI,GAAG;YACV,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,kBAAkB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAC5C,YAAY,EAAE;QAChB;QAEA,OAAO,IAAI,CAAC,IAAI;IAClB;IAEA,oBAAoBC,SAAwB,EAAE;QAE5C,MAAMC,mBAAmBC,sBAAsBF;QAC/C,MAAMG,cAAc,IAAI,CAAC,IAAI;QAC7BA,YAAY,UAAU,CAAC,IAAI,CAACF;IAC9B;IAEA,iBAAiB;QAEf,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;QACvD,OAAOG,kBAAkB,IAAI,CAAC,IAAI;IACpC;IAEA,mBAAmB;QACjB,OAAOC,kBAAkB,IAAI,CAAC,cAAc;IAC9C;IAEA,sBAAsB;QACpB,IAAI,IAAI,CAAC,SAAS,EAChB,MAAM,IAAIC,MACR;QAGJ,MAAM,EAAEC,cAAc,EAAEC,kBAAkB,EAAE,GAAG,IAAI,CAAC,IAAI;QACxD,IAAI,CAAC,UAAU,GAAGC,aAAa;YAC7B,UAAU,IAAI,CAAC,cAAc;YAC7B,SAASC;YACT,aAAa,IAAI,CAAC,cAAc;YAChC,MAAM;YACNH;QACF;QACAhC,MAAM,uBAAuB,IAAI,CAAC,UAAU;QAC5C,IAAIgC,kBAAkBC,sBAAsB,IAAI,CAAC,UAAU,EACzDG,eAAe,IAAI,CAAC,UAAU;IAElC;IAEA,MAAc,uBAAuBC,IAAmB,EAAE;QACxD,MAAMC,QAAQC,SAASF;QACvB,MAAMG,MAAMF,QAAQ,GAAGG,QAAQJ,MAAM,GAAG,EAAEC,OAAO,GAAGG,QAAQJ;QAE5D,IAAI,IAAI,CAAC,cAAc,EACrB,MAAM,IAAI,CAAC,cAAc,CAACG;IAE9B;IAEA,MAAc,iBAAiBE,QAAkB,EAAEC,kBAAkB,KAAK,EAAE;QAC1E,IAAI,CAAC,mBAAmB,CAACD,SAAS,IAAI;QAEtC,IAAI;gBACIE,oBAAAA;YAAN,eAAMA,CAAAA,qBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,YAAY,AAAD,IAAhBA,KAAAA,IAAAA,mBAAAA,IAAAA,CAAAA,OAAoB,IAAI,CAAC,cAAc,GAAE;QACjD,EAAE,OAAOC,OAAO;YACdC,QAAQ,KAAK,CAAC,yBAAyBD;QACzC;QAEA,IAAI,CAAC,mBAAmB;QAExB,IAAIH,SAAS,cAAc,MAAM,CAACC,iBAAiB;YACjD,MAAMI,YAAYL,SAAS,eAAe;YAC1C,MAAM,IAAIX,MAAM,GAAGgB,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,YAAY,CAAC,EAAE,EAAEA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,UAAU,EAAE,EAAE;gBACtE,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK;YACzB;QACF;IACF;IAEQ,yBACNC,YAAyB,EACzBC,GAAkB,EACG;QACrBC,OAAOF,cAAc;QAErB,IAAI,AAAe,YAAf,OAAOC,OAAoBA,AAAQ,SAARA,KAAc;YAC3C,MAAMzB,SAASwB;YACf,MAAMG,YAAYF,IAAI,SAAS,IAAI;YACnC,MAAMG,YAAYH,IAAI,SAAS,IAAI;YACnC,MAAMI,QAAQJ,IAAI,KAAK;YAEvB,OAAO;gBACLzB;gBACA2B;gBACAC;gBACAC;YACF;QACF;QAEA,OAAO;YACL,QAAQL;QACV;IACF;IAEQ,oBAAoBxB,MAAmB,EAAEyB,GAAkB,EAAE;QACnE,MAAMK,sBAAsB,IAAI,CAAC,wBAAwB,CAAC9B,QAAQyB;QAClE,MAAMM,aAAkD;YACtD,MAAM;YACN,QAAQD;YACR,OAAOA;YACP,SAAS;QACX;QACA,OAAOC;IACT;IAEA,MAAM,wBACJC,IAAY,EACZR,YAA0B,EAC1BC,GAAkB,EAClB;QACAjD,MAAM,2BAA2BwD,MAAM,KAAKR,cAAc,KAAKC;QAC/D,MAAMM,aAAaP,eACf,IAAI,CAAC,mBAAmB,CAACA,cAAcC,OACvC;QAEJ,MAAMQ,aAAgC;YACpC,MAAMD;YACN,OAAOP;YACP,SAAS;YACT,QAAQM,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,MAAM;QAC5B;QAEA,MAAMG,QAA0B;YAACH;YAAYE;SAAW,CAAC,MAAM,CAC7DE;QAGF,MAAMC,QAAQC,aACZL,MACAM,eAAeP,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,MAAM,AAAD,KAAKQ;QAGvC,MAAM,EAAErB,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAACkB,OAAOF,OAAO;YAClE,WAAWT,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAC3B;QACA,MAAM,IAAI,CAAC,gBAAgB,CAACP;IAC9B;IAEA,MAAM,MAAMM,YAAyB,EAAEC,GAAkB,EAAE;QACzD,OAAO,IAAI,CAAC,uBAAuB,CAAC,OAAOD,cAAcC;IAC3D;IAEA,MAAM,aAAaD,YAAyB,EAAEC,GAAkB,EAAE;QAChE,OAAO,IAAI,CAAC,uBAAuB,CAAC,cAAcD,cAAcC;IAClE;IAEA,MAAM,QAAQD,YAAyB,EAAEC,GAAkB,EAAE;QAC3D,OAAO,IAAI,CAAC,uBAAuB,CAAC,SAASD,cAAcC;IAC7D;IAmBA,MAAM,QACJe,mBAAyC,EACzCC,iBAGa,EACbC,cAAqD,EACrD;QACA,IAAIC;QACJ,IAAInB;QACJ,IAAIC;QAKJ,IACE,AAA6B,YAA7B,OAAOgB,qBACPA,AAAsB,SAAtBA,qBACA,WAAWA,mBACX;YAEAjB,eAAegB;YACf,MAAMI,eAAeH;YAErBE,QAAQC,aAAa,KAAK;YAC1BnB,MAAMmB;QACR,OAAO;YAELD,QAAQH;YACRhB,eAAeiB;YACfhB,MAAM;gBACJ,GAAGiB,cAAc;gBACjBC;YACF;QACF;QAEAjB,OACE,AAAiB,YAAjB,OAAOiB,OACP;QAEFjB,OAAOF,cAAc;QACrB,OAAO,IAAI,CAAC,uBAAuB,CAAC,SAASA,cAAcC;IAC7D;IAmBA,MAAM,gBACJoB,qBAA2C,EAC3CJ,iBAGa,EACbC,cAA6B,EAC7B;QACA,IAAII;QACJ,IAAItB;QACJ,IAAIC;QAGJ,IACE,AAA6B,YAA7B,OAAOgB,qBACPA,AAAsB,SAAtBA,qBACA,aAAaA,mBACb;YAEAjB,eAAeqB;YACfpB,MAAMgB;QAGR,OAAO;YAELK,UAAUD;YACVrB,eAAeiB;YACfhB,MAAM;gBACJ,GAAIiB,kBAAkB,CAAC,CAAC;gBACxBI;YACF;QACF;QAEApB,OAAOD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,EAAE;QACrB,OAAO,IAAI,CAAC,uBAAuB,CAAC,iBAAiBD,cAAcC;IACrE;IAmBA,MAAM,SACJsB,yBAAgE,EAChEN,iBAAyE,EACzEC,cAA6B,EAC7B;QACA,IAAIM;QACJ,IAAIxB;QACJ,IAAIC;QAGJ,IACE,AAA6B,YAA7B,OAAOgB,qBACN,gBAAeA,qBACd,gBAAgBA,qBAChB,cAAcA,iBAAgB,GAChC;YAEAjB,eAAeuB;YACftB,MAAMgB;QACR,OAAO;YAELO,cAAcD;YACdvB,eAAeiB;YACfhB,MAAM;gBACJ,GAAIiB,kBAAkB,CAAC,CAAC;gBACxB,GAAIM,eAAe,CAAC,CAAC;YACvB;QACF;QAEA,OAAO,IAAI,CAAC,uBAAuB,CAAC,UAAUxB,cAAcC;IAC9D;IAEA,MAAM,SACJwB,UAAkB,EAClBxB,GAEC,EACD;YAOMyB,iBACcC;QAPpB,MAAMvB,YAAYH,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAEhC,MAAM2B,cAAcrD,AAAmB,kBAAnBA;QACpB,MAAMsD,eACJD,eAAexB,AAAc,UAAdA,YACXW,SAAAA,QACAW,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,cAAc,CAACD;QACrC,IAAII,gBAAAA,SAAgBF,CAAAA,mBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,iBAAgB,iBAAiB,AAAD,GAAG;gBAInDG,4BAMWC;YARb,MAAM,EAAErC,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,sBAAsB,CACjE+B,YAAAA,QACAK,CAAAA,6BAAAA,aAAa,YAAY,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,YAAY;YAGzC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAACpC;YAElC1C,MAAM;YACN,MAAMgF,OAAO,QAAAD,CAAAA,8BAAAA,aAAa,YAAY,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,YAAY;YACpD,OAAO,IAAI,CAAC,OAAO,CAACC;QACtB;QAEA,MAAM,EAAEC,MAAM,EAAEvC,QAAQ,EAAE,GAAG,MAAOkC,CAAAA,cAChC,IAAI,CAAC,YAAY,CAAC,YAAY,CAACH,YAAY;YAAErB;QAAU,KACvD,IAAI,CAAC,YAAY,CAAC,MAAM,CAACqB,YAAY,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YAC9DrB;QACF,EAAC;QAGL,IAAI,IAAI,CAAC,SAAS,IAAI6B,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,QAAQ,AAAD,KAAK7B,AAAc,UAAdA,WAAqB;YAC7D,MAAM8B,cAAkC;gBACtC,OAAO;oBACL;wBACE,MAAMT;wBACN,MAAMQ,OAAO,QAAQ;oBACvB;iBACD;YACH;YACA,MAAME,cAAcH,QAAAA,IAAS,CAACE;YAC9B,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gBACE,MAAM;gBACN,QAAQT;gBACR,cAAcU;YAChB,GACAN;QAEJ;QAEA,MAAM,IAAI,CAAC,gBAAgB,CAACnC;QAC5B,OAAOuC;IACT;IAEA,MAAM,QACJG,MAA2B,EAC3BnC,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAE8D,MAAM,EAAEvC,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC0C,QAAQnC;QACnE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOuC;IACT;IAEA,MAAM,UACJzD,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAE8D,MAAM,EAAEvC,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAClB,QAAQyB;QACrE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOuC;IACT;IAEA,MAAM,SACJzD,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAE8D,MAAM,EAAEvC,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAClB,QAAQyB;QACpE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOuC;IACT;IAEA,MAAM,SACJzD,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,MAAM,EAAE8D,MAAM,EAAEvC,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAClB,QAAQyB;QACpE,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAC5B,OAAOuC;IACT;IAEA,MAAM,MACJzD,MAAmB,EACnByB,MAA4B9B,2BAA2B,EACvD;QACA,OAAO,IAAI,CAAC,QAAQ,CAACK,QAAQyB;IAC/B;IAEA,MAAM,uBACJoC,MAAwB,EACxBpC,GAI0B,EACkB;QAC5C,MAAM,EAAEqC,eAAe,IAAI,EAAEC,aAAa,CAAC,EAAE,GAAGtC,OAAO,CAAC;QAExD,IAAIuC,UAAU;QACd,IAAIC,aAAa;QACjB,IAAIC,eAAe;QACnB,IAAIvC,YAAYF,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,AAAD,KAAK;QAClC,IAAI0C;QAEJ,MAAO,CAACH,WAAWC,aAAaF,WAAY;YAC1C,IAAIE,cAAc,GAChBtC,YAAY;YAEdnD,MACE,cACAqF,QACA,gBACAC,cACA,cACAG,YACA,aACAtC;YAEF,MAAMyC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACP,QAAQ;gBAAElC;YAAU;YAC7DnD,MAAM,mBAAmB4F;YACzB1C,OAAO0C,KAAK,WAAW,EAAE,CAAC,+BAA+B,EAAEP,OAAO,CAAC,CAAC;YACpEK,eAAeE,KAAK,WAAW;YAE/BD,eAAe,MAAM,IAAI,CAAC,aAAa,CACrCD,cACAvC,YAAY;gBAAE,WAAW;YAAK,IAAIY,QAClCsB,QACApC;YAEF,IAAI0C,aAAa,IAAI,EACnBH,UAAU;iBAEVC;QAEJ;QAEA,OAAO;YACL,QAAQC;YACRvC;YACAwC;QACF;IACF;IAEA,MAAM,cACJnE,MAAc,EACdqE,SAAmC,EACnCC,YAA8B,EAC9BC,kBAA2C,EACX;QAChC/F,MAAM,iBAAiBwB,QAAQqE,WAAWC,cAAcC;QAExD,MAAM,EAAE,QAAQC,YAAY,EAAE,MAAMC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CACpEzE,QACAqE;QAEF,MAAMK,WAAWhG,oBAAoB4F,cAAcE;QACnD,MAAMG,WAAWzF,eAAeoF,cAAcG;QAC9C,MAAMG,OACJF,YAAaH,CAAAA,CAAAA,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,uBAAuB,AAAD,KAAK,EAAC,KAC7DI;QACF,MAAMR,eAAe;YACnBS;YACA,MAAMH;YACN,QAAQD;YACR,gBAAgBE;QAClB;QACAlG,MAAM,2BAA2B2F;QACjC,OAAOA;IACT;IAEA,MAAM,SAASnE,MAAmB,EAAEyB,GAAkB,EAAE;QACtD,MAAMM,aAAa,IAAI,CAAC,mBAAmB,CAAC/B,QAAQyB;QACpD,MAAMS,QAAQ;YAACH;SAAW;QAC1B,MAAM,EAAEb,QAAQ,EAAEuC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC3DpB,aAAa,UAAUC,eAAeP,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,MAAM,AAAD,KAAKQ,UAC5DL,OACA;YAAE,WAAWT,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS;QAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAACP;QAE5B,MAAM,EAAE2D,OAAO,EAAE,GAAGpB;QAEpB,OAAO;YACL,MAAMoB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,IAAI;YACnB,QAAQA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM;YACvB,OAAQ,OAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAC,EAAG,GAAG;QACrC;IAGF;IAEA,MAAM,SAASC,SAAsB,EAAEC,GAAY,EAAEtD,GAAoB,EAAE;QACzE,MAAM,EAAEgC,MAAM,EAAEvC,QAAQ,EAAE8D,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAClEF,WACA;YACE,eAAe;QACjB;QAEF,MAAM,IAAI,CAAC,gBAAgB,CAAC5D,UAAU;QAEtC,IAAIO,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,EACtB,OAAO;YACL,MAAMgC;YACNuB;QACF;QAGF,IAAI,CAACvB,QAAQ;gBAGEwB;YAFb,MAAMC,SAASH,OAAO,CAAC,kBAAkB,EAAED,WAAW;YACtD,MAAMK,YAAY,CAAC,QAAQ,EACzBH,WAAAA,SAAWC,CAAAA,4BAAAA,SAAS,eAAe,EAAC,IAAzBA,KAAAA,IAAAA,0BAA4B,KAAK,AAAD,KAAK,eAChD;YACF,MAAM,IAAI1E,MAAM,GAAG2E,OAAO,EAAE,EAAEC,WAAW;QAC3C;IACF;IAEA,MAAM,UAAUL,SAAiB,EAAErD,GAAqB,EAAE;QACxD,MAAM,EAAEP,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC4D,WAAW;YAC9D,WAAWrD,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,AAAD,KAAK;YAC7B,iBAAiBA,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,KAAK;YACzCqD;QACF;QACA,MAAM,IAAI,CAAC,gBAAgB,CAAC5D,UAAU;QAEtC,IAAIA,SAAS,cAAc,IAAI;YAC7B,MAAMK,YAAYL,SAAS,eAAe;YAC1C,MAAM,IAAIX,MAAM,GAAGgB,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK,CAAC,EAAE,EAAEA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,UAAU,EAAE;QACjE;IACF;IAEA,MAAM,GAAG0B,UAAkB,EAAEjB,OAAO,QAAQ,EAAE;QAC5C,IAAIA,AAAS,aAATA,MACF,OAAO,IAAI,CAAC,QAAQ,CAACiB;QAEvB,IAAIjB,AAAS,YAATA,MACF,OAAO,IAAI,CAAC,OAAO,CAACiB;QAGtB,IAAIjB,AAAS,aAATA,MACF,OAAO,IAAI,CAAC,QAAQ,CAACiB;QAGvB,IAAIjB,AAAS,UAATA,MACF,OAAO,IAAI,CAAC,KAAK,CAACiB;QAGpB,IAAIjB,AAAS,iBAATA,MACF,OAAO,IAAI,CAAC,YAAY,CAACiB;QAG3B,MAAM,IAAI1C,MACR,CAAC,cAAc,EAAEyB,KAAK,+DAA+D,CAAC;IAE1F;IAEA,MAAM,QAAQoD,iBAAyB,EAEpC;QACD,MAAMC,SAASC,gBAAgBF,mBAAmB,QAAQ;QACtC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW;QAC/C,MAAMG,SAAS,IAAIC,aAAaH,QAAQ,OAAOI,SACtC;gBAAE,OAAO,IAAI;gBAAE,QAAQ,EAAE;YAAC;QAEnC,MAAMF,OAAO,GAAG;QAEhB,IAAIA,AAAkB,YAAlBA,OAAO,MAAM,EAAc;YAC7B,MAAMG,SAASH,OAAO,cAAc,CACjC,MAAM,CAAC,CAAC1E,OAASA,AAAgB,YAAhBA,KAAK,MAAM,EAC5B,GAAG,CAAC,CAACA;oBAC2B8E;gBAA/B,OAAO,CAAC,OAAO,EAAE9E,KAAK,IAAI,CAAC,EAAE,EAAE,QAAA8E,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,EAAE;YACtD,GACC,IAAI,CAAC;YACR,MAAM,IAAIpF,MAAM,CAAC,2CAA2C,EAAEmF,QAAQ;QACxE;QAEA,OAAO;YACL,QAAQH,OAAO,MAAM;QACvB;IACF;IAEA,MAAM,mBAAmBF,MAAc,EAAE;QACvC3D,OACE,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B;QAEF,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC2D;IACtC;IAEA,MAAM,UAAU;QACd,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO;QACvB,IAAI,CAAC,SAAS;QACd,IAAI,CAAC,SAAS,GAAG;IACnB;IAEA,MAAM,cACJjD,KAAc,EACdX,GAEC,EACD;QAEA,MAAMmE,SAAS,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAC/C,MAAMC,MAAMC,KAAK,GAAG;QAEpB,MAAMC,WAAoC;YACxC;gBACE,MAAM;gBACN,IAAIF;gBACJ,YAAYD;YACd;SACD;QAED,MAAM/E,OAAyB;YAC7B,MAAM;YACN,SAAS;YACT,QAAQ;YACRkF;YACA,QAAQ;gBACN,OAAOF;gBACP,KAAKA;gBACL,MAAM;YACR;YACA,OAAO;gBACL,SAASpE,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAK;YAC3B;YACA,UAAU,WAAa;QACzB;QAEA,MAAMuE,gBAA+B;YACnC,YAAY;YACZ,SAASH;YACT,YAAY;YACZ,mBAAmB;YACnB,MAAM,CAAC,MAAM,EAAEzD,SAAS,YAAY;YACpC,aAAaX,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAK;YAC7B,OAAO;gBAACZ;aAAK;QACf;QAEA,IAAI,CAAC,mBAAmB,CAACmF;QAEzB,IAAI;gBACF5E,oBAAAA;oBAAAA,CAAAA,qBAAAA,AAAAA,CAAAA,QAAAA,IAAI,AAAD,EAAE,YAAY,AAAD,KAAhBA,mBAAAA,IAAAA,CAAAA,OAAoB,IAAI,CAAC,cAAc;QACzC,EAAE,OAAOC,OAAO;YACdC,QAAQ,KAAK,CAAC,yBAAyBD;QACzC;QAEA,IAAI,CAAC,mBAAmB;IAC1B;IAEA,sBAAsB;QACpB,MAAM,EAAE4E,SAAS,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAG,IAAI,CAAC,IAAI;QAC7D,MAAMC,gBAAgBC,MAAM,OAAO,CAACF,cAChCA,WAAW,GAAG,CAAC,CAAClG;YACd,MAAM,EAAEqG,KAAK,EAAE,GAAGC,eAAe,GAAGtG;YACpC,IAAIuG,WAAWF;YACf,IAAID,MAAM,OAAO,CAACC,QAChBE,WAAWF,MAAM,GAAG,CAAC,CAACzF;gBAEpB,MAAM,EAAE4F,WAAW,EAAEC,GAAG,EAAE,GAAGC,UAAU,GAAG9F;gBAC1C,OAAO8F;YACT;YAEF,OAAO;gBAAE,GAAGJ,aAAa;gBAAE,GAAIC,WAAW;oBAAE,OAAOA;gBAAS,IAAI,CAAC,CAAC;YAAE;QACtE,KACA,EAAE;QACN,OAAO;YACLP;YACAC;YACA,YAAYE;QACd;IACF;IAMA,MAAM,oBAAmC;QACvC5H,MAAM;QACN,MAAMoI,UAAU,MAAM,IAAI,CAAC,gBAAgB;QAE3CA,QAAQ,SAAS,GAAG;QACpB,IAAI,CAAC,iBAAiB,GAAGA;QACzBpI,MAAM;IACR;IAKA,MAAM,sBAAqC;QACzCA,MAAM;QACN,IAAI,CAAC,iBAAiB,GAAG+D;QACzB/D,MAAM;IACR;IA1yBA,YAAYqI,IAAc,EAAEC,IAAmB,CAAE;QAhCjD;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA;QAKA,kCAAU;QAEV;QAEA;QAEA;QAEA,oCAAY;QAKZ,uBAAQ,qBAAR;QAGE,IAAI,CAAC,IAAI,GAAGD;QACZ,IAAI,CAAC,IAAI,GAAGE,OAAO,MAAM,CACvB;YACE,gBAAgB;YAChB,oBAAoB;YACpB,WAAW;YACX,kBAAkB;QACpB,GACAD,QAAQ,CAAC;QAGX,IACE,AAAuB,gBAAvB,IAAI,CAAC,IAAI,CAAC,QAAQ,IAClB,AAAuB,iBAAvB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAClB;YAEE,IAAI,CAAC,IAAI,CACT,wBAAwB,GACvB,IAAI,CAAC,IAAI,CAAqB,wBAAwB,IACvDE;YAEA,IAAI,CAAC,IAAI,CACT,yBAAyB,GACxB,IAAI,CAAC,IAAI,CAAqB,yBAAyB,IACxDC;QACJ;QAEA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc;QAI9C,IAAI,CAAC,OAAO,GAAG,IAAIC,QACjB,OAAOrH,SACE,IAAI,CAAC,YAAY,CAACA;QAI7B,IAAIiH,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAK,AAAuB,cAAvB,IAAI,CAAC,IAAI,CAAC,QAAQ,EACrC,IAAI,CAAC,SAAS,GAAG,IAAIK,UACnBL,KAAK,OAAO,EACZM,qBAAqB;QAIzB,IAAI,CAAC,YAAY,GAAG,IAAIC,iBAAiB,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE;YAChE,WAAW,IAAI,CAAC,SAAS;YACzB,aAAa,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI;QACpD;QACA,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS;QAC1B,IAAI,CAAC,cAAc,GACjBP,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,cAAc,AAAD,KACnBQ,kBAAkBR,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI;IAC5D;AAsvBF"}

package/dist/es/common/tasks.mjs

@@ -412,14 +412,14 @@ class PageTaskExecutor {
                             param: null,
                             thought: planningAction.locate.prompt
                         });
-                    } else if ([
-                        'Tap',
-                        'Hover',
-                        'Input'
-                    ].includes(planningAction.type)) {
-                        planParsingError = `invalid planning response: ${JSON.stringify(planningAction)}`;
-                        stopCollecting = true;
-                        return acc;
+                    } else {
+                        const actionType = planningAction.type;
+                        const action = actionSpace.find((action)=>action.name === actionType);
+                        if (action && (null == action ? void 0 : action.location) === 'required') {
+                            planParsingError = `Location is required for action: ${actionType}, but not provided by planning`;
+                            stopCollecting = true;
+                            return acc;
+                        }
                     }
                     acc.push(planningAction);
                     return acc;