@midscene/core 1.2.2-beta-20260116092125.0 → 1.2.2-beta-20260116114131.0

package/dist/es/task-runner.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"task-runner.mjs","sources":["../../src/task-runner.ts"],"sourcesContent":["import type { ScreenshotItem } from '@/screenshot-item';\nimport {\n  ExecutionDump,\n  type ExecutionRecorderItem,\n  type ExecutionTask,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskPlanningLocateOutput,\n  type ExecutionTaskProgressOptions,\n  type ExecutionTaskReturn,\n  type ExecutorContext,\n  type PlanningActionParamError,\n  type UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\ntype TaskRunnerInitOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n};\n\ntype TaskRunnerOperationOptions = {\n  allowWhenError?: boolean;\n};\n\nexport class TaskRunner {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  private readonly onTaskUpdate?:\n    | ((runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void)\n    | undefined;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: TaskRunnerInitOptions,\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n    this.onTaskUpdate = options?.onTaskUpdate;\n  }\n\n  private async emitOnTaskUpdate(error?: TaskExecutionError): Promise<void> {\n    if (!this.onTaskUpdate) {\n      return;\n    }\n    await this.onTaskUpdate(this, error);\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<ScreenshotItem | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshot;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    screenshot: ScreenshotItem | undefined,\n    phase: 'after-calling',\n  ): void {\n    if (!phase || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing: phase,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private normalizeStatusFromError(\n    options?: TaskRunnerOperationOptions,\n    errorMessage?: string,\n  ): void {\n    if (this.status !== 'error') {\n      return;\n    }\n    assert(\n      options?.allowWhenError,\n      errorMessage ||\n        `task runner is in error state, cannot proceed\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    // reset runner state so new tasks can run\n    this.status = this.tasks.length > 0 ? 'pending' : 'init';\n  }\n\n  private findPreviousNonSubTaskUIContext(\n    currentIndex: number,\n  ): UIContext | undefined {\n    for (let i = currentIndex - 1; i >= 0; i--) {\n      const candidate = this.tasks[i];\n      if (!candidate || candidate.subTask) {\n        continue;\n      }\n      if (candidate.uiContext) {\n        return candidate.uiContext;\n      }\n    }\n    return undefined;\n  }\n\n  async append(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<void> {\n    this.normalizeStatusFromError(\n      options,\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n    await this.emitOnTaskUpdate();\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task, options);\n    return this.flush(options);\n  }\n\n  async flush(\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    this.normalizeStatusFromError(options, 'task runner is in error state');\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    await this.emitOnTaskUpdate();\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskPlanningLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        await this.emitOnTaskUpdate();\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action Space', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        let uiContext: UIContext | undefined;\n        if (task.subTask) {\n          uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);\n          assert(\n            uiContext,\n            'subTask requires uiContext from previous non-subTask task',\n          );\n        } else {\n          uiContext = await this.getUiContext();\n        }\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        } else if (task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskPlanningLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action Space') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n    if (taskIndex + 1 < this.tasks.length) {\n      await this.emitOnTaskUpdate();\n    }\n\n    let finalizeError: TaskExecutionError | undefined;\n    if (!successfullyCompleted) {\n      this.status = 'error';\n      const errorTask = this.latestErrorTask();\n      const messageBase =\n        errorTask?.errorMessage ||\n        (errorTask?.error ? String(errorTask.error) : 'Task execution failed');\n      const stack = errorTask?.errorStack;\n      const message = stack ? `${messageBase}\\n${stack}` : messageBase;\n      finalizeError = new TaskExecutionError(message, this, errorTask, {\n        cause: errorTask?.error,\n      });\n      await this.emitOnTaskUpdate(finalizeError);\n    } else {\n      this.status = 'completed';\n      await this.emitOnTaskUpdate();\n    }\n\n    if (finalizeError) {\n      throw finalizeError;\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    return new ExecutionDump({\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    });\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action Space',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n\nexport class TaskExecutionError extends Error {\n  runner: TaskRunner;\n\n  errorTask: ExecutionTask | null;\n\n  constructor(\n    message: string,\n    runner: TaskRunner,\n    errorTask: ExecutionTask | null,\n    options?: { cause?: unknown },\n  ) {\n    super(message, options);\n    this.runner = runner;\n    this.errorTask = errorTask;\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","error","options","now","Date","shouldReuse","uiContext","undefined","console","task","screenshot","phase","recorderItem","errorMessage","assert","currentIndex","i","candidate","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","executorContext","isLastTask","Object","finalizeError","errorTask","messageBase","String","stack","message","TaskExecutionError","outputIndex","Math","thought","output","errorTaskIndex","ExecutionDump","errorMsg","Error","name","uiContextBuilder","runner"],"mappings":";;;;;;;;;;;;;AAiBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAczB,MAAMC;IAgCX,MAAc,iBAAiBC,KAA0B,EAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAEA;IAChC;IAOA,MAAc,aAAaC,OAAoC,EAE7D;QACA,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACH,SAAS,gBACV,IAAI,CAAC,aAAa,IAClBC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIJ;QAEzC,IAAIM,eAAe,IAAI,CAAC,aAAa,EAAE,SAAS;YAC9CR,MACE,CAAC,gCAAgC,EAAEM,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,IAAI,CAAC,aAAa,EAAE;QAC7B;QAEA,IAAI;YACF,MAAMG,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYF,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGG;YAEvB,OAAOD;QACT,EAAE,OAAOL,OAAO;YACd,IAAI,CAAC,aAAa,GAAGM;YACrB,MAAMN;QACR;IACF;IAEA,MAAc,oBAAyD;QACrE,IAAI;YACF,MAAMK,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,WAAW;QACpB,EAAE,OAAOL,OAAO;YACdO,QAAQ,KAAK,CAAC,oCAAoCP;QACpD;IAEF;IAEQ,mBACNQ,IAAmB,EACnBC,UAAsC,EACtCC,KAAsB,EAChB;QACN,IAAI,CAACA,SAAS,CAACD,YACb;QAGF,MAAME,eAAsC;YAC1C,MAAM;YACN,IAAIR,KAAK,GAAG;YACZM;YACA,QAAQC;QACV;QAEA,IAAI,CAACF,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACG;aAAa;YAC9B;QACF;QACAH,KAAK,QAAQ,CAAC,IAAI,CAACG;IACrB;IAEQ,kBAAkBH,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEQ,yBACNP,OAAoC,EACpCW,YAAqB,EACf;QACN,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb;QAEFC,OACEZ,SAAS,gBACTW,gBACE,CAAC,qDAAqD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAGlI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;IACpD;IAEQ,gCACNE,YAAoB,EACG;QACvB,IAAK,IAAIC,IAAID,eAAe,GAAGC,KAAK,GAAGA,IAAK;YAC1C,MAAMC,YAAY,IAAI,CAAC,KAAK,CAACD,EAAE;YAC/B,IAAI,AAACC,cAAaA,UAAU,OAAO,EAGnC;gBAAA,IAAIA,UAAU,SAAS,EACrB,OAAOA,UAAU,SAAS;YAC5B;QACF;IAEF;IAEA,MAAM,OACJR,IAA+C,EAC/CP,OAAoC,EACrB;QACf,IAAI,CAAC,wBAAwB,CAC3BA,SACA,CAAC,yDAAyD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAEpI,IAAIgB,MAAM,OAAO,CAACT,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACU,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACV;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;QAEhB,MAAM,IAAI,CAAC,gBAAgB;IAC7B;IAEA,MAAM,eACJA,IAA+C,EAC/CP,OAAoC,EACoB;QACxD,MAAM,IAAI,CAAC,MAAM,CAACO,MAAMP;QACxB,OAAO,IAAI,CAAC,KAAK,CAACA;IACpB;IAEA,MAAM,MACJA,OAAoC,EACoB;QACxD,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDM,QAAQ,IAAI,CACV;QAIJ,IAAI,CAAC,wBAAwB,CAACN,SAAS;QACvCY,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QAEpC,MAAMM,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACX,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIW,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,MAAM,IAAI,CAAC,gBAAgB;QAC3B,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMZ,OAAO,IAAI,CAAC,KAAK,CAACY,UAAU;YAClCP,OACEL,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOL,KAAK,GAAG;YACjB;YACA,IAAI;gBACFK,KAAK,MAAM,GAAG;gBACd,MAAM,IAAI,CAAC,gBAAgB;gBAC3B,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOe,GAAG;oBACVhB,QAAQ,KAAK,CAAC,wBAAwBgB;gBACxC;gBACAV,OACE;oBAAC;oBAAW;oBAAgB;iBAAW,CAAC,OAAO,CAACL,KAAK,IAAI,KAAK,GAC9D,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEgB,QAAQ,EAAEC,KAAK,EAAE,GAAGjB;gBAC5BK,OAAOW,UAAU,CAAC,oCAAoC,EAAEhB,KAAK,IAAI,EAAE;gBAEnE,IAAIkB;gBACJ,IAAIrB;gBACJ,IAAIG,KAAK,OAAO,EAAE;oBAChBH,YAAY,IAAI,CAAC,+BAA+B,CAACe;oBACjDP,OACER,WACA;gBAEJ,OACEA,YAAY,MAAM,IAAI,CAAC,YAAY;gBAErCG,KAAK,SAAS,GAAGH;gBACjB,MAAMsB,kBAAmC;oBACvCnB;oBACA,SAASc,oBAAoB;oBAC7BjB;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BK,OACEL,AAAiB,YAAjBA,KAAK,OAAO,IACVA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;gBAC3C,OAAO,IAAInB,AAAc,eAAdA,KAAK,IAAI,EAAiB;oBACnCkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;oBACzC,IAAInB,AAAiB,aAAjBA,KAAK,OAAO,EACdc,qBACEI,aACC;gBAEP,OAAO,IAAIlB,AAAc,mBAAdA,KAAK,IAAI,EAClBkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;qBACpC;oBACLpB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;gBAC3C;gBAEA,MAAMC,aAAaR,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIQ,YAAY;oBACd,MAAMnB,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACD,MAAMC,YAAY;gBAC5C;gBAEAoB,OAAO,MAAM,CAACrB,MAAMkB;gBACpBlB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3BY;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBb,KAAK,KAAK,GAAGe;gBACbf,KAAK,YAAY,GACfe,GAAG,WAAY,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEf,KAAK,UAAU,GAAGe,EAAE,KAAK;gBAEzBf,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3B;YACF;QACF;QAGA,IAAK,IAAIO,IAAIK,YAAY,GAAGL,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAEzB,IAAIK,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EACnC,MAAM,IAAI,CAAC,gBAAgB;QAG7B,IAAIU;QACJ,IAAKT,uBAYE;YACL,IAAI,CAAC,MAAM,GAAG;YACd,MAAM,IAAI,CAAC,gBAAgB;QAC7B,OAf4B;YAC1B,IAAI,CAAC,MAAM,GAAG;YACd,MAAMU,YAAY,IAAI,CAAC,eAAe;YACtC,MAAMC,cACJD,WAAW,gBACVA,CAAAA,WAAW,QAAQE,OAAOF,UAAU,KAAK,IAAI,uBAAsB;YACtE,MAAMG,QAAQH,WAAW;YACzB,MAAMI,UAAUD,QAAQ,GAAGF,YAAY,EAAE,EAAEE,OAAO,GAAGF;YACrDF,gBAAgB,IAAIM,mBAAmBD,SAAS,IAAI,EAAEJ,WAAW;gBAC/D,OAAOA,WAAW;YACpB;YACA,MAAM,IAAI,CAAC,gBAAgB,CAACD;QAC9B;QAKA,IAAIA,eACF,MAAMA;QAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMO,cAAcC,KAAK,GAAG,CAAClB,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEmB,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAACjC,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAIiC,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,OAAO,IAAIC,cAAc;YACvB,SAASvC,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;IACF;IAEA,MAAM,gBAAgBwC,QAAgB,EAGnC;QACD,MAAMZ,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASY;YACX;YACA,SAASA;YACT,UAAU;gBACR,MAAM,IAAIC,MAAMD,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACZ;QAE1B,OAAO;YACL,QAAQzB;YACR,QAAQ,IAAI;QACd;IACF;IAtXA,YACEuC,IAAY,EACZC,gBAA0C,EAC1C7C,OAA+B,CAC/B;QAnBF;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAEA,uBAAiB,gBAAjB;QA2BA,uBAAQ,iBAAR;QAlBE,IAAI,CAAC,MAAM,GACTA,SAAS,SAASA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAG4C;QACZ,IAAI,CAAC,KAAK,GAAI5C,AAAAA,CAAAA,SAAS,SAAS,EAAC,EAAG,GAAG,CAAC,CAACiB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGjB,SAAS;QAC5B,IAAI,CAAC,gBAAgB,GAAG6C;QACxB,IAAI,CAAC,YAAY,GAAG7C,SAAS;IAC/B;AAyWF;AAEO,MAAMmC,2BAA2BQ;IAKtC,YACET,OAAe,EACfY,MAAkB,EAClBhB,SAA+B,EAC/B9B,OAA6B,CAC7B;QACA,KAAK,CAACkC,SAASlC,UAVjB,0CAEA;QASE,IAAI,CAAC,MAAM,GAAG8C;QACd,IAAI,CAAC,SAAS,GAAGhB;IACnB;AACF"}
+{"version":3,"file":"task-runner.mjs","sources":["../../src/task-runner.ts"],"sourcesContent":["import type { ScreenshotItem } from '@/screenshot-item';\nimport {\n  ExecutionDump,\n  type ExecutionRecorderItem,\n  type ExecutionTask,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskPlanningLocateOutput,\n  type ExecutionTaskProgressOptions,\n  type ExecutionTaskReturn,\n  type ExecutorContext,\n  type PlanningActionParamError,\n  type UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\ntype TaskRunnerInitOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n};\n\ntype TaskRunnerOperationOptions = {\n  allowWhenError?: boolean;\n};\n\nexport class TaskRunner {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  private readonly onTaskUpdate?:\n    | ((runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void)\n    | undefined;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: TaskRunnerInitOptions,\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n    this.onTaskUpdate = options?.onTaskUpdate;\n  }\n\n  private async emitOnTaskUpdate(error?: TaskExecutionError): Promise<void> {\n    if (!this.onTaskUpdate) {\n      return;\n    }\n    await this.onTaskUpdate(this, error);\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<ScreenshotItem | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshot;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    screenshot: ScreenshotItem | undefined,\n    phase: 'after-calling',\n  ): void {\n    if (!phase || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing: phase,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private normalizeStatusFromError(\n    options?: TaskRunnerOperationOptions,\n    errorMessage?: string,\n  ): void {\n    if (this.status !== 'error') {\n      return;\n    }\n    assert(\n      options?.allowWhenError,\n      errorMessage ||\n        `task runner is in error state, cannot proceed\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    // reset runner state so new tasks can run\n    this.status = this.tasks.length > 0 ? 'pending' : 'init';\n  }\n\n  private findPreviousNonSubTaskUIContext(\n    currentIndex: number,\n  ): UIContext | undefined {\n    for (let i = currentIndex - 1; i >= 0; i--) {\n      const candidate = this.tasks[i];\n      if (!candidate || candidate.subTask) {\n        continue;\n      }\n      if (candidate.uiContext) {\n        return candidate.uiContext;\n      }\n    }\n    return undefined;\n  }\n\n  async append(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<void> {\n    this.normalizeStatusFromError(\n      options,\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n    await this.emitOnTaskUpdate();\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task, options);\n    return this.flush(options);\n  }\n\n  async flush(\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    this.normalizeStatusFromError(options, 'task runner is in error state');\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    await this.emitOnTaskUpdate();\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskPlanningLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        await this.emitOnTaskUpdate();\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action Space', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        let uiContext: UIContext | undefined;\n        if (task.subTask) {\n          uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);\n          assert(\n            uiContext,\n            'subTask requires uiContext from previous non-subTask task',\n          );\n        } else {\n          uiContext = await this.getUiContext();\n        }\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        } else if (task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskPlanningLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action Space') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n    if (taskIndex + 1 < this.tasks.length) {\n      await this.emitOnTaskUpdate();\n    }\n\n    let finalizeError: TaskExecutionError | undefined;\n    if (!successfullyCompleted) {\n      this.status = 'error';\n      const errorTask = this.latestErrorTask();\n      const messageBase =\n        errorTask?.errorMessage ||\n        (errorTask?.error ? String(errorTask.error) : 'Task execution failed');\n      const stack = errorTask?.errorStack;\n      const message = stack ? `${messageBase}\\n${stack}` : messageBase;\n      finalizeError = new TaskExecutionError(message, this, errorTask, {\n        cause: errorTask?.error,\n      });\n      await this.emitOnTaskUpdate(finalizeError);\n    } else {\n      this.status = 'completed';\n      await this.emitOnTaskUpdate();\n    }\n\n    if (finalizeError) {\n      throw finalizeError;\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    // Find the LAST failed task (not the first one)\n    // This is important when using allowWhenError to continue after errors\n    for (let i = this.tasks.length - 1; i >= 0; i--) {\n      if (this.tasks[i].status === 'failed') {\n        return this.tasks[i];\n      }\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    return new ExecutionDump({\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    });\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action Space',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n\nexport class TaskExecutionError extends Error {\n  runner: TaskRunner;\n\n  errorTask: ExecutionTask | null;\n\n  constructor(\n    message: string,\n    runner: TaskRunner,\n    errorTask: ExecutionTask | null,\n    options?: { cause?: unknown },\n  ) {\n    super(message, options);\n    this.runner = runner;\n    this.errorTask = errorTask;\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","error","options","now","Date","shouldReuse","uiContext","undefined","console","task","screenshot","phase","recorderItem","errorMessage","assert","currentIndex","i","candidate","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","executorContext","isLastTask","Object","finalizeError","errorTask","messageBase","String","stack","message","TaskExecutionError","outputIndex","Math","thought","output","ExecutionDump","errorMsg","Error","name","uiContextBuilder","runner"],"mappings":";;;;;;;;;;;;;AAiBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAczB,MAAMC;IAgCX,MAAc,iBAAiBC,KAA0B,EAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAEA;IAChC;IAOA,MAAc,aAAaC,OAAoC,EAE7D;QACA,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACH,SAAS,gBACV,IAAI,CAAC,aAAa,IAClBC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIJ;QAEzC,IAAIM,eAAe,IAAI,CAAC,aAAa,EAAE,SAAS;YAC9CR,MACE,CAAC,gCAAgC,EAAEM,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,IAAI,CAAC,aAAa,EAAE;QAC7B;QAEA,IAAI;YACF,MAAMG,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYF,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGG;YAEvB,OAAOD;QACT,EAAE,OAAOL,OAAO;YACd,IAAI,CAAC,aAAa,GAAGM;YACrB,MAAMN;QACR;IACF;IAEA,MAAc,oBAAyD;QACrE,IAAI;YACF,MAAMK,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,WAAW;QACpB,EAAE,OAAOL,OAAO;YACdO,QAAQ,KAAK,CAAC,oCAAoCP;QACpD;IAEF;IAEQ,mBACNQ,IAAmB,EACnBC,UAAsC,EACtCC,KAAsB,EAChB;QACN,IAAI,CAACA,SAAS,CAACD,YACb;QAGF,MAAME,eAAsC;YAC1C,MAAM;YACN,IAAIR,KAAK,GAAG;YACZM;YACA,QAAQC;QACV;QAEA,IAAI,CAACF,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACG;aAAa;YAC9B;QACF;QACAH,KAAK,QAAQ,CAAC,IAAI,CAACG;IACrB;IAEQ,kBAAkBH,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEQ,yBACNP,OAAoC,EACpCW,YAAqB,EACf;QACN,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb;QAEFC,OACEZ,SAAS,gBACTW,gBACE,CAAC,qDAAqD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAGlI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;IACpD;IAEQ,gCACNE,YAAoB,EACG;QACvB,IAAK,IAAIC,IAAID,eAAe,GAAGC,KAAK,GAAGA,IAAK;YAC1C,MAAMC,YAAY,IAAI,CAAC,KAAK,CAACD,EAAE;YAC/B,IAAI,AAACC,cAAaA,UAAU,OAAO,EAGnC;gBAAA,IAAIA,UAAU,SAAS,EACrB,OAAOA,UAAU,SAAS;YAC5B;QACF;IAEF;IAEA,MAAM,OACJR,IAA+C,EAC/CP,OAAoC,EACrB;QACf,IAAI,CAAC,wBAAwB,CAC3BA,SACA,CAAC,yDAAyD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAEpI,IAAIgB,MAAM,OAAO,CAACT,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACU,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACV;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;QAEhB,MAAM,IAAI,CAAC,gBAAgB;IAC7B;IAEA,MAAM,eACJA,IAA+C,EAC/CP,OAAoC,EACoB;QACxD,MAAM,IAAI,CAAC,MAAM,CAACO,MAAMP;QACxB,OAAO,IAAI,CAAC,KAAK,CAACA;IACpB;IAEA,MAAM,MACJA,OAAoC,EACoB;QACxD,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDM,QAAQ,IAAI,CACV;QAIJ,IAAI,CAAC,wBAAwB,CAACN,SAAS;QACvCY,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QAEpC,MAAMM,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACX,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIW,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,MAAM,IAAI,CAAC,gBAAgB;QAC3B,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMZ,OAAO,IAAI,CAAC,KAAK,CAACY,UAAU;YAClCP,OACEL,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOL,KAAK,GAAG;YACjB;YACA,IAAI;gBACFK,KAAK,MAAM,GAAG;gBACd,MAAM,IAAI,CAAC,gBAAgB;gBAC3B,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOe,GAAG;oBACVhB,QAAQ,KAAK,CAAC,wBAAwBgB;gBACxC;gBACAV,OACE;oBAAC;oBAAW;oBAAgB;iBAAW,CAAC,OAAO,CAACL,KAAK,IAAI,KAAK,GAC9D,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEgB,QAAQ,EAAEC,KAAK,EAAE,GAAGjB;gBAC5BK,OAAOW,UAAU,CAAC,oCAAoC,EAAEhB,KAAK,IAAI,EAAE;gBAEnE,IAAIkB;gBACJ,IAAIrB;gBACJ,IAAIG,KAAK,OAAO,EAAE;oBAChBH,YAAY,IAAI,CAAC,+BAA+B,CAACe;oBACjDP,OACER,WACA;gBAEJ,OACEA,YAAY,MAAM,IAAI,CAAC,YAAY;gBAErCG,KAAK,SAAS,GAAGH;gBACjB,MAAMsB,kBAAmC;oBACvCnB;oBACA,SAASc,oBAAoB;oBAC7BjB;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BK,OACEL,AAAiB,YAAjBA,KAAK,OAAO,IACVA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;gBAC3C,OAAO,IAAInB,AAAc,eAAdA,KAAK,IAAI,EAAiB;oBACnCkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;oBACzC,IAAInB,AAAiB,aAAjBA,KAAK,OAAO,EACdc,qBACEI,aACC;gBAEP,OAAO,IAAIlB,AAAc,mBAAdA,KAAK,IAAI,EAClBkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;qBACpC;oBACLpB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEkB,cAAc,MAAMlB,KAAK,QAAQ,CAACiB,OAAOE;gBAC3C;gBAEA,MAAMC,aAAaR,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIQ,YAAY;oBACd,MAAMnB,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACD,MAAMC,YAAY;gBAC5C;gBAEAoB,OAAO,MAAM,CAACrB,MAAMkB;gBACpBlB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3BY;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBb,KAAK,KAAK,GAAGe;gBACbf,KAAK,YAAY,GACfe,GAAG,WAAY,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEf,KAAK,UAAU,GAAGe,EAAE,KAAK;gBAEzBf,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3B;YACF;QACF;QAGA,IAAK,IAAIO,IAAIK,YAAY,GAAGL,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAEzB,IAAIK,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EACnC,MAAM,IAAI,CAAC,gBAAgB;QAG7B,IAAIU;QACJ,IAAKT,uBAYE;YACL,IAAI,CAAC,MAAM,GAAG;YACd,MAAM,IAAI,CAAC,gBAAgB;QAC7B,OAf4B;YAC1B,IAAI,CAAC,MAAM,GAAG;YACd,MAAMU,YAAY,IAAI,CAAC,eAAe;YACtC,MAAMC,cACJD,WAAW,gBACVA,CAAAA,WAAW,QAAQE,OAAOF,UAAU,KAAK,IAAI,uBAAsB;YACtE,MAAMG,QAAQH,WAAW;YACzB,MAAMI,UAAUD,QAAQ,GAAGF,YAAY,EAAE,EAAEE,OAAO,GAAGF;YACrDF,gBAAgB,IAAIM,mBAAmBD,SAAS,IAAI,EAAEJ,WAAW;gBAC/D,OAAOA,WAAW;YACpB;YACA,MAAM,IAAI,CAAC,gBAAgB,CAACD;QAC9B;QAKA,IAAIA,eACF,MAAMA;QAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMO,cAAcC,KAAK,GAAG,CAAClB,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEmB,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAIT,IAAK,IAAIzB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAGA,KAAK,GAAGA,IAC1C,IAAI,AAAyB,aAAzB,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,EACtB,OAAO,IAAI,CAAC,KAAK,CAACA,EAAE;QAGxB,OAAO;IACT;IAEA,OAAsB;QACpB,OAAO,IAAI0B,cAAc;YACvB,SAAStC,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;IACF;IAEA,MAAM,gBAAgBuC,QAAgB,EAGnC;QACD,MAAMX,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASW;YACX;YACA,SAASA;YACT,UAAU;gBACR,MAAM,IAAIC,MAAMD,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACX;QAE1B,OAAO;YACL,QAAQzB;YACR,QAAQ,IAAI;QACd;IACF;IAvXA,YACEsC,IAAY,EACZC,gBAA0C,EAC1C5C,OAA+B,CAC/B;QAnBF;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAEA,uBAAiB,gBAAjB;QA2BA,uBAAQ,iBAAR;QAlBE,IAAI,CAAC,MAAM,GACTA,SAAS,SAASA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAG2C;QACZ,IAAI,CAAC,KAAK,GAAI3C,AAAAA,CAAAA,SAAS,SAAS,EAAC,EAAG,GAAG,CAAC,CAACiB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGjB,SAAS;QAC5B,IAAI,CAAC,gBAAgB,GAAG4C;QACxB,IAAI,CAAC,YAAY,GAAG5C,SAAS;IAC/B;AA0WF;AAEO,MAAMmC,2BAA2BO;IAKtC,YACER,OAAe,EACfW,MAAkB,EAClBf,SAA+B,EAC/B9B,OAA6B,CAC7B;QACA,KAAK,CAACkC,SAASlC,UAVjB,0CAEA;QASE,IAAI,CAAC,MAAM,GAAG6C;QACd,IAAI,CAAC,SAAS,GAAGf;IACnB;AACF"}

package/dist/es/types.mjs

@@ -10,11 +10,6 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-var types_AIResponseFormat = /*#__PURE__*/ function(AIResponseFormat) {
-    AIResponseFormat["JSON"] = "json_object";
-    AIResponseFormat["TEXT"] = "text";
-    return AIResponseFormat;
-}({});
 class UIContext {
 }
 class ServiceError extends Error {
@@ -100,6 +95,6 @@ class GroupedActionDump {
         this.executions = data.executions.map((exec)=>exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec));
     }
 }
-export { types_AIResponseFormat as AIResponseFormat, ExecutionDump, GroupedActionDump, ServiceError, UIContext };
+export { ExecutionDump, GroupedActionDump, ServiceError, UIContext };
 
 //# sourceMappingURL=types.mjs.map

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshot: ScreenshotItem;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  note?: string;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/**\n * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n */\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  // Handle ScreenshotItem serialization\n  if (value && typeof value.toSerializable === 'function') {\n    return value.toSerializable();\n  }\n  return value;\n}\n\n/**\n * Reviver function for JSON deserialization that restores ScreenshotItem from base64 strings\n * Automatically converts screenshot fields (in uiContext and recorder) from strings back to ScreenshotItem\n */\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n  // Restore screenshot fields in uiContext and recorder\n  if (key === 'screenshot' && ScreenshotItem.isSerializedData(value)) {\n    return ScreenshotItem.fromSerializedData(value);\n  }\n  return value;\n}\n\n/**\n * ExecutionDump class for serializing and deserializing execution dumps\n */\nexport class ExecutionDump implements IExecutionDump {\n  logTime: number;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n\n  constructor(data: IExecutionDump) {\n    this.logTime = data.logTime;\n    this.name = data.name;\n    this.description = data.description;\n    this.tasks = data.tasks;\n    this.aiActContext = data.aiActContext;\n  }\n\n  /**\n   * Serialize the ExecutionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IExecutionDump {\n    return {\n      logTime: this.logTime,\n      name: this.name,\n      description: this.description,\n      tasks: this.tasks,\n      aiActContext: this.aiActContext,\n    };\n  }\n\n  /**\n   * Create an ExecutionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ExecutionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IExecutionDump;\n    return new ExecutionDump(parsed);\n  }\n\n  /**\n   * Create an ExecutionDump instance from a plain object\n   */\n  static fromJSON(data: IExecutionDump): ExecutionDump {\n    return new ExecutionDump(data);\n  }\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nGrouped dump\n*/\nexport interface IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: IExecutionDump[];\n}\n\n/**\n * GroupedActionDump class for serializing and deserializing grouped action dumps\n */\nexport class GroupedActionDump implements IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n\n  constructor(data: IGroupedActionDump) {\n    this.sdkVersion = data.sdkVersion;\n    this.groupName = data.groupName;\n    this.groupDescription = data.groupDescription;\n    this.modelBriefs = data.modelBriefs;\n    this.executions = data.executions.map((exec) =>\n      exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n    );\n  }\n\n  /**\n   * Serialize the GroupedActionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IGroupedActionDump {\n    return {\n      sdkVersion: this.sdkVersion,\n      groupName: this.groupName,\n      groupDescription: this.groupDescription,\n      modelBriefs: this.modelBriefs,\n      executions: this.executions.map((exec) => exec.toJSON()),\n    };\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): GroupedActionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IGroupedActionDump;\n    return new GroupedActionDump(parsed);\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a plain object\n   */\n  static fromJSON(data: IGroupedActionDump): GroupedActionDump {\n    return new GroupedActionDump(data);\n  }\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","serialized","parsed","data","GroupedActionDump","exec"],"mappings":";;AAAqD;;;;;;;;;;AA6C9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AAiEL,MAAeC;AAMtB;AA4EO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AAsMA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAMA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwBC,eAAe,gBAAgB,CAACH,QAC1D,OAAOG,eAAe,kBAAkB,CAACH;IAE3C,OAAOA;AACT;AAKO,MAAMI;IAkBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK;YACjB,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBE,UAAkB,EAAiB;QAC7D,MAAMC,SAASF,KAAK,KAAK,CACvBC,YACAN;QAEF,OAAO,IAAIG,cAAcI;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIL,cAAcK;IAC3B;IA5CA,YAAYA,IAAoB,CAAE;QANlC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAuCF;AAuIO,MAAMC;IAoBX,UAAUL,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAA6B;QAC3B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACM,OAASA,KAAK,MAAM;QACvD;IACF;IAKA,OAAO,qBAAqBJ,UAAkB,EAAqB;QACjE,MAAMC,SAASF,KAAK,KAAK,CACvBC,YACAN;QAEF,OAAO,IAAIS,kBAAkBF;IAC/B;IAKA,OAAO,SAASC,IAAwB,EAAqB;QAC3D,OAAO,IAAIC,kBAAkBD;IAC/B;IA9CA,YAAYA,IAAwB,CAAE;QANtC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACE,OACrCA,gBAAgBP,gBAAgBO,OAAOP,cAAc,QAAQ,CAACO;IAElE;AAuCF"}
+{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshot: ScreenshotItem;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  note?: string;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/**\n * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n */\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  // Handle ScreenshotItem serialization\n  if (value && typeof value.toSerializable === 'function') {\n    return value.toSerializable();\n  }\n  return value;\n}\n\n/**\n * Reviver function for JSON deserialization that restores ScreenshotItem from base64 strings\n * Automatically converts screenshot fields (in uiContext and recorder) from strings back to ScreenshotItem\n */\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n  // Restore screenshot fields in uiContext and recorder\n  if (key === 'screenshot' && ScreenshotItem.isSerializedData(value)) {\n    return ScreenshotItem.fromSerializedData(value);\n  }\n  return value;\n}\n\n/**\n * ExecutionDump class for serializing and deserializing execution dumps\n */\nexport class ExecutionDump implements IExecutionDump {\n  logTime: number;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n\n  constructor(data: IExecutionDump) {\n    this.logTime = data.logTime;\n    this.name = data.name;\n    this.description = data.description;\n    this.tasks = data.tasks;\n    this.aiActContext = data.aiActContext;\n  }\n\n  /**\n   * Serialize the ExecutionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IExecutionDump {\n    return {\n      logTime: this.logTime,\n      name: this.name,\n      description: this.description,\n      tasks: this.tasks,\n      aiActContext: this.aiActContext,\n    };\n  }\n\n  /**\n   * Create an ExecutionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ExecutionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IExecutionDump;\n    return new ExecutionDump(parsed);\n  }\n\n  /**\n   * Create an ExecutionDump instance from a plain object\n   */\n  static fromJSON(data: IExecutionDump): ExecutionDump {\n    return new ExecutionDump(data);\n  }\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nGrouped dump\n*/\nexport interface IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: IExecutionDump[];\n}\n\n/**\n * GroupedActionDump class for serializing and deserializing grouped action dumps\n */\nexport class GroupedActionDump implements IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n\n  constructor(data: IGroupedActionDump) {\n    this.sdkVersion = data.sdkVersion;\n    this.groupName = data.groupName;\n    this.groupDescription = data.groupDescription;\n    this.modelBriefs = data.modelBriefs;\n    this.executions = data.executions.map((exec) =>\n      exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n    );\n  }\n\n  /**\n   * Serialize the GroupedActionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IGroupedActionDump {\n    return {\n      sdkVersion: this.sdkVersion,\n      groupName: this.groupName,\n      groupDescription: this.groupDescription,\n      modelBriefs: this.modelBriefs,\n      executions: this.executions.map((exec) => exec.toJSON()),\n    };\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): GroupedActionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IGroupedActionDump;\n    return new GroupedActionDump(parsed);\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a plain object\n   */\n  static fromJSON(data: IGroupedActionDump): GroupedActionDump {\n    return new GroupedActionDump(data);\n  }\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","serialized","parsed","data","GroupedActionDump","exec"],"mappings":";;AAAqD;;;;;;;;;;AAqG9C,MAAeA;AAMtB;AA4EO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AAsMA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAMA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwBC,eAAe,gBAAgB,CAACH,QAC1D,OAAOG,eAAe,kBAAkB,CAACH;IAE3C,OAAOA;AACT;AAKO,MAAMI;IAkBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK;YACjB,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBE,UAAkB,EAAiB;QAC7D,MAAMC,SAASF,KAAK,KAAK,CACvBC,YACAN;QAEF,OAAO,IAAIG,cAAcI;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIL,cAAcK;IAC3B;IA5CA,YAAYA,IAAoB,CAAE;QANlC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAuCF;AAuIO,MAAMC;IAoBX,UAAUL,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAA6B;QAC3B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACM,OAASA,KAAK,MAAM;QACvD;IACF;IAKA,OAAO,qBAAqBJ,UAAkB,EAAqB;QACjE,MAAMC,SAASF,KAAK,KAAK,CACvBC,YACAN;QAEF,OAAO,IAAIS,kBAAkBF;IAC/B;IAKA,OAAO,SAASC,IAAwB,EAAqB;QAC3D,OAAO,IAAIC,kBAAkBD;IAC/B;IA9CA,YAAYA,IAAwB,CAAE;QANtC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACE,OACrCA,gBAAgBP,gBAAgBO,OAAOP,cAAc,QAAQ,CAACO;IAElE;AAuCF"}