@midscene/core 1.1.1-beta-20260106151323.0 → 1.1.1-beta-20260108085624.0

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type {\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  subTask?: boolean;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  subTask: boolean;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      subTask: !!options?.subTask,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n      [\n        'Sleep',\n        (plan) =>\n          this.handleSleepPlan(\n            plan as PlanningAction<PlanningActionParamSleep>,\n            context,\n          ),\n      ],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      subTask: context.subTask || undefined,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private handleSleepPlan(\n    plan: PlanningAction<PlanningActionParamSleep>,\n    context: PlanBuildContext,\n  ): void {\n    const sleepTask = this.createSleepTask(plan.param, {\n      thought: plan.thought,\n    });\n    if (context.subTask) {\n      sleepTask.subTask = true;\n    }\n    context.tasks.push(sleepTask);\n  }\n\n  public createSleepTask(\n    param: PlanningActionParamSleep,\n    meta?: { thought?: string },\n  ): ExecutionTaskActionApply<PlanningActionParamSleep> {\n    return {\n      type: 'Action Space',\n      subType: 'Sleep',\n      param,\n      thought: meta?.thought,\n      executor: async (taskParam) => {\n        await sleep(taskParam?.timeMs || 3000);\n      },\n    };\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      subTask: context.subTask || undefined,\n      executor: async (param, taskContext) => {\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug('will call \"beforeInvokeAction\" for interface');\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug('called \"beforeInvokeAction\" for interface');\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema);\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        debug('called action', action.name, 'result:', actionResult);\n\n        const delayAfterRunner = action.delayAfterRunner ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug('will call \"afterInvokeAction\" for interface');\n            await this.interface.afterInvokeAction(action.name, param);\n            debug('called \"afterInvokeAction\" for interface');\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent } = context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      subTask: context.subTask || undefined,\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n        const elementFromXpath = rectFromXpath\n          ? generateElementByPosition(\n              {\n                x: rectFromXpath.left + rectFromXpath.width / 2,\n                y: rectFromXpath.top + rectFromXpath.height / 2,\n              },\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCache =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          try {\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n              },\n              modelConfigForDefaultIntent,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached (for bbox/plan hit case, avoid redundant writes)\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          !locateCacheAlreadyExists &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForRect) {\n            try {\n              const feature = await this.interface.cacheFeatureForRect(\n                element.rect,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForRect failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForRect is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element,\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","undefined","sleepTask","meta","taskParam","sleep","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","uiContext","Promise","originalError","originalMessage","String","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByPosition","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","isCacheHit","elementFromAiLocate","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","feature","hitBy","interfaceInstance","service","taskCache"],"mappings":";;;;;;;;;;;;;;;;;AAgCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAsBO,MAAMC;IAqBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,SAAS,CAAC,CAACF,SAAS;QACtB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;YAC9D;gBACE;gBACA,CAACG,OACC,IAAI,CAAC,eAAe,CAClBA,MACAH;aAEL;SACF;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,WAAa;QACzB;QACAP,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEQ,gBACNH,IAA8C,EAC9CH,OAAyB,EACnB;QACN,MAAMQ,YAAY,IAAI,CAAC,eAAe,CAACL,KAAK,KAAK,EAAE;YACjD,SAASA,KAAK,OAAO;QACvB;QACA,IAAIH,QAAQ,OAAO,EACjBQ,UAAU,OAAO,GAAG;QAEtBR,QAAQ,KAAK,CAAC,IAAI,CAACQ;IACrB;IAEO,gBACLlB,KAA+B,EAC/BmB,IAA2B,EACyB;QACpD,OAAO;YACL,MAAM;YACN,SAAS;YACTnB;YACA,SAASmB,MAAM;YACf,UAAU,OAAOC;gBACf,MAAMC,MAAMD,WAAW,UAAU;YACnC;QACF;IACF;IAEA,MAAc,iBACZP,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMY,aAAa,IAAI,CAAC,gBAAgB,CAACT,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACY;IACrB;IAEA,MAAc,iBACZT,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMa,WAAWV,KAAK,IAAI;QAC1B,MAAMW,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMvB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACY,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI/B,KAAK,CAAC+B,MAAM,EAAE;gBAGhB,MAAM7B,aAAaH,oBAAoBC,KAAK,CAAC+B,MAAM;gBACnDrC,MACE,uCACA,CAAC,YAAY,EAAE6B,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAChC,KAAK,CAAC+B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAAC9B,aAAa,EAC1C,CAAC,QAAQ,EAAE+B,wBAAwBjC,KAAK,CAAC+B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtChC,YACAF,KAAK,CAAC+B,MAAM,EACZrB,SACA,CAACyB;oBACCnC,KAAK,CAAC+B,MAAM,GAAGI;gBACjB;gBAEFzB,QAAQ,KAAK,CAAC,IAAI,CAACwB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3E7B,MAAM,CAAC,OAAO,EAAEqC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASV,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,OAAOjB,OAAOsC;gBACtB5C,MACE,oBACA6B,UACAvB,OACA,CAAC,4BAA4B,EAAEsC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMC,YAAYD,YAAY,SAAS;gBACvCF,OAAOG,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACEpC,KAAK,CAAC+B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEA,IAAI;oBACF,MAAMiB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC9C,MAAM;gCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC+B,OAAO,IAAI,EAAEzB;gCACrDN,MAAM;4BACR;wBACF;wBACA2B,MAAM;qBACP;gBACH,EAAE,OAAOoB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,IAAIhB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQ4C,iBAAiB5C,OAAOyB,OAAO,WAAW;gBACpD,EAAE,OAAOoB,OAAY;oBACnB,MAAM,IAAIlB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,MAAM,OAAO,CAAC,cAAc,EAAEb,KAAK,SAAS,CAAChC,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFnD,MAAM,kBAAkB+B,OAAO,IAAI;gBACnC,MAAMqB,WAAWrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAMsB,eAAe,MAAMD,SAAS9C,OAAOsC;gBAC3C5C,MAAM,iBAAiB+B,OAAO,IAAI,EAAE,WAAWsB;gBAE/C,MAAMC,mBAAmBvB,OAAO,gBAAgB,IAAI;gBACpD,IAAIuB,mBAAmB,GACrB,MAAM3B,MAAM2B;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MAAM;wBACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC+B,OAAO,IAAI,EAAEzB;wBACpDN,MAAM;oBACR;gBACF,EAAE,OAAO+C,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,OAAO;oBACL,QAAQM;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAAC2B;IACrB;IAEQ,iBACNxB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE,GAAGI;QAEnD,IAAIyC,cAAcF;QAElB,IAAI,AAAuB,YAAvB,OAAOE,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI1C,AAAcQ,WAAdR,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,MAAM2C,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,SAAS1C,QAAQ,OAAO,IAAIO;YAC5B,OAAOkC;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOsC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEC,SAAS,EAAE,GAAGD;gBAEpBF,OACEpC,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAEgC,KAAK,SAAS,CACpEhC,QACC;gBAGL,IAAI,CAACuC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDH,OAAOG,WAAW;gBAElB,IAAIc;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbnB,KAAK,GAAG,GAAG;wBACTmB;oBACF;oBACAnB,KAAK,KAAK,GAAGmB,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjBnB,KAAK,eAAe,GAAGmB,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjBnB,KAAK,iBAAiB,GAAGmB,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkBxB,wBAAwBjC,SAC5C0D,qBAAqB1D,SACrBiB;gBACJ,MAAM0C,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD3D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF4D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC5D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAEF,MAAM6D,mBAAmBD,gBACrBE,0BACE;oBACE,GAAGF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;oBAC9C,GAAGA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;gBAChD,GACA,AAAwB,YAAxB,OAAO5D,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BiB;gBACJ,MAAM8C,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchE,MAAM,MAAM;gBAChC,MAAMiE,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,mBACJR,aAAaI,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhE,MAAM,SAAS;gBAEvB,MAAMqE,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,IAAI,CAACP,cAAc,CAACM,cAAc,CAACV,WACjC,IAAI;oBACFL,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCtD,OACA;wBACE,SAASuC;oBACX,GACAjC;oBAEFiD,UAAUD,aAAa,IAAI;oBAC3BgB,sBAAsBhB,aAAa,OAAO;gBAC5C,EAAE,OAAOT,OAAO;oBACd,IAAIA,iBAAiB0B,cACnBhB,UAAUV,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAGF,MAAM2B,UACJf,mBACAI,oBACAM,oBACAG;gBAGF,MAAMG,2BAA2B7E,iBAC/BqE,mBAAmB,cAAc;gBAGnC,IAAIS;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAACH,cACD,CAACI,4BACDzE,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;oBACF,MAAM2E,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDH,QAAQ,IAAI,EACZ;wBACE,mBACE,AAAwB,YAAxB,OAAOxE,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiB+E,UAAU;wBAC7BjF,MACE,uCACAsE,aACAW;wBAEFD,oBAAoBC;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQX;4BACR,OAAOW;wBACT,GACAV;oBAEJ,OACEvE,MACE,yDACAsE;gBAGN,EAAE,OAAOnB,OAAO;oBACdnD,MAAM,kCAAkCmD;gBAC1C;qBAEAnD,MAAM;gBAIV,IAAI,CAAC8E,SAAS;oBACZ,IAAInB,YACF,MAAM,IAAIkB,aACR,CAAC,oBAAoB,EAAEvE,MAAM,MAAM,EAAE,EACrCqD;oBAGJ,MAAM,IAAI1B,MAAM,CAAC,mBAAmB,EAAE3B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAI4E;gBAEJ,IAAIjB,WACFiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAM5E,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAI+D,YACTa,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO5E,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIqE,YACTO,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPV;wBACA,aAAaQ;oBACf;gBACF;gBAGFxB,WAAWsB;gBAEX,OAAO;oBACL,QAAQ;wBACNA;oBACF;oBACAI;gBACF;YACF;QACF;QAEA,OAAOxB;IACT;IAhgBA,YAAY,EACVyB,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTvD,WAAW,EACK,CAAE;QAbpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAQE,IAAI,CAAC,SAAS,GAAGqD;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGvD;IACrB;AAufF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type {\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  subTask?: boolean;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  subTask: boolean;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      subTask: !!options?.subTask,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n      [\n        'Sleep',\n        (plan) =>\n          this.handleSleepPlan(\n            plan as PlanningAction<PlanningActionParamSleep>,\n            context,\n          ),\n      ],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      subTask: context.subTask || undefined,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private handleSleepPlan(\n    plan: PlanningAction<PlanningActionParamSleep>,\n    context: PlanBuildContext,\n  ): void {\n    const sleepTask = this.createSleepTask(plan.param, {\n      thought: plan.thought,\n    });\n    if (context.subTask) {\n      sleepTask.subTask = true;\n    }\n    context.tasks.push(sleepTask);\n  }\n\n  public createSleepTask(\n    param: PlanningActionParamSleep,\n    meta?: { thought?: string },\n  ): ExecutionTaskActionApply<PlanningActionParamSleep> {\n    return {\n      type: 'Action Space',\n      subType: 'Sleep',\n      param,\n      thought: meta?.thought,\n      executor: async (taskParam) => {\n        await sleep(taskParam?.timeMs || 3000);\n      },\n    };\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      subTask: context.subTask || undefined,\n      executor: async (param, taskContext) => {\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug('will call \"beforeInvokeAction\" for interface');\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug('called \"beforeInvokeAction\" for interface');\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema);\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        debug('called action', action.name, 'result:', actionResult);\n\n        const delayAfterRunner = action.delayAfterRunner ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug('will call \"afterInvokeAction\" for interface');\n            await this.interface.afterInvokeAction(action.name, param);\n            debug('called \"afterInvokeAction\" for interface');\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent } = context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      subTask: context.subTask || undefined,\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n        const elementFromXpath = rectFromXpath\n          ? generateElementByPosition(\n              {\n                x: rectFromXpath.left + rectFromXpath.width / 2,\n                y: rectFromXpath.top + rectFromXpath.height / 2,\n              },\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCache =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          try {\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n              },\n              modelConfigForDefaultIntent,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForRect) {\n            try {\n              const feature = await this.interface.cacheFeatureForRect(\n                element.rect,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForRect failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForRect is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element,\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","undefined","sleepTask","meta","taskParam","sleep","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","uiContext","Promise","originalError","originalMessage","String","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByPosition","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","isCacheHit","elementFromAiLocate","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","feature","hitBy","interfaceInstance","service","taskCache"],"mappings":";;;;;;;;;;;;;;;;;AAgCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAsBO,MAAMC;IAqBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,SAAS,CAAC,CAACF,SAAS;QACtB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;YAC9D;gBACE;gBACA,CAACG,OACC,IAAI,CAAC,eAAe,CAClBA,MACAH;aAEL;SACF;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,WAAa;QACzB;QACAP,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEQ,gBACNH,IAA8C,EAC9CH,OAAyB,EACnB;QACN,MAAMQ,YAAY,IAAI,CAAC,eAAe,CAACL,KAAK,KAAK,EAAE;YACjD,SAASA,KAAK,OAAO;QACvB;QACA,IAAIH,QAAQ,OAAO,EACjBQ,UAAU,OAAO,GAAG;QAEtBR,QAAQ,KAAK,CAAC,IAAI,CAACQ;IACrB;IAEO,gBACLlB,KAA+B,EAC/BmB,IAA2B,EACyB;QACpD,OAAO;YACL,MAAM;YACN,SAAS;YACTnB;YACA,SAASmB,MAAM;YACf,UAAU,OAAOC;gBACf,MAAMC,MAAMD,WAAW,UAAU;YACnC;QACF;IACF;IAEA,MAAc,iBACZP,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMY,aAAa,IAAI,CAAC,gBAAgB,CAACT,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACY;IACrB;IAEA,MAAc,iBACZT,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMa,WAAWV,KAAK,IAAI;QAC1B,MAAMW,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMvB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACY,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI/B,KAAK,CAAC+B,MAAM,EAAE;gBAGhB,MAAM7B,aAAaH,oBAAoBC,KAAK,CAAC+B,MAAM;gBACnDrC,MACE,uCACA,CAAC,YAAY,EAAE6B,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAChC,KAAK,CAAC+B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAAC9B,aAAa,EAC1C,CAAC,QAAQ,EAAE+B,wBAAwBjC,KAAK,CAAC+B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtChC,YACAF,KAAK,CAAC+B,MAAM,EACZrB,SACA,CAACyB;oBACCnC,KAAK,CAAC+B,MAAM,GAAGI;gBACjB;gBAEFzB,QAAQ,KAAK,CAAC,IAAI,CAACwB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3E7B,MAAM,CAAC,OAAO,EAAEqC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASV,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,OAAOjB,OAAOsC;gBACtB5C,MACE,oBACA6B,UACAvB,OACA,CAAC,4BAA4B,EAAEsC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMC,YAAYD,YAAY,SAAS;gBACvCF,OAAOG,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACEpC,KAAK,CAAC+B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEA,IAAI;oBACF,MAAMiB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC9C,MAAM;gCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC+B,OAAO,IAAI,EAAEzB;gCACrDN,MAAM;4BACR;wBACF;wBACA2B,MAAM;qBACP;gBACH,EAAE,OAAOoB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,IAAIhB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQ4C,iBAAiB5C,OAAOyB,OAAO,WAAW;gBACpD,EAAE,OAAOoB,OAAY;oBACnB,MAAM,IAAIlB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,MAAM,OAAO,CAAC,cAAc,EAAEb,KAAK,SAAS,CAAChC,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFnD,MAAM,kBAAkB+B,OAAO,IAAI;gBACnC,MAAMqB,WAAWrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAMsB,eAAe,MAAMD,SAAS9C,OAAOsC;gBAC3C5C,MAAM,iBAAiB+B,OAAO,IAAI,EAAE,WAAWsB;gBAE/C,MAAMC,mBAAmBvB,OAAO,gBAAgB,IAAI;gBACpD,IAAIuB,mBAAmB,GACrB,MAAM3B,MAAM2B;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MAAM;wBACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC+B,OAAO,IAAI,EAAEzB;wBACpDN,MAAM;oBACR;gBACF,EAAE,OAAO+C,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,OAAO;oBACL,QAAQM;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAAC2B;IACrB;IAEQ,iBACNxB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE,GAAGI;QAEnD,IAAIyC,cAAcF;QAElB,IAAI,AAAuB,YAAvB,OAAOE,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI1C,AAAcQ,WAAdR,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,MAAM2C,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,SAAS1C,QAAQ,OAAO,IAAIO;YAC5B,OAAOkC;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOsC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEC,SAAS,EAAE,GAAGD;gBAEpBF,OACEpC,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAEgC,KAAK,SAAS,CACpEhC,QACC;gBAGL,IAAI,CAACuC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDH,OAAOG,WAAW;gBAElB,IAAIc;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbnB,KAAK,GAAG,GAAG;wBACTmB;oBACF;oBACAnB,KAAK,KAAK,GAAGmB,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjBnB,KAAK,eAAe,GAAGmB,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjBnB,KAAK,iBAAiB,GAAGmB,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkBxB,wBAAwBjC,SAC5C0D,qBAAqB1D,SACrBiB;gBACJ,MAAM0C,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD3D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF4D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC5D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAEF,MAAM6D,mBAAmBD,gBACrBE,0BACE;oBACE,GAAGF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;oBAC9C,GAAGA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;gBAChD,GACA,AAAwB,YAAxB,OAAO5D,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BiB;gBACJ,MAAM8C,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchE,MAAM,MAAM;gBAChC,MAAMiE,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,mBACJR,aAAaI,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhE,MAAM,SAAS;gBAEvB,MAAMqE,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,IAAI,CAACP,cAAc,CAACM,cAAc,CAACV,WACjC,IAAI;oBACFL,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCtD,OACA;wBACE,SAASuC;oBACX,GACAjC;oBAEFiD,UAAUD,aAAa,IAAI;oBAC3BgB,sBAAsBhB,aAAa,OAAO;gBAC5C,EAAE,OAAOT,OAAO;oBACd,IAAIA,iBAAiB0B,cACnBhB,UAAUV,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAGF,MAAM2B,UACJf,mBACAI,oBACAM,oBACAG;gBAGF,MAAMG,2BAA2B7E,iBAC/BqE,mBAAmB,cAAc;gBAGnC,IAAIS;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAACH,cACA,EAACV,aAAa,CAACc,wBAAuB,KACvCzE,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;oBACF,MAAM2E,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDH,QAAQ,IAAI,EACZ;wBACE,mBACE,AAAwB,YAAxB,OAAOxE,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiB+E,UAAU;wBAC7BjF,MACE,uCACAsE,aACAW;wBAEFD,oBAAoBC;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQX;4BACR,OAAOW;wBACT,GACAV;oBAEJ,OACEvE,MACE,yDACAsE;gBAGN,EAAE,OAAOnB,OAAO;oBACdnD,MAAM,kCAAkCmD;gBAC1C;qBAEAnD,MAAM;gBAIV,IAAI,CAAC8E,SAAS;oBACZ,IAAInB,YACF,MAAM,IAAIkB,aACR,CAAC,oBAAoB,EAAEvE,MAAM,MAAM,EAAE,EACrCqD;oBAGJ,MAAM,IAAI1B,MAAM,CAAC,mBAAmB,EAAE3B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAI4E;gBAEJ,IAAIjB,WACFiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAM5E,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAI+D,YACTa,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO5E,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIqE,YACTO,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPV;wBACA,aAAaQ;oBACf;gBACF;gBAGFxB,WAAWsB;gBAEX,OAAO;oBACL,QAAQ;wBACNA;oBACF;oBACAI;gBACF;YACF;QACF;QAEA,OAAOxB;IACT;IAhgBA,YAAY,EACVyB,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTvD,WAAW,EACK,CAAE;QAbpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAQE,IAAI,CAAC,SAAS,GAAGqD;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGvD;IACrB;AAufF"}

package/dist/es/agent/tasks.mjs

@@ -84,7 +84,10 @@ class TaskExecutor {
             runner
         };
     }
-    async action(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink) {
+    async action(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, fileChooserAccept) {
+        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink));
+    }
+    async runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink) {
         this.conversationHistory.reset();
         const session = this.createExecutionSession(taskTitleStr('Action', userPrompt));
         const runner = session.getRunner();
@@ -100,7 +103,8 @@ class TaskExecutor {
                 param: {
                     userInstruction: userPrompt,
                     aiActContext,
-                    imagesIncludeCount
+                    imagesIncludeCount,
+                    deepThink
                 },
                 executor: async (param, executorContext)=>{
                     const startTime = Date.now();
@@ -188,13 +192,12 @@ class TaskExecutor {
             }
             if (!this.conversationHistory.pendingFeedbackMessage) this.conversationHistory.pendingFeedbackMessage = 'I have finished the action previously planned.';
         }
-        const finalResult = {
+        return {
             output: {
                 yamlFlow
             },
             runner
         };
-        return finalResult;
     }
     createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt) {
         const queryTask = {
@@ -354,6 +357,22 @@ class TaskExecutor {
         });
     }
 }
-export { TaskExecutionError, TaskExecutor, locatePlanForLocate };
+async function withFileChooser(interfaceInstance, fileChooserAccept, action) {
+    if (!fileChooserAccept?.length) return action();
+    if (!interfaceInstance.registerFileChooserListener) throw new Error(`File upload is not supported on ${interfaceInstance.interfaceType}`);
+    const handler = async (chooser)=>{
+        await chooser.accept(fileChooserAccept);
+    };
+    const { dispose, getError } = await interfaceInstance.registerFileChooserListener(handler);
+    try {
+        const result = await action();
+        const error = getError();
+        if (error) throw error;
+        return result;
+    } finally{
+        dispose();
+    }
+}
+export { TaskExecutionError, TaskExecutor, locatePlanForLocate, withFileChooser };
 
 //# sourceMappingURL=tasks.mjs.map

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeepThinkOption,\n  DeviceAction,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.hooks = opts.hooks;\n    this.conversationHistory = new ConversationHistory();\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      options,\n    );\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActContext,\n            imagesIncludeCount,\n          },\n          executor: async (param, executorContext) => {\n            const startTime = Date.now();\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const { vlMode } = modelConfigForPlanning;\n            const uiTarsModelVersion =\n              vlMode === 'vlm-ui-tars'\n                ? modelConfigForPlanning.uiTarsModelVersion\n                : undefined;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planResult = await (uiTarsModelVersion\n              ? uiTarsPlanning\n              : plan)(param.userInstruction, {\n              context: uiContext,\n              actionContext: param.aiActContext,\n              interfaceType: this.interface.interfaceType as InterfaceType,\n              actionSpace,\n              modelConfig: modelConfigForPlanning,\n              conversationHistory: this.conversationHistory,\n              includeBbox: includeBboxInPlanning,\n              imagesIncludeCount,\n              deepThink,\n            });\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              log,\n              more_actions_needed_by_instruction,\n              error,\n              usage,\n              rawResponse,\n              sleep,\n              reasoning_content,\n            } = planResult;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = usage;\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              more_actions_needed_by_instruction,\n              log,\n              yamlFlow: planResult.yamlFlow,\n            };\n            executorContext.uiContext = uiContext;\n\n            const finalActions = [...(actions || [])];\n\n            if (sleep) {\n              const timeNow = Date.now();\n              const timeRemaining = sleep - (timeNow - startTime);\n              if (timeRemaining > 0) {\n                finalActions.push(this.sleepPlan(timeRemaining));\n              }\n            }\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfigForPlanning,\n          modelConfigForDefaultIntent,\n          {\n            cacheable,\n            subTask: true,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (this.conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          this.conversationHistory.pendingFeedbackMessage,\n        );\n      }\n      let errorFlag = false;\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        this.conversationHistory.pendingFeedbackMessage = `Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check if task is complete\n      if (!planResult?.more_actions_needed_by_instruction) {\n        if (errorFlag) {\n          debug(\n            'more_actions_needed_by_instruction is false, but there are errors in one planning loop, continue to run',\n          );\n        } else {\n          break;\n        }\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!this.conversationHistory.pendingFeedbackMessage) {\n        this.conversationHistory.pendingFeedbackMessage =\n          'I have finished the action previously planned.';\n      }\n    }\n\n    const finalResult = {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n    return finalResult;\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump, reasoning_content } = extractResult;\n        applyDump(dump);\n        task.reasoning_content = reasoning_content;\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.usage = usage;\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        undefined,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - currentCheckStart);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["debug","getDebug","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","startTime","Date","vlMode","uiTarsModelVersion","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","reasoning_content","finalActions","timeNow","timeRemaining","executables","errorFlag","String","Error","errorMsg","finalResult","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","thought","outputResult","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","lastCheckStart","errorThought","currentCheckStart","now","sleepTask","interfaceInstance","service","opts","ConversationHistory","TaskBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;;AAgDA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,wCAAwC;AAIvC,MAAMC;IAoBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IA6BQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCL,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BG,OACAC,wBACAC,6BACAL;IAEJ;IAEA,MAAM,uBAAuBM,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJhB,KAAa,EACbI,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACT;QAC5C,MAAM,EAAEiB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAQ3B;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMjB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUU;QAEzB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIkB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJL,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEc,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAGlC,MAAO,KAAM;YACX,MAAMb,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;gBACF;gBACA,UAAU,OAAOb,OAAOC;oBACtB,MAAMmB,YAAYC,KAAK,GAAG;oBAC1B,MAAM,EAAEnB,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM,EAAEoB,MAAM,EAAE,GAAG7B;oBACnB,MAAM8B,qBACJD,AAAW,kBAAXA,SACI7B,uBAAuB,kBAAkB,GACzCyB;oBAEN,MAAMM,cAAc,IAAI,CAAC,cAAc;oBACvCxC,MACE,sCACAwC,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDtB,OAAOuB,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,aAAa,MAAOL,AAAAA,CAAAA,qBACtBM,iBACAC,IAAG,EAAG9B,MAAM,eAAe,EAAE;wBAC/B,SAASE;wBACT,eAAeF,MAAM,YAAY;wBACjC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;wBAC3CwB;wBACA,aAAa/B;wBACb,qBAAqB,IAAI,CAAC,mBAAmB;wBAC7C,aAAagB;wBACbI;wBACAC;oBACF;oBACA9B,MAAM,cAAc+C,KAAK,SAAS,CAACH,YAAY,MAAM;oBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACLC,iBAAiB,EAClB,GAAGX;oBAEJ3B,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCoC;oBACF;oBACApC,gBAAgB,IAAI,CAAC,KAAK,GAAGmC;oBAC7BnC,gBAAgB,IAAI,CAAC,iBAAiB,GAAGsC;oBACzCtC,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS+B,WAAW,EAAE;wBACtBE;wBACAD;wBACA,UAAUL,WAAW,QAAQ;oBAC/B;oBACA3B,gBAAgB,SAAS,GAAGC;oBAE5B,MAAMsC,eAAe;2BAAKR,WAAW,EAAE;qBAAE;oBAEzC,IAAIM,OAAO;wBACT,MAAMG,UAAUpB,KAAK,GAAG;wBACxB,MAAMqB,gBAAgBJ,QAASG,CAAAA,UAAUrB,SAAQ;wBACjD,IAAIsB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;oBAErC;oBAEAvC,OAAO,CAACgC,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEF,OAAO,IAAI;oBAE3D,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAML,aAAatB,QAAQ;YAG3B,MAAMd,QAAQoC,YAAY,WAAW,EAAE;YACvCZ,SAAS,IAAI,IAAKY,YAAY,YAAY,EAAE;YAE5C,IAAIe;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CnD,OACAC,wBACAC,6BACA;oBACEiB;oBACA,SAAS;gBACX;YAEJ,EAAE,OAAOwB,OAAO;gBACd,OAAOtC,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEsC,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5EvC,QACC;YAEP;YACA,IAAI,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EACjDmC,QAAQ,IAAI,CACV,8FACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB;YAGnD,IAAIiB,YAAY;YAChB,IAAI;gBACF,MAAM/C,QAAQ,YAAY,CAAC8C,YAAY,KAAK;YAC9C,EAAE,OAAOR,OAAY;gBACnBS,YAAY;gBACZzB;gBACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAAG,CAAC,+BAA+B,EAAEgB,OAAO,WAAWU,OAAOV,QAAQ;gBACrHnD,MACE,yFACAmD,iBAAiBW,QAAQX,MAAM,OAAO,GAAGU,OAAOV,QAChD,6CACAhB;YAEJ;YAEA,IAAIA,8BAA8BjC,uCAChC,OAAOW,QAAQ,eAAe,CAAC;YAIjC,IAAI,CAAC+B,YAAY,oCACf,IAAIgB,WACF5D,MACE;iBAGF;YAKJ,EAAE+B;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAM8B,WAAW,CAAC,UAAU,EAAE9B,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOpB,QAAQ,eAAe,CAACkD;YACjC;YAEA,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EAClD,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAC7C;QAEN;QAEA,MAAMC,cAAc;YAClB,QAAQ;gBACNhC;YACF;YACAZ;QACF;QACA,OAAO4C;IACT;IAEQ,oBACNC,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,YAAYI,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOlD,OAAOuD;gBACtB,MAAM,EAAExD,IAAI,EAAE,GAAGwD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ3D,KAAK,GAAG,GAAG;wBACT2D;oBACF;gBACF;gBAGA,MAAMxD,YAAYqD,YAAY,SAAS;gBACvCpD,OAAOD,WAAW;gBAElB,MAAMyD,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACd,MAAMC,gBACJb,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIU,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGZ,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DpE,MAAM;oBACN,MAAMiF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX;gBAEJ,EAAE,OAAOlB,OAAO;oBACd,IAAIA,iBAAiBgC,cACnBV,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEiC,IAAI,EAAEhC,KAAK,EAAEiC,OAAO,EAAEX,IAAI,EAAEnB,iBAAiB,EAAE,GAAGwB;gBAC1DN,UAAUC;gBACV3D,KAAK,iBAAiB,GAAGwC;gBAEzB,IAAI+B,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAInB,AAAS,cAATA,MAEPqB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACLnE,OACEiE,MAAM,CAACP,YAAY,KAAK3C,QACxB;oBAEFoD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,IAAIZ,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtCvE,KAAK,KAAK,GAAGqC;oBACbrC,KAAK,OAAO,GAAGsE;oBACf,MAAM,IAAIvB,MAAM,CAAC,kBAAkB,EAAEuB,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQC;oBACR,KAAKd;oBACLpB;oBACAiC;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMxD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEmD,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASnB,KAAK,SAAS,CAACmB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAMjD,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACyD;QAE1C,IAAI,CAAChD,QACH,MAAM,IAAIwC,MACR;QAIJ,MAAM,EAAEvC,MAAM,EAAE8D,OAAO,EAAE,GAAG/D;QAE5B,OAAO;YACLC;YACA8D;YACAjE;QACF;IACF;IAEQ,UAAUmE,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;QACF;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBrB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEuB,UAAU,EAAErB,gBAAgB,EAAE,GAAGsB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM7E,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW8E;QAE1B,MAAMxE,SAASP,QAAQ,SAAS;QAChC,MAAM,EAAEgF,SAAS,EAAEC,eAAe,EAAE,GAAG1B;QAEvCjD,OAAOsE,WAAW;QAClBtE,OAAO0E,WAAW;QAClB1E,OAAO2E,iBAAiB;QAExB3E,OACE2E,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmB1D,KAAK,GAAG;QACjC,IAAI2D,iBAAiBD;QACrB,IAAIE,eAAe;QAEnB,MAAOD,iBAAiBD,oBAAoBF,UAAW;YACrD,MAAMK,oBAAoB7D,KAAK,GAAG;YAClC2D,iBAAiBE;YACjB,MAAM5B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAoB,YACAvB,aACAjC,QACAmC;YAGF,MAAM/C,SAAU,MAAMT,QAAQ,YAAY,CAACyD;YAO3C,IAAIhD,QAAQ,QACV,OAAO;gBACL,QAAQY;gBACRd;YACF;YAGF6E,eACE3E,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEoE,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMS,MAAM9D,KAAK,GAAG;YACpB,IAAI8D,MAAMD,oBAAoBJ,iBAAiB;gBAC7C,MAAMpC,gBAAgBoC,kBAAmBK,CAAAA,MAAMD,iBAAgB;gBAC/D,MAAME,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQ1C;gBACV;gBACA,MAAM7C,QAAQ,MAAM,CAACuF;YACvB;QACF;QAEA,OAAOvF,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEoF,cAAc;IACnE;IAzkBA,YACEI,iBAAoC,EACpCC,OAAgB,EAChBC,IAMC,CACD;QAjCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAkBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAG,IAAIC;QAC/B,IAAI,CAAC,mBAAmB,GAAGD,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIE,YAAY;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;QAClC;IACF;AAijBF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeepThinkOption,\n  DeviceAction,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.hooks = opts.hooks;\n    this.conversationHistory = new ConversationHistory();\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      options,\n    );\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n    fileChooserAccept?: string[],\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        modelConfigForPlanning,\n        modelConfigForDefaultIntent,\n        includeBboxInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n      );\n    });\n  }\n\n  private async runAction(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n          },\n          executor: async (param, executorContext) => {\n            const startTime = Date.now();\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const { vlMode } = modelConfigForPlanning;\n            const uiTarsModelVersion =\n              vlMode === 'vlm-ui-tars'\n                ? modelConfigForPlanning.uiTarsModelVersion\n                : undefined;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planResult = await (uiTarsModelVersion\n              ? uiTarsPlanning\n              : plan)(param.userInstruction, {\n              context: uiContext,\n              actionContext: param.aiActContext,\n              interfaceType: this.interface.interfaceType as InterfaceType,\n              actionSpace,\n              modelConfig: modelConfigForPlanning,\n              conversationHistory: this.conversationHistory,\n              includeBbox: includeBboxInPlanning,\n              imagesIncludeCount,\n              deepThink,\n            });\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              log,\n              more_actions_needed_by_instruction,\n              error,\n              usage,\n              rawResponse,\n              sleep,\n              reasoning_content,\n            } = planResult;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = usage;\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              more_actions_needed_by_instruction,\n              log,\n              yamlFlow: planResult.yamlFlow,\n            };\n            executorContext.uiContext = uiContext;\n\n            const finalActions = [...(actions || [])];\n\n            if (sleep) {\n              const timeNow = Date.now();\n              const timeRemaining = sleep - (timeNow - startTime);\n              if (timeRemaining > 0) {\n                finalActions.push(this.sleepPlan(timeRemaining));\n              }\n            }\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfigForPlanning,\n          modelConfigForDefaultIntent,\n          {\n            cacheable,\n            subTask: true,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (this.conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          this.conversationHistory.pendingFeedbackMessage,\n        );\n      }\n      let errorFlag = false;\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        this.conversationHistory.pendingFeedbackMessage = `Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check if task is complete\n      if (!planResult?.more_actions_needed_by_instruction) {\n        if (errorFlag) {\n          debug(\n            'more_actions_needed_by_instruction is false, but there are errors in one planning loop, continue to run',\n          );\n        } else {\n          break;\n        }\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!this.conversationHistory.pendingFeedbackMessage) {\n        this.conversationHistory.pendingFeedbackMessage =\n          'I have finished the action previously planned.';\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump, reasoning_content } = extractResult;\n        applyDump(dump);\n        task.reasoning_content = reasoning_content;\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.usage = usage;\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        undefined,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - currentCheckStart);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","withFileChooser","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","startTime","Date","vlMode","uiTarsModelVersion","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","reasoning_content","finalActions","timeNow","timeRemaining","executables","errorFlag","String","Error","errorMsg","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","thought","outputResult","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","lastCheckStart","errorThought","currentCheckStart","now","sleepTask","interfaceInstance","service","opts","ConversationHistory","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;AAgDA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,wCAAwC;AAIvC,MAAMC;IAoBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IA6BQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCL,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BG,OACAC,wBACAC,6BACAL;IAEJ;IAEA,MAAM,uBAAuBM,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJhB,KAAa,EACbI,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACT;QAC5C,MAAM,EAAEiB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAC3BC,iBAA4B,EAQ5B;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAED,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,wBACAC,6BACAe,uBACAC,cACAC,WACAC,8BACAC,oBACAC;IAGN;IAEA,MAAc,UACZN,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAQ3B;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMjB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUU;QAEzB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIoB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJP,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEgB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAGlC,MAAO,KAAM;YACX,MAAMf,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;oBACAC;gBACF;gBACA,UAAU,OAAOd,OAAOC;oBACtB,MAAMqB,YAAYC,KAAK,GAAG;oBAC1B,MAAM,EAAErB,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM,EAAEsB,MAAM,EAAE,GAAG/B;oBACnB,MAAMgC,qBACJD,AAAW,kBAAXA,SACI/B,uBAAuB,kBAAkB,GACzC2B;oBAEN,MAAMM,cAAc,IAAI,CAAC,cAAc;oBACvC1C,MACE,sCACA0C,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDxB,OAAOyB,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,aAAa,MAAOL,AAAAA,CAAAA,qBACtBM,iBACAC,IAAG,EAAGhC,MAAM,eAAe,EAAE;wBAC/B,SAASE;wBACT,eAAeF,MAAM,YAAY;wBACjC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;wBAC3C0B;wBACA,aAAajC;wBACb,qBAAqB,IAAI,CAAC,mBAAmB;wBAC7C,aAAagB;wBACbI;wBACAC;oBACF;oBACA9B,MAAM,cAAciD,KAAK,SAAS,CAACH,YAAY,MAAM;oBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACLC,iBAAiB,EAClB,GAAGX;oBAEJ7B,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCsC;oBACF;oBACAtC,gBAAgB,IAAI,CAAC,KAAK,GAAGqC;oBAC7BrC,gBAAgB,IAAI,CAAC,iBAAiB,GAAGwC;oBACzCxC,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAASiC,WAAW,EAAE;wBACtBE;wBACAD;wBACA,UAAUL,WAAW,QAAQ;oBAC/B;oBACA7B,gBAAgB,SAAS,GAAGC;oBAE5B,MAAMwC,eAAe;2BAAKR,WAAW,EAAE;qBAAE;oBAEzC,IAAIM,OAAO;wBACT,MAAMG,UAAUpB,KAAK,GAAG;wBACxB,MAAMqB,gBAAgBJ,QAASG,CAAAA,UAAUrB,SAAQ;wBACjD,IAAIsB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;oBAErC;oBAEAzC,OAAO,CAACkC,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEF,OAAO,IAAI;oBAE3D,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAML,aAAaxB,QAAQ;YAG3B,MAAMd,QAAQsC,YAAY,WAAW,EAAE;YACvCZ,SAAS,IAAI,IAAKY,YAAY,YAAY,EAAE;YAE5C,IAAIe;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CrD,OACAC,wBACAC,6BACA;oBACEiB;oBACA,SAAS;gBACX;YAEJ,EAAE,OAAO0B,OAAO;gBACd,OAAOxC,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEwC,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5EzC,QACC;YAEP;YACA,IAAI,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EACjDqC,QAAQ,IAAI,CACV,8FACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB;YAGnD,IAAIiB,YAAY;YAChB,IAAI;gBACF,MAAMjD,QAAQ,YAAY,CAACgD,YAAY,KAAK;YAC9C,EAAE,OAAOR,OAAY;gBACnBS,YAAY;gBACZzB;gBACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAAG,CAAC,+BAA+B,EAAEgB,OAAO,WAAWU,OAAOV,QAAQ;gBACrHrD,MACE,yFACAqD,iBAAiBW,QAAQX,MAAM,OAAO,GAAGU,OAAOV,QAChD,6CACAhB;YAEJ;YAEA,IAAIA,8BAA8BnC,uCAChC,OAAOW,QAAQ,eAAe,CAAC;YAIjC,IAAI,CAACiC,YAAY,oCACf,IAAIgB,WACF9D,MACE;iBAGF;YAKJ,EAAEiC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAM8B,WAAW,CAAC,UAAU,EAAE9B,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOtB,QAAQ,eAAe,CAACoD;YACjC;YAEA,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EAClD,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAC7C;QAEN;QAEA,OAAO;YACL,QAAQ;gBACN/B;YACF;YACAd;QACF;IACF;IAEQ,oBACN8C,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,YAAYI,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOnD,OAAOwD;gBACtB,MAAM,EAAEzD,IAAI,EAAE,GAAGyD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ5D,KAAK,GAAG,GAAG;wBACT4D;oBACF;gBACF;gBAGA,MAAMzD,YAAYsD,YAAY,SAAS;gBACvCrD,OAAOD,WAAW;gBAElB,MAAM0D,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACd,MAAMC,gBACJb,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIU,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGZ,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DrE,MAAM;oBACN,MAAMkF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX;gBAEJ,EAAE,OAAOjB,OAAO;oBACd,IAAIA,iBAAiB+B,cACnBV,UAAUrB,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEgC,IAAI,EAAE/B,KAAK,EAAEgC,OAAO,EAAEX,IAAI,EAAElB,iBAAiB,EAAE,GAAGuB;gBAC1DN,UAAUC;gBACV5D,KAAK,iBAAiB,GAAG0C;gBAEzB,IAAI8B,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAInB,AAAS,cAATA,MAEPqB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACLpE,OACEkE,MAAM,CAACP,YAAY,KAAK1C,QACxB;oBAEFmD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,IAAIZ,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtCxE,KAAK,KAAK,GAAGuC;oBACbvC,KAAK,OAAO,GAAGuE;oBACf,MAAM,IAAItB,MAAM,CAAC,kBAAkB,EAAEsB,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQC;oBACR,KAAKd;oBACLnB;oBACAgC;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMzD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEoD,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASlB,KAAK,SAAS,CAACkB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAMlD,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAAC0D;QAE1C,IAAI,CAACjD,QACH,MAAM,IAAI0C,MACR;QAIJ,MAAM,EAAEzC,MAAM,EAAE+D,OAAO,EAAE,GAAGhE;QAE5B,OAAO;YACLC;YACA+D;YACAlE;QACF;IACF;IAEQ,UAAUoE,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;QACF;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBrB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEuB,UAAU,EAAErB,gBAAgB,EAAE,GAAGsB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM9E,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW+E;QAE1B,MAAMzE,SAASP,QAAQ,SAAS;QAChC,MAAM,EAAEiF,SAAS,EAAEC,eAAe,EAAE,GAAG1B;QAEvClD,OAAOuE,WAAW;QAClBvE,OAAO2E,WAAW;QAClB3E,OAAO4E,iBAAiB;QAExB5E,OACE4E,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBzD,KAAK,GAAG;QACjC,IAAI0D,iBAAiBD;QACrB,IAAIE,eAAe;QAEnB,MAAOD,iBAAiBD,oBAAoBF,UAAW;YACrD,MAAMK,oBAAoB5D,KAAK,GAAG;YAClC0D,iBAAiBE;YACjB,MAAM5B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAoB,YACAvB,aACAhC,QACAkC;YAGF,MAAMhD,SAAU,MAAMT,QAAQ,YAAY,CAAC0D;YAO3C,IAAIjD,QAAQ,QACV,OAAO;gBACL,QAAQc;gBACRhB;YACF;YAGF8E,eACE5E,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEqE,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMS,MAAM7D,KAAK,GAAG;YACpB,IAAI6D,MAAMD,oBAAoBJ,iBAAiB;gBAC7C,MAAMnC,gBAAgBmC,kBAAmBK,CAAAA,MAAMD,iBAAgB;gBAC/D,MAAME,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQzC;gBACV;gBACA,MAAM/C,QAAQ,MAAM,CAACwF;YACvB;QACF;QAEA,OAAOxF,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqF,cAAc;IACnE;IA3mBA,YACEI,iBAAoC,EACpCC,OAAgB,EAChBC,IAMC,CACD;QAjCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAkBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAG,IAAIC;QAC/B,IAAI,CAAC,mBAAmB,GAAGD,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIE,YAAY;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;QAClC;IACF;AAmlBF;AAEO,eAAexE,gBACpBsE,iBAAoC,EACpCvE,iBAAuC,EACvCY,MAAwB;IAExB,IAAI,CAACZ,mBAAmB,QACtB,OAAOY;IAGT,IAAI,CAAC2D,kBAAkB,2BAA2B,EAChD,MAAM,IAAItC,MACR,CAAC,gCAAgC,EAAEsC,kBAAkB,aAAa,EAAE;IAIxE,MAAMK,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC7E;IACvB;IAEA,MAAM,EAAE8E,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMR,kBAAkB,2BAA2B,CAACK;IACtD,IAAI;QACF,MAAMrF,SAAS,MAAMqB;QAErB,MAAMU,QAAQyD;QACd,IAAIzD,OACF,MAAMA;QAER,OAAO/B;IACT,SAAU;QACRuF;IACF;AACF"}

package/dist/es/agent/utils.mjs

@@ -101,7 +101,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.1.1-beta-20260106151323.0";
+const getMidsceneVersion = ()=>"1.1.1-beta-20260108085624.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/index.mjs

@@ -5,7 +5,7 @@ import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/p
 import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
 import { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection } from "./inspect.mjs";
 import { plan } from "./llm-planning.mjs";
-import { AIActionType, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, loadActionParam, parseActionParam } from "../common.mjs";
+import { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, loadActionParam, parseActionParam } from "../common.mjs";
 import { resizeImageForUiTars, uiTarsPlanning } from "./ui-tars-planning.mjs";
 import { ConversationHistory } from "./conversation-history.mjs";
-export { AIActionType, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, callAI, callAIWithObjectResponse, callAIWithStringResponse, describeUserPage, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, loadActionParam, parseActionParam, plan, resizeImageForUiTars, systemPromptToLocateElement, uiTarsPlanning };
+export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, callAI, callAIWithObjectResponse, callAIWithStringResponse, describeUserPage, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, loadActionParam, parseActionParam, plan, resizeImageForUiTars, systemPromptToLocateElement, uiTarsPlanning };

package/dist/es/ai-model/inspect.mjs

@@ -2,7 +2,7 @@ import { generateElementByPosition } from "@midscene/shared/extractor/dom-util";
 import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { AIActionType, adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
+import { adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
 import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
@@ -109,10 +109,10 @@ async function AiLocateElement(options) {
         });
         msgs.push(...addOns);
     }
-    const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);
+    const res = await callAIFn(msgs, modelConfig);
     const rawResponse = JSON.stringify(res.content);
     let resRect;
-    let matchedElements = 'elements' in res.content ? res.content.elements : [];
+    let matchedElements = [];
     let errors = 'errors' in res.content ? res.content.errors : [];
     try {
         if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
@@ -181,7 +181,7 @@ async function AiLocateSection(options) {
         });
         msgs.push(...addOns);
     }
-    const result = await callAIWithObjectResponse(msgs, AIActionType.EXTRACT_DATA, modelConfig);
+    const result = await callAIWithObjectResponse(msgs, modelConfig);
     let sectionRect;
     const sectionBbox = result.content.bbox;
     if (sectionBbox) {
@@ -248,7 +248,7 @@ async function AiExtractElementInfo(options) {
         });
         msgs.push(...addOns);
     }
-    const result = await callAIWithObjectResponse(msgs, AIActionType.EXTRACT_DATA, modelConfig);
+    const result = await callAIWithObjectResponse(msgs, modelConfig);
     return {
         parseResult: result.content,
         usage: result.usage,
@@ -268,7 +268,7 @@ async function AiJudgeOrderSensitive(description, callAIFn, modelConfig) {
             content: userPrompt
         }
     ];
-    const result = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);
+    const result = await callAIFn(msgs, modelConfig);
     return {
         isOrderSensitive: result.content.isOrderSensitive ?? false,
         usage: result.usage

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from '../common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n    reasoning_content: result.reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(\n    msgs,\n    AIActionType.INSPECT_ELEMENT, // Reuse existing action type for now\n    modelConfig,\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","adaptBboxToRect","rectCenter","element","generateElementByPosition","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;AAsDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IAUC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,+BAA+BjB,wBACnCW;IAEF,MAAMO,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;QACxBO,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAad,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCe,cAAcf,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCgB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,iBAAXA,QAAyB;QAClC,MAAMa,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMvB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMkB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMC,MAAM,MAAMlB,SAASR,MAAM2B,aAAa,eAAe,EAAElB;IAE/D,MAAMmB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAI,UAAUI,gBACRR,IAAI,OAAO,CAAC,IAAI,EAChBP,YACAC,aACAf,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BgB,oBACAC,qBACAZ;YAGFjB,aAAa,WAAWqC;YAExB,MAAMK,aAAa;gBACjB,GAAGL,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMM,UAA+BC,0BACnCF,YACAtB;YAEFmB,SAAS,EAAE;YAEX,IAAII,SACFL,kBAAkB;gBAACK;aAAQ;QAE/B;IACF,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACN,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;aAFtBP,SAAS;YAACO;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMT;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAee,gBAAgBpC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEoC,kBAAkB,EAAEjC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAe2B,4BAA4BjC;IACjD,MAAMkC,gCAAgCC,0BACpCjD,wBAAwB8C;IAE1B,MAAM1C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMiC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMjB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQ4C,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA1C,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMqB,SAAS,MAAMC,yBACnB/C,MACA2B,aAAa,YAAY,EACzBlB;IAGF,IAAIuC;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAahB,gBACjBe,aACA3C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFf,aAAa,0BAA0BuD;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DnD,aAAa,wBAAwBwD;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASpB,MAAM,OAAO,CAACoB,OAC/B,GAAG,CAAC,CAACA,OACGnB,gBACLmB,MACA/C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNf,aAAa,qBAAqByD;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DzD,aAAa,iBAAiB2D;QAG9BN,cAAcQ,iBAAiBF,YAAYhD,QAAQ,IAAI,EAAEI;QACzDf,aAAa,2BAA2BqD;IAC1C;IAEA,IAAIS,cAAc9C;IAClB,IAAIqC,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BhD,kBACAqC,aACAtC,AAAW,iBAAXA;QAEF+C,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAajB,KAAK,SAAS,CAACiB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwBvD,OAO7C;IACC,MAAM,EAAEwD,SAAS,EAAEvD,OAAO,EAAEwD,aAAa,EAAE/D,gBAAgB,EAAEU,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAe+C;IACrB,MAAM,EAAEpD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM0D,wBAAwBC,uBAC5B5D,QAAQ,eAAe,IAAI,IAC3BwD;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKvD;YACL,QAAQ;QACV;IACF;IAGFuD,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMhE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASkD;QACX;KACD;IAED,IAAInE,kBAAkB;QACpB,MAAM0B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMqB,SAAS,MAAMC,yBACnB/C,MACA2B,aAAa,YAAY,EACzBlB;IAEF,OAAO;QACL,aAAaqC,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;QACnB,mBAAmBA,OAAO,iBAAiB;IAC7C;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnB5D,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeqD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMpE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASsD;QACX;KACD;IAED,MAAMxB,SAAS,MAAMtC,SACnBR,MACA2B,aAAa,eAAe,EAC5BlB;IAGF,OAAO;QACL,kBAAkBqC,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: LocateResultElement[] = [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n    reasoning_content: result.reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(msgs, modelConfig);\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","res","rawResponse","JSON","resRect","matchedElements","errors","Array","adaptBboxToRect","rectCenter","element","generateElementByPosition","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;AAiDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IAUC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,+BAA+BjB,wBACnCW;IAEF,MAAMO,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;QACxBO,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAad,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCe,cAAcf,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCgB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,iBAAXA,QAAyB;QAClC,MAAMa,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMvB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMkB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMC,MAAM,MAAMlB,SAASR,MAAMS;IAEjC,MAAMkB,cAAcC,KAAK,SAAS,CAACF,IAAI,OAAO;IAE9C,IAAIG;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBM,MAAM,OAAO,CAACN,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAG,UAAUI,gBACRP,IAAI,OAAO,CAAC,IAAI,EAChBP,YACAC,aACAf,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BgB,oBACAC,qBACAZ;YAGFjB,aAAa,WAAWoC;YAExB,MAAMK,aAAa;gBACjB,GAAGL,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMM,UAA+BC,0BACnCF,YACArB;YAEFkB,SAAS,EAAE;YAEX,IAAII,SACFL,kBAAkB;gBAACK;aAAQ;QAE/B;IACF,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACN,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;aAFtBP,SAAS;YAACO;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMT;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOD,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAec,gBAAgBnC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEmC,kBAAkB,EAAEhC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAe0B,4BAA4BhC;IACjD,MAAMiC,gCAAgCC,0BACpChD,wBAAwB6C;IAE1B,MAAMzC,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMgC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMhB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQ2C,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAzC,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMoB,SAAS,MAAMC,yBACnB9C,MACAS;IAGF,IAAIsC;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAahB,gBACjBe,aACA1C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFf,aAAa,0BAA0BsD;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DlD,aAAa,wBAAwBuD;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASpB,MAAM,OAAO,CAACoB,OAC/B,GAAG,CAAC,CAACA,OACGnB,gBACLmB,MACA9C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNf,aAAa,qBAAqBwD;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DxD,aAAa,iBAAiB0D;QAG9BN,cAAcQ,iBAAiBF,YAAY/C,QAAQ,IAAI,EAAEI;QACzDf,aAAa,2BAA2BoD;IAC1C;IAEA,IAAIS,cAAc7C;IAClB,IAAIoC,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1B/C,kBACAoC,aACArC,AAAW,iBAAXA;QAEF8C,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAajB,KAAK,SAAS,CAACiB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwBtD,OAO7C;IACC,MAAM,EAAEuD,SAAS,EAAEtD,OAAO,EAAEuD,aAAa,EAAE9D,gBAAgB,EAAEU,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAe8C;IACrB,MAAM,EAAEnD,gBAAgB,EAAE,GAAGL;IAE7B,MAAMyD,wBAAwBC,uBAC5B3D,QAAQ,eAAe,IAAI,IAC3BuD;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKtD;YACL,QAAQ;QACV;IACF;IAGFsD,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM/D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASiD;QACX;KACD;IAED,IAAIlE,kBAAkB;QACpB,MAAM0B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMoB,SAAS,MAAMC,yBACnB9C,MACAS;IAEF,OAAO;QACL,aAAaoC,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;QACnB,mBAAmBA,OAAO,iBAAiB;IAC7C;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnB3D,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeoD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMnE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASqD;QACX;KACD;IAED,MAAMxB,SAAS,MAAMrC,SAASR,MAAMS;IAEpC,OAAO;QACL,kBAAkBoC,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -1,7 +1,7 @@
 import { paddingToMatchBlockByBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { AIActionType, buildYamlFlowFromPlans, fillBboxParam, findAllMidsceneLocatorField } from "../common.mjs";
+import { buildYamlFlowFromPlans, fillBboxParam, findAllMidsceneLocatorField } from "../common.mjs";
 import { systemPromptToTaskPlanning } from "./prompt/llm-planning.mjs";
 import { callAIWithObjectResponse } from "./service-caller/index.mjs";
 const debug = getDebug('planning');
@@ -82,7 +82,7 @@ async function plan(userInstruction, opts) {
         ...instruction,
         ...historyLog
     ];
-    const { content: planFromAI, contentString: rawResponse, usage, reasoning_content } = await callAIWithObjectResponse(msgs, AIActionType.PLAN, modelConfig, {
+    const { content: planFromAI, contentString: rawResponse, usage, reasoning_content } = await callAIWithObjectResponse(msgs, modelConfig, {
         deepThink: 'unset' === opts.deepThink ? void 0 : opts.deepThink
     });
     const actions = planFromAI.action ? [

package/dist/es/ai-model/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeepThinkOption,\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  AIActionType,\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory: ConversationHistory;\n    includeBbox: boolean;\n    imagesIncludeCount?: number;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { vlMode } = modelConfig;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode,\n    includeBbox: opts.includeBbox,\n  });\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = size.width;\n  let imageHeight = size.height;\n  const rightLimit = imageWidth;\n  const bottomLimit = imageHeight;\n\n  // Process image based on VL mode requirements\n  if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n        },\n      ],\n    },\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The last screenshot is attached. Please going on according to the instruction.`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'this is the latest screenshot',\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    content: planFromAI,\n    contentString: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAIWithObjectResponse<RawResponsePlanningAIResponse>(\n    msgs,\n    AIActionType.PLAN,\n    modelConfig,\n    {\n      deepThink: opts.deepThink === 'unset' ? undefined : opts.deepThink,\n    },\n  );\n\n  const actions = planFromAI.action ? [planFromAI.action] : [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    reasoning_content,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult && vlMode !== undefined) {\n        // Always use VL mode to fill bbox parameters\n        action.param[field] = fillBboxParam(\n          locateResult,\n          imageWidth,\n          imageHeight,\n          rightLimit,\n          bottomLimit,\n          vlMode,\n        );\n      }\n    });\n  });\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","plan","userInstruction","opts","context","modelConfig","conversationHistory","screenshotBase64","size","vlMode","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","rightLimit","bottomLimit","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","historyLog","msgs","planFromAI","rawResponse","usage","reasoning_content","callAIWithObjectResponse","AIActionType","undefined","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","console"],"mappings":";;;;;;AAuBA,MAAMA,QAAQC,SAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAUC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGJ;IAEnC,MAAM,EAAEK,MAAM,EAAE,GAAGJ;IAEnB,MAAMK,eAAe,MAAMC,2BAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7BM;QACA,aAAaN,KAAK,WAAW;IAC/B;IAEA,IAAIS,eAAeL;IACnB,IAAIM,aAAaL,KAAK,KAAK;IAC3B,IAAIM,cAAcN,KAAK,MAAM;IAC7B,MAAMO,aAAaF;IACnB,MAAMG,cAAcF;IAGpB,IAAIL,AAAW,iBAAXA,QAAyB;QAC3B,MAAMQ,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBhB,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEjB,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAImB;IAEJ,IAAIf,oBAAoB,sBAAsB,EAAE;QAC9Ce,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGf,oBAAoB,sBAAsB,CAAC,gFAAgF,CAAC;gBACvI;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKM;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEe,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACe;IAC3B,MAAMC,aAAahB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMoB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASb;QAAa;WACrCU;WACAE;KACJ;IAED,MAAM,EACJ,SAASE,UAAU,EACnB,eAAeC,WAAW,EAC1BC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,yBACRL,MACAM,aAAa,IAAI,EACjBxB,aACA;QACE,WAAWF,AAAmB,YAAnBA,KAAK,SAAS,GAAe2B,SAAY3B,KAAK,SAAS;IACpE;IAGF,MAAM4B,UAAUP,WAAW,MAAM,GAAG;QAACA,WAAW,MAAM;KAAC,GAAG,EAAE;IAC5D,MAAMQ,cAAkC;QACtC,GAAGR,UAAU;QACbO;QACAN;QACAC;QACAC;QACA,UAAUM,uBACRF,SACA5B,KAAK,WAAW,EAChBqB,WAAW,KAAK;IAEpB;IAEAU,OAAOV,YAAY;IAEnBO,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBlC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACgC,SAAWA,OAAO,IAAI,KAAKC;QAG9BrC,MAAM,+BAA+BsC;QACrC,MAAMC,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENtC,MAAM,gBAAgBuC;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,gBAAgBhC,AAAWqB,WAAXrB,QAElB0B,OAAO,KAAK,CAACK,MAAM,GAAGE,cACpBD,cACA5B,YACAC,aACAC,YACAC,aACAP;QAGN;IACF;IAEA,IACEsB,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBW,QAAQ,IAAI,CACV,8EACAzC;IAIJI,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMmB;YACR;SACD;IACH;IAEA,OAAOO;AACT"}
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeepThinkOption,\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory: ConversationHistory;\n    includeBbox: boolean;\n    imagesIncludeCount?: number;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { vlMode } = modelConfig;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode,\n    includeBbox: opts.includeBbox,\n  });\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = size.width;\n  let imageHeight = size.height;\n  const rightLimit = imageWidth;\n  const bottomLimit = imageHeight;\n\n  // Process image based on VL mode requirements\n  if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n        },\n      ],\n    },\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The last screenshot is attached. Please going on according to the instruction.`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'this is the latest screenshot',\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    content: planFromAI,\n    contentString: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAIWithObjectResponse<RawResponsePlanningAIResponse>(\n    msgs,\n    modelConfig,\n    {\n      deepThink: opts.deepThink === 'unset' ? undefined : opts.deepThink,\n    },\n  );\n\n  const actions = planFromAI.action ? [planFromAI.action] : [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    reasoning_content,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult && vlMode !== undefined) {\n        // Always use VL mode to fill bbox parameters\n        action.param[field] = fillBboxParam(\n          locateResult,\n          imageWidth,\n          imageHeight,\n          rightLimit,\n          bottomLimit,\n          vlMode,\n        );\n      }\n    });\n  });\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","plan","userInstruction","opts","context","modelConfig","conversationHistory","screenshotBase64","size","vlMode","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","rightLimit","bottomLimit","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","historyLog","msgs","planFromAI","rawResponse","usage","reasoning_content","callAIWithObjectResponse","undefined","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","console"],"mappings":";;;;;;AAsBA,MAAMA,QAAQC,SAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAUC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGJ;IAEnC,MAAM,EAAEK,MAAM,EAAE,GAAGJ;IAEnB,MAAMK,eAAe,MAAMC,2BAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7BM;QACA,aAAaN,KAAK,WAAW;IAC/B;IAEA,IAAIS,eAAeL;IACnB,IAAIM,aAAaL,KAAK,KAAK;IAC3B,IAAIM,cAAcN,KAAK,MAAM;IAC7B,MAAMO,aAAaF;IACnB,MAAMG,cAAcF;IAGpB,IAAIL,AAAW,iBAAXA,QAAyB;QAC3B,MAAMQ,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBhB,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEjB,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAImB;IAEJ,IAAIf,oBAAoB,sBAAsB,EAAE;QAC9Ce,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGf,oBAAoB,sBAAsB,CAAC,gFAAgF,CAAC;gBACvI;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKM;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEe,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACe;IAC3B,MAAMC,aAAahB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMoB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASb;QAAa;WACrCU;WACAE;KACJ;IAED,MAAM,EACJ,SAASE,UAAU,EACnB,eAAeC,WAAW,EAC1BC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,yBACRL,MACAlB,aACA;QACE,WAAWF,AAAmB,YAAnBA,KAAK,SAAS,GAAe0B,SAAY1B,KAAK,SAAS;IACpE;IAGF,MAAM2B,UAAUN,WAAW,MAAM,GAAG;QAACA,WAAW,MAAM;KAAC,GAAG,EAAE;IAC5D,MAAMO,cAAkC;QACtC,GAAGP,UAAU;QACbM;QACAL;QACAC;QACAC;QACA,UAAUK,uBACRF,SACA3B,KAAK,WAAW,EAChBqB,WAAW,KAAK;IAEpB;IAEAS,OAAOT,YAAY;IAEnBM,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBjC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAAC+B,SAAWA,OAAO,IAAI,KAAKC;QAG9BpC,MAAM,+BAA+BqC;QACrC,MAAMC,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENrC,MAAM,gBAAgBsC;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,gBAAgB/B,AAAWoB,WAAXpB,QAElByB,OAAO,KAAK,CAACK,MAAM,GAAGE,cACpBD,cACA3B,YACAC,aACAC,YACAC,aACAP;QAGN;IACF;IAEA,IACEqB,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBW,QAAQ,IAAI,CACV,8EACAxC;IAIJI,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMmB;YACR;SACD;IACH;IAEA,OAAOM;AACT"}