@midscene/core 1.10.8-beta-20260724035136.0 → 1.10.8

package/dist/es/agent/file-chooser.mjs

@@ -0,0 +1,82 @@
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import { normalizeFilePaths } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function normalizeFileChooserAccept(files) {
+    const filesArray = Array.isArray(files) ? files : [
+        files
+    ];
+    return normalizeFilePaths(filesArray);
+}
+function normalizeFileChooserAcceptInAllowedDir(files, allowedDir) {
+    const absoluteAllowedDir = resolve(allowedDir);
+    const filesArray = Array.isArray(files) ? files : [
+        files
+    ];
+    const resolvedFiles = filesArray.map((file)=>{
+        const resolvedFile = resolve(absoluteAllowedDir, file);
+        const relativePath = relative(absoluteAllowedDir, resolvedFile);
+        if ('..' === relativePath || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) throw new Error(`File chooser path must be inside the configured fileChooserAllowedDir: ${file}`);
+        return resolvedFile;
+    });
+    return normalizeFilePaths(resolvedFiles);
+}
+class FileChooserAccepter {
+    async register(files) {
+        await this.replaceAcceptedFiles(normalizeFileChooserAccept(files));
+    }
+    async registerFromAllowedDir(files, allowedDir) {
+        await this.replaceAcceptedFiles(normalizeFileChooserAcceptInAllowedDir(files, allowedDir));
+    }
+    async replaceAcceptedFiles(acceptedFiles) {
+        const previousRegistrationError = await this.clear();
+        if (previousRegistrationError) throw previousRegistrationError;
+        if (!acceptedFiles.length) return;
+        if (!this.interfaceInstance.registerFileChooserListener) throw new Error(`File upload is not supported on ${this.interfaceInstance.interfaceType}`);
+        this.registration = await this.interfaceInstance.registerFileChooserListener(async (chooser)=>{
+            await chooser.accept(acceptedFiles);
+        });
+    }
+    async clear() {
+        const registration = this.registration;
+        this.registration = void 0;
+        if (!registration) return;
+        try {
+            return await registration.getError();
+        } finally{
+            registration.dispose();
+        }
+    }
+    constructor(interfaceInstance){
+        _define_property(this, "interfaceInstance", void 0);
+        _define_property(this, "registration", void 0);
+        this.interfaceInstance = interfaceInstance;
+    }
+}
+async function withFileChooser(interfaceInstance, fileChooserAccept, action) {
+    if (!fileChooserAccept?.length) return action();
+    if (!interfaceInstance.registerFileChooserListener) throw new Error(`File upload is not supported on ${interfaceInstance.interfaceType}`);
+    const handler = async (chooser)=>{
+        await chooser.accept(fileChooserAccept);
+    };
+    const { dispose, getError } = await interfaceInstance.registerFileChooserListener(handler);
+    try {
+        const result = await action();
+        const error = await getError();
+        if (error) throw error;
+        return result;
+    } finally{
+        dispose();
+    }
+}
+export { FileChooserAccepter, normalizeFileChooserAccept, normalizeFileChooserAcceptInAllowedDir, withFileChooser };
+
+//# sourceMappingURL=file-chooser.mjs.map

package/dist/es/agent/file-chooser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/file-chooser.mjs","sources":["../../../src/agent/file-chooser.ts"],"sourcesContent":["import { isAbsolute, relative, resolve, sep } from 'node:path';\nimport type {\n  AbstractInterface,\n  FileChooserHandler,\n  FileChooserRegistration,\n} from '@/device';\nimport { normalizeFilePaths } from './utils';\n\nexport type FileChooserAccept = string | string[];\n\nexport function normalizeFileChooserAccept(files: FileChooserAccept): string[] {\n  const filesArray = Array.isArray(files) ? files : [files];\n  return normalizeFilePaths(filesArray);\n}\n\nexport function normalizeFileChooserAcceptInAllowedDir(\n  files: FileChooserAccept,\n  allowedDir: string,\n): string[] {\n  const absoluteAllowedDir = resolve(allowedDir);\n  const filesArray = Array.isArray(files) ? files : [files];\n  const resolvedFiles = filesArray.map((file) => {\n    const resolvedFile = resolve(absoluteAllowedDir, file);\n    const relativePath = relative(absoluteAllowedDir, resolvedFile);\n    if (\n      relativePath === '..' ||\n      relativePath.startsWith(`..${sep}`) ||\n      isAbsolute(relativePath)\n    ) {\n      throw new Error(\n        `File chooser path must be inside the configured fileChooserAllowedDir: ${file}`,\n      );\n    }\n\n    return resolvedFile;\n  });\n\n  return normalizeFilePaths(resolvedFiles);\n}\n\n/**\n * Holds the current file chooser configuration for one aiAct execution.\n * Registering new files replaces the previous configuration; callers must\n * clear the accepter when the aiAct scope ends.\n */\nexport class FileChooserAccepter {\n  private registration?: FileChooserRegistration;\n\n  constructor(private readonly interfaceInstance: AbstractInterface) {}\n\n  async register(files: FileChooserAccept): Promise<void> {\n    await this.replaceAcceptedFiles(normalizeFileChooserAccept(files));\n  }\n\n  async registerFromAllowedDir(\n    files: FileChooserAccept,\n    allowedDir: string,\n  ): Promise<void> {\n    await this.replaceAcceptedFiles(\n      normalizeFileChooserAcceptInAllowedDir(files, allowedDir),\n    );\n  }\n\n  private async replaceAcceptedFiles(acceptedFiles: string[]): Promise<void> {\n    const previousRegistrationError = await this.clear();\n    if (previousRegistrationError) {\n      throw previousRegistrationError;\n    }\n\n    if (!acceptedFiles.length) {\n      return;\n    }\n\n    if (!this.interfaceInstance.registerFileChooserListener) {\n      throw new Error(\n        `File upload is not supported on ${this.interfaceInstance.interfaceType}`,\n      );\n    }\n\n    this.registration =\n      await this.interfaceInstance.registerFileChooserListener(\n        async (chooser: FileChooserHandler) => {\n          await chooser.accept(acceptedFiles);\n        },\n      );\n  }\n\n  async clear(): Promise<Error | undefined> {\n    const registration = this.registration;\n    this.registration = undefined;\n    if (!registration) {\n      return undefined;\n    }\n\n    try {\n      return await registration.getError();\n    } finally {\n      registration.dispose();\n    }\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    const error = await getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["normalizeFileChooserAccept","files","filesArray","Array","normalizeFilePaths","normalizeFileChooserAcceptInAllowedDir","allowedDir","absoluteAllowedDir","resolve","resolvedFiles","file","resolvedFile","relativePath","relative","sep","isAbsolute","Error","FileChooserAccepter","acceptedFiles","previousRegistrationError","chooser","registration","undefined","interfaceInstance","withFileChooser","fileChooserAccept","action","handler","dispose","getError","result","error"],"mappings":";;;;;;;;;;;;AAUO,SAASA,2BAA2BC,KAAwB;IACjE,MAAMC,aAAaC,MAAM,OAAO,CAACF,SAASA,QAAQ;QAACA;KAAM;IACzD,OAAOG,mBAAmBF;AAC5B;AAEO,SAASG,uCACdJ,KAAwB,EACxBK,UAAkB;IAElB,MAAMC,qBAAqBC,QAAQF;IACnC,MAAMJ,aAAaC,MAAM,OAAO,CAACF,SAASA,QAAQ;QAACA;KAAM;IACzD,MAAMQ,gBAAgBP,WAAW,GAAG,CAAC,CAACQ;QACpC,MAAMC,eAAeH,QAAQD,oBAAoBG;QACjD,MAAME,eAAeC,SAASN,oBAAoBI;QAClD,IACEC,AAAiB,SAAjBA,gBACAA,aAAa,UAAU,CAAC,CAAC,EAAE,EAAEE,KAAK,KAClCC,WAAWH,eAEX,MAAM,IAAII,MACR,CAAC,uEAAuE,EAAEN,MAAM;QAIpF,OAAOC;IACT;IAEA,OAAOP,mBAAmBK;AAC5B;AAOO,MAAMQ;IAKX,MAAM,SAAShB,KAAwB,EAAiB;QACtD,MAAM,IAAI,CAAC,oBAAoB,CAACD,2BAA2BC;IAC7D;IAEA,MAAM,uBACJA,KAAwB,EACxBK,UAAkB,EACH;QACf,MAAM,IAAI,CAAC,oBAAoB,CAC7BD,uCAAuCJ,OAAOK;IAElD;IAEA,MAAc,qBAAqBY,aAAuB,EAAiB;QACzE,MAAMC,4BAA4B,MAAM,IAAI,CAAC,KAAK;QAClD,IAAIA,2BACF,MAAMA;QAGR,IAAI,CAACD,cAAc,MAAM,EACvB;QAGF,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,2BAA2B,EACrD,MAAM,IAAIF,MACR,CAAC,gCAAgC,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;QAI7E,IAAI,CAAC,YAAY,GACf,MAAM,IAAI,CAAC,iBAAiB,CAAC,2BAA2B,CACtD,OAAOI;YACL,MAAMA,QAAQ,MAAM,CAACF;QACvB;IAEN;IAEA,MAAM,QAAoC;QACxC,MAAMG,eAAe,IAAI,CAAC,YAAY;QACtC,IAAI,CAAC,YAAY,GAAGC;QACpB,IAAI,CAACD,cACH;QAGF,IAAI;YACF,OAAO,MAAMA,aAAa,QAAQ;QACpC,SAAU;YACRA,aAAa,OAAO;QACtB;IACF;IAnDA,YAA6BE,iBAAoC,CAAE;;QAFnE,uBAAQ,gBAAR;aAE6BA,iBAAiB,GAAjBA;IAAuC;AAoDtE;AAEO,eAAeC,gBACpBD,iBAAoC,EACpCE,iBAAuC,EACvCC,MAAwB;IAExB,IAAI,CAACD,mBAAmB,QACtB,OAAOC;IAGT,IAAI,CAACH,kBAAkB,2BAA2B,EAChD,MAAM,IAAIP,MACR,CAAC,gCAAgC,EAAEO,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOP;QACrB,MAAMA,QAAQ,MAAM,CAACK;IACvB;IAEA,MAAM,EAAEG,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMN,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMG,SAAS,MAAMJ;QACrB,MAAMK,QAAQ,MAAMF;QACpB,IAAIE,OACF,MAAMA;QAER,OAAOD;IACT,SAAU;QACRF;IACF;AACF"}

package/dist/es/agent/prompt-context.mjs

@@ -9,6 +9,17 @@ const buildPromptWithContext = (prompt, context)=>{
         prompt: promptWithContext
     };
 };
-export { buildPromptWithContext };
+const buildLocatePromptWithContext = (prompt, context)=>{
+    const trimmedContext = context?.trim();
+    if (!trimmedContext) return prompt;
+    const promptText = 'string' == typeof prompt ? prompt : prompt.prompt;
+    const promptWithContext = `<CONTEXT>\n${trimmedContext}\n</CONTEXT>\n\n<LOCATE_TARGET>\n${promptText}\n</LOCATE_TARGET>`;
+    if ('string' == typeof prompt) return promptWithContext;
+    return {
+        ...prompt,
+        prompt: promptWithContext
+    };
+};
+export { buildLocatePromptWithContext, buildPromptWithContext };
 
 //# sourceMappingURL=prompt-context.mjs.map

package/dist/es/agent/prompt-context.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/prompt-context.mjs","sources":["../../../src/agent/prompt-context.ts"],"sourcesContent":["import type { TUserPrompt } from '@/ai-model';\n\nexport const buildPromptWithContext = (\n  prompt: TUserPrompt,\n  context: string | undefined,\n): TUserPrompt => {\n  const trimmedContext = context?.trim();\n  if (!trimmedContext) {\n    return prompt;\n  }\n\n  const promptText = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const promptWithContext = `Context for this request:\\n${trimmedContext}\\n\\n${promptText}`;\n\n  if (typeof prompt === 'string') {\n    return promptWithContext;\n  }\n\n  return {\n    ...prompt,\n    prompt: promptWithContext,\n  };\n};\n"],"names":["buildPromptWithContext","prompt","context","trimmedContext","promptText","promptWithContext"],"mappings":"AAEO,MAAMA,yBAAyB,CACpCC,QACAC;IAEA,MAAMC,iBAAiBD,SAAS;IAChC,IAAI,CAACC,gBACH,OAAOF;IAGT,MAAMG,aAAa,AAAkB,YAAlB,OAAOH,SAAsBA,SAASA,OAAO,MAAM;IACtE,MAAMI,oBAAoB,CAAC,2BAA2B,EAAEF,eAAe,IAAI,EAAEC,YAAY;IAEzF,IAAI,AAAkB,YAAlB,OAAOH,QACT,OAAOI;IAGT,OAAO;QACL,GAAGJ,MAAM;QACT,QAAQI;IACV;AACF"}
+{"version":3,"file":"agent/prompt-context.mjs","sources":["../../../src/agent/prompt-context.ts"],"sourcesContent":["import type { TUserPrompt } from '@/ai-model';\n\nexport const buildPromptWithContext = (\n  prompt: TUserPrompt,\n  context: string | undefined,\n): TUserPrompt => {\n  const trimmedContext = context?.trim();\n  if (!trimmedContext) {\n    return prompt;\n  }\n\n  const promptText = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const promptWithContext = `Context for this request:\\n${trimmedContext}\\n\\n${promptText}`;\n\n  if (typeof prompt === 'string') {\n    return promptWithContext;\n  }\n\n  return {\n    ...prompt,\n    prompt: promptWithContext,\n  };\n};\n\nexport const buildLocatePromptWithContext = (\n  prompt: TUserPrompt,\n  context: string | undefined,\n): TUserPrompt => {\n  const trimmedContext = context?.trim();\n  if (!trimmedContext) {\n    return prompt;\n  }\n\n  const promptText = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const promptWithContext = `<CONTEXT>\\n${trimmedContext}\\n</CONTEXT>\\n\\n<LOCATE_TARGET>\\n${promptText}\\n</LOCATE_TARGET>`;\n\n  if (typeof prompt === 'string') {\n    return promptWithContext;\n  }\n\n  return {\n    ...prompt,\n    prompt: promptWithContext,\n  };\n};\n"],"names":["buildPromptWithContext","prompt","context","trimmedContext","promptText","promptWithContext","buildLocatePromptWithContext"],"mappings":"AAEO,MAAMA,yBAAyB,CACpCC,QACAC;IAEA,MAAMC,iBAAiBD,SAAS;IAChC,IAAI,CAACC,gBACH,OAAOF;IAGT,MAAMG,aAAa,AAAkB,YAAlB,OAAOH,SAAsBA,SAASA,OAAO,MAAM;IACtE,MAAMI,oBAAoB,CAAC,2BAA2B,EAAEF,eAAe,IAAI,EAAEC,YAAY;IAEzF,IAAI,AAAkB,YAAlB,OAAOH,QACT,OAAOI;IAGT,OAAO;QACL,GAAGJ,MAAM;QACT,QAAQI;IACV;AACF;AAEO,MAAMC,+BAA+B,CAC1CL,QACAC;IAEA,MAAMC,iBAAiBD,SAAS;IAChC,IAAI,CAACC,gBACH,OAAOF;IAGT,MAAMG,aAAa,AAAkB,YAAlB,OAAOH,SAAsBA,SAASA,OAAO,MAAM;IACtE,MAAMI,oBAAoB,CAAC,WAAW,EAAEF,eAAe,iCAAiC,EAAEC,WAAW,kBAAkB,CAAC;IAExH,IAAI,AAAkB,YAAlB,OAAOH,QACT,OAAOI;IAGT,OAAO;QACL,GAAGJ,MAAM;QACT,QAAQI;IACV;AACF"}

package/dist/es/agent/task-builder.mjs

@@ -331,11 +331,20 @@ class TaskBuilder {
                         cacheToSave: currentCacheEntry
                     }
                 };
-                onResult?.(element);
+                const promptDisplay = param.promptDisplay;
+                const elementForAction = promptDisplay ? {
+                    ...element,
+                    description: promptDisplay
+                } : element;
+                if (promptDisplay && locateDump?.matchedElement) locateDump.matchedElement = locateDump.matchedElement.map((matchedElement)=>({
+                        ...matchedElement,
+                        description: promptDisplay
+                    }));
+                onResult?.(elementForAction);
                 return {
                     output: {
                         element: {
-                            ...element,
+                            ...elementForAction,
                             dpr: uiContext.deprecatedDpr
                         }
                     },

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifPlanLocateParamHasLocatedPixelBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [\n    element.center?.[0],\n    element.center?.[1],\n    element.rect?.left,\n    element.rect?.top,\n    element.rect?.width,\n    element.rect?.height,\n  ];\n  if (\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  if (element.rect.width <= 0 || element.rect.height <= 0) {\n    return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    const requiredLocateFields = findAllMidsceneLocatorField(\n      action.paramSchema,\n      true,\n    );\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelBbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n          param,\n        )\n          ? param\n          : undefined;\n\n        assert(\n          param?.prompt || paramWithLocatedPixelBbox,\n          `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelBbox\n          ? matchElementFromPlan(paramWithLocatedPixelBbox)\n          : undefined;\n\n        // from locatedPixelBbox (direct plan hit)\n        // when deepLocate is enabled, locatedPixelBbox should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,4BAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEwB,qCAAqC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIrB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,iBAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOoC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpChE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDZ,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMsB,4BAA4B3B,qCAChC/B,SAEEA,QACAI;gBAEJ8B,OACElC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACsC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIa;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3B,KAAK,KAAK,GAAG4B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD3B,KAAK,UAAU,GAAG2B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB3B,KAAK,eAAe,GAAG4B,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,oCACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF5B,mBAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASsC;wBACT0B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAIb,MAAMqC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B9F,iBAC/BoF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD3D,MACE,8DACAM,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIrB,iBAAiBkG,UAAU;wBAC7BpG,MACE,uCACAqF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEtF,MACE,yDACAqF;gBAGN,EAAE,OAAOxB,OAAO;oBACd7D,MAAM,mCAAmC6D;gBAC3C;qBAEA7D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIb,MAAM,CAAC,mBAAmB,EAAE9C,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW5D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK4C,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IApkBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTpE,WAAW,EACXqE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGpE;QACnB,IAAI,CAAC,eAAe,GAAGqE;IACzB;AAyjBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifPlanLocateParamHasLocatedPixelBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [\n    element.center?.[0],\n    element.center?.[1],\n    element.rect?.left,\n    element.rect?.top,\n    element.rect?.width,\n    element.rect?.height,\n  ];\n  if (\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  if (element.rect.width <= 0 || element.rect.height <= 0) {\n    return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    const requiredLocateFields = findAllMidsceneLocatorField(\n      action.paramSchema,\n      true,\n    );\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelBbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n          param,\n        )\n          ? param\n          : undefined;\n\n        assert(\n          param?.prompt || paramWithLocatedPixelBbox,\n          `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelBbox\n          ? matchElementFromPlan(paramWithLocatedPixelBbox)\n          : undefined;\n\n        // from locatedPixelBbox (direct plan hit)\n        // when deepLocate is enabled, locatedPixelBbox should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        const promptDisplay = param.promptDisplay;\n        const elementForAction = promptDisplay\n          ? {\n              ...element,\n              description: promptDisplay,\n            }\n          : element;\n\n        if (promptDisplay && locateDump?.matchedElement) {\n          locateDump.matchedElement = locateDump.matchedElement.map(\n            (matchedElement) => ({\n              ...matchedElement,\n              description: promptDisplay,\n            }),\n          );\n        }\n\n        onResult?.(elementForAction);\n\n        return {\n          output: {\n            element: {\n              ...elementForAction,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,4BAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEwB,qCAAqC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIrB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,iBAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOoC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpChE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDZ,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMsB,4BAA4B3B,qCAChC/B,SAEEA,QACAI;gBAEJ8B,OACElC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACsC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIa;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3B,KAAK,KAAK,GAAG4B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD3B,KAAK,UAAU,GAAG2B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB3B,KAAK,eAAe,GAAG4B,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,oCACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF5B,mBAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASsC;wBACT0B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAIb,MAAMqC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B9F,iBAC/BoF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD3D,MACE,8DACAM,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIrB,iBAAiBkG,UAAU;wBAC7BpG,MACE,uCACAqF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEtF,MACE,yDACAqF;gBAGN,EAAE,OAAOxB,OAAO;oBACd7D,MAAM,mCAAmC6D;gBAC3C;qBAEA7D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIb,MAAM,CAAC,mBAAmB,EAAE9C,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF,MAAMK,gBAAgB1F,MAAM,aAAa;gBACzC,MAAM2F,mBAAmBD,gBACrB;oBACE,GAAGhG,OAAO;oBACV,aAAagG;gBACf,IACAhG;gBAEJ,IAAIgG,iBAAiB/B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAACiC,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJpC,WAAWqC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKrD,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IArlBA,YAAY,EACVoC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTvE,WAAW,EACXwE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGvE;QACnB,IAAI,CAAC,eAAe,GAAGwE;IACzB;AA0kBF"}

package/dist/es/agent/tasks.mjs

@@ -7,6 +7,7 @@ import { ServiceError, aiActProgressScope } from "../types.mjs";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { ExecutionSession } from "./execution-session.mjs";
+import { withFileChooser } from "./file-chooser.mjs";
 import { createAiActActionReporter, errorMessageForAiAct } from "./progress/index.mjs";
 import { TaskBuilder, locatePlanForLocate } from "./task-builder.mjs";
 import { setTimingFieldOnce } from "../task-timing.mjs";
@@ -381,6 +382,9 @@ class TaskExecutor {
             subType: type,
             param: {
                 domIncluded: opt?.domIncluded,
+                ...opt?.context !== void 0 ? {
+                    context: opt.context
+                } : {},
                 dataDemand: multimodalPrompt ? {
                     demand,
                     multimodalPrompt
@@ -565,22 +569,6 @@ function recordAndReleaseScreenshotSequence(task, uiContext) {
     }
     if (uiContext?.screenshotSequence) uiContext.screenshotSequence = void 0;
 }
-async function withFileChooser(interfaceInstance, fileChooserAccept, action) {
-    if (!fileChooserAccept?.length) return action();
-    if (!interfaceInstance.registerFileChooserListener) throw new Error(`File upload is not supported on ${interfaceInstance.interfaceType}`);
-    const handler = async (chooser)=>{
-        await chooser.accept(fileChooserAccept);
-    };
-    const { dispose, getError } = await interfaceInstance.registerFileChooserListener(handler);
-    try {
-        const result = await action();
-        const error = await getError();
-        if (error) throw error;
-        return result;
-    } finally{
-        dispose();
-    }
-}
 export { TaskExecutionError, TaskExecutor, locatePlanForLocate, recordAndReleaseScreenshotSequence, withFileChooser };
 
 //# sourceMappingURL=tasks.mjs.map