@midscene/core 1.15.5 → 1.16.0

package/dist/es/agent/insight.mjs

@@ -39,7 +39,7 @@ class Insight {
         };
     }
     async aiQuery(demand, options = defaultQueryOptions) {
-        const modelRuntime = this.resolveModelRuntime();
+        const modelRuntime = this.resolveModelRuntime('extract');
         const executionOptions = this.executionOptions();
         const serviceOptions = this.withContext('aiQuery', options);
         const { output } = executionOptions ? await this.taskExecutor.createTypeQueryExecution('Query', demand, modelRuntime, serviceOptions, void 0, executionOptions) : await this.taskExecutor.createTypeQueryExecution('Query', demand, modelRuntime, serviceOptions);
@@ -60,7 +60,7 @@ class Insight {
     async queryPrimitive(type, apiName, prompt, options) {
         const { textPrompt, multimodalPrompt } = parsePrompt(prompt);
         const serviceOptions = this.withContext(apiName, options);
-        const { output } = await this.taskExecutor.createTypeQueryExecution(type, textPrompt, this.resolveModelRuntime(), serviceOptions, multimodalPrompt, this.executionOptions());
+        const { output } = await this.taskExecutor.createTypeQueryExecution(type, textPrompt, this.resolveModelRuntime('Boolean' === type ? 'boolean' : 'extract'), serviceOptions, multimodalPrompt, this.executionOptions());
         return output;
     }
     async aiAssert(assertion, message, options) {
@@ -75,7 +75,7 @@ class Insight {
         const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
         const assertionText = 'string' == typeof assertion ? assertion : assertion.prompt;
         try {
-            const { output, thought } = await this.taskExecutor.createTypeQueryExecution('Assert', textPrompt, this.resolveModelRuntime(), serviceOptions, multimodalPrompt, this.executionOptions(options, true));
+            const { output, thought } = await this.taskExecutor.createTypeQueryExecution('Assert', textPrompt, this.resolveModelRuntime('boolean'), serviceOptions, multimodalPrompt, this.executionOptions(options, true));
             const pass = Boolean(output);
             const failureMessage = pass ? void 0 : `Assertion failed: ${message || assertionText}\nReason: ${thought || '(no_reason)'}`;
             if (options?.keepRawResponse) return {

package/dist/es/agent/insight.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/insight.mjs","sources":["../../../src/agent/insight.ts"],"sourcesContent":["import type { TUserPrompt } from '@/ai-model/index';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport type {\n  AgentAssertResult,\n  AssertOptions,\n  InsightAPI,\n  QueryOptions,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport type { AiApiName } from '@midscene/shared/agent-tools/agent-context';\nimport { TaskExecutionError, type TaskExecutor } from './tasks';\nimport { parsePrompt } from './utils';\n\nconst defaultQueryOptions: QueryOptions = {\n  domIncluded: false,\n  screenshotIncluded: true,\n};\n\ntype InsightTaskExecutor = Pick<TaskExecutor, 'createTypeQueryExecution'>;\n\ntype InsightContextOptions = { context?: string };\n\n/** Execute read-only AI operations against either live or fixed UI context. */\nexport class Insight implements InsightAPI {\n  constructor(\n    private readonly taskExecutor: InsightTaskExecutor,\n    private readonly resolveModelRuntime: () => ModelRuntime,\n    private readonly resolveAgentContext: (\n      apiName: AiApiName,\n      callContext?: string,\n    ) => string | undefined,\n    private readonly getUIContext?: () => UIContext,\n  ) {}\n\n  private resolveContext(\n    apiName: AiApiName,\n    options?: InsightContextOptions,\n  ): string | undefined {\n    return this.resolveAgentContext(apiName, options?.context);\n  }\n\n  private withContext<T extends InsightContextOptions>(\n    apiName: AiApiName,\n    options: T,\n  ): T {\n    const resolvedContext = this.resolveContext(apiName, options);\n    if (resolvedContext === undefined) {\n      return options;\n    }\n\n    return {\n      ...options,\n      context: resolvedContext,\n    };\n  }\n\n  private executionOptions(\n    options?: AssertOptions,\n    includeAbortSignal = false,\n  ): { abortSignal?: AbortSignal; uiContext?: UIContext } | undefined {\n    const uiContext = this.getUIContext?.();\n    if (!uiContext && !includeAbortSignal && !options?.abortSignal) {\n      return undefined;\n    }\n    return {\n      ...(includeAbortSignal || options?.abortSignal\n        ? { abortSignal: options?.abortSignal }\n        : {}),\n      ...(uiContext ? { uiContext } : {}),\n    };\n  }\n\n  async aiQuery<ReturnType = any>(\n    demand: ServiceExtractParam,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<ReturnType> {\n    const modelRuntime = this.resolveModelRuntime();\n    const executionOptions = this.executionOptions();\n    const serviceOptions = this.withContext('aiQuery', options);\n    const { output } = executionOptions\n      ? await this.taskExecutor.createTypeQueryExecution(\n          'Query',\n          demand,\n          modelRuntime,\n          serviceOptions,\n          undefined,\n          executionOptions,\n        )\n      : await this.taskExecutor.createTypeQueryExecution(\n          'Query',\n          demand,\n          modelRuntime,\n          serviceOptions,\n        );\n    return output as ReturnType;\n  }\n\n  async aiBoolean(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<boolean> {\n    return this.queryPrimitive('Boolean', 'aiBoolean', prompt, options);\n  }\n\n  async aiNumber(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<number> {\n    return this.queryPrimitive('Number', 'aiNumber', prompt, options);\n  }\n\n  async aiString(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<string> {\n    return this.queryPrimitive('String', 'aiString', prompt, options);\n  }\n\n  async aiAsk(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<string> {\n    return this.queryPrimitive('String', 'aiAsk', prompt, options);\n  }\n\n  private async queryPrimitive<ReturnType>(\n    type: 'Boolean' | 'Number' | 'String',\n    apiName: 'aiBoolean' | 'aiNumber' | 'aiString' | 'aiAsk',\n    prompt: TUserPrompt,\n    options: QueryOptions,\n  ): Promise<ReturnType> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(prompt);\n    const serviceOptions = this.withContext(apiName, options);\n    const { output } = await this.taskExecutor.createTypeQueryExecution(\n      type,\n      textPrompt,\n      this.resolveModelRuntime(),\n      serviceOptions,\n      multimodalPrompt,\n      this.executionOptions(),\n    );\n    return output as ReturnType;\n  }\n\n  async aiAssert(\n    assertion: TUserPrompt,\n    message?: string,\n    options?: AssertOptions,\n  ): Promise<AgentAssertResult | undefined> {\n    const contextOptions = this.withContext('aiAssert', options ?? {});\n    const serviceOptions: QueryOptions = {\n      domIncluded: options?.domIncluded ?? defaultQueryOptions.domIncluded,\n      screenshotIncluded:\n        options?.screenshotIncluded ?? defaultQueryOptions.screenshotIncluded,\n      ...(contextOptions.context !== undefined\n        ? { context: contextOptions.context }\n        : {}),\n    };\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n    const assertionText =\n      typeof assertion === 'string' ? assertion : assertion.prompt;\n\n    try {\n      const { output, thought } =\n        await this.taskExecutor.createTypeQueryExecution<boolean>(\n          'Assert',\n          textPrompt,\n          this.resolveModelRuntime(),\n          serviceOptions,\n          multimodalPrompt,\n          this.executionOptions(options, true),\n        );\n\n      const pass = Boolean(output);\n      const failureMessage = pass\n        ? undefined\n        : `Assertion failed: ${message || assertionText}\\nReason: ${thought || '(no_reason)'}`;\n\n      if (options?.keepRawResponse) {\n        return { pass, thought, message: failureMessage };\n      }\n      if (!pass) {\n        throw new Error(failureMessage);\n      }\n    } catch (error) {\n      if (error instanceof TaskExecutionError) {\n        const thought = error.task?.thought;\n        const diagnosticMessage =\n          error.task?.errorMessage || error.cause.message;\n        const reason = thought || diagnosticMessage || '(no_reason)';\n        const failureMessage = `Assertion failed: ${message || assertionText}\\nReason: ${reason}`;\n\n        if (options?.keepRawResponse) {\n          return { pass: false, thought, message: failureMessage };\n        }\n        throw new Error(failureMessage, { cause: error.cause });\n      }\n      throw error;\n    }\n  }\n}\n"],"names":["defaultQueryOptions","Insight","apiName","options","resolvedContext","undefined","includeAbortSignal","uiContext","demand","modelRuntime","executionOptions","serviceOptions","output","prompt","type","textPrompt","multimodalPrompt","parsePrompt","assertion","message","contextOptions","assertionText","thought","pass","Boolean","failureMessage","Error","error","TaskExecutionError","diagnosticMessage","reason","taskExecutor","resolveModelRuntime","resolveAgentContext","getUIContext"],"mappings":";;;;;;;;;;;;AAcA,MAAMA,sBAAoC;IACxC,aAAa;IACb,oBAAoB;AACtB;AAOO,MAAMC;IAWH,eACNC,OAAkB,EAClBC,OAA+B,EACX;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAACD,SAASC,SAAS;IACpD;IAEQ,YACND,OAAkB,EAClBC,OAAU,EACP;QACH,MAAMC,kBAAkB,IAAI,CAAC,cAAc,CAACF,SAASC;QACrD,IAAIC,AAAoBC,WAApBD,iBACF,OAAOD;QAGT,OAAO;YACL,GAAGA,OAAO;YACV,SAASC;QACX;IACF;IAEQ,iBACND,OAAuB,EACvBG,qBAAqB,KAAK,EACwC;QAClE,MAAMC,YAAY,IAAI,CAAC,YAAY;QACnC,IAAI,CAACA,aAAa,CAACD,sBAAsB,CAACH,SAAS,aACjD;QAEF,OAAO;YACL,GAAIG,sBAAsBH,SAAS,cAC/B;gBAAE,aAAaA,SAAS;YAAY,IACpC,CAAC,CAAC;YACN,GAAII,YAAY;gBAAEA;YAAU,IAAI,CAAC,CAAC;QACpC;IACF;IAEA,MAAM,QACJC,MAA2B,EAC3BL,UAAwBH,mBAAmB,EACtB;QACrB,MAAMS,eAAe,IAAI,CAAC,mBAAmB;QAC7C,MAAMC,mBAAmB,IAAI,CAAC,gBAAgB;QAC9C,MAAMC,iBAAiB,IAAI,CAAC,WAAW,CAAC,WAAWR;QACnD,MAAM,EAAES,MAAM,EAAE,GAAGF,mBACf,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAC9C,SACAF,QACAC,cACAE,gBACAN,QACAK,oBAEF,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAC9C,SACAF,QACAC,cACAE;QAEN,OAAOC;IACT;IAEA,MAAM,UACJC,MAAmB,EACnBV,UAAwBH,mBAAmB,EACzB;QAClB,OAAO,IAAI,CAAC,cAAc,CAAC,WAAW,aAAaa,QAAQV;IAC7D;IAEA,MAAM,SACJU,MAAmB,EACnBV,UAAwBH,mBAAmB,EAC1B;QACjB,OAAO,IAAI,CAAC,cAAc,CAAC,UAAU,YAAYa,QAAQV;IAC3D;IAEA,MAAM,SACJU,MAAmB,EACnBV,UAAwBH,mBAAmB,EAC1B;QACjB,OAAO,IAAI,CAAC,cAAc,CAAC,UAAU,YAAYa,QAAQV;IAC3D;IAEA,MAAM,MACJU,MAAmB,EACnBV,UAAwBH,mBAAmB,EAC1B;QACjB,OAAO,IAAI,CAAC,cAAc,CAAC,UAAU,SAASa,QAAQV;IACxD;IAEA,MAAc,eACZW,IAAqC,EACrCZ,OAAwD,EACxDW,MAAmB,EACnBV,OAAqB,EACA;QACrB,MAAM,EAAEY,UAAU,EAAEC,gBAAgB,EAAE,GAAGC,YAAYJ;QACrD,MAAMF,iBAAiB,IAAI,CAAC,WAAW,CAACT,SAASC;QACjD,MAAM,EAAES,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CACjEE,MACAC,YACA,IAAI,CAAC,mBAAmB,IACxBJ,gBACAK,kBACA,IAAI,CAAC,gBAAgB;QAEvB,OAAOJ;IACT;IAEA,MAAM,SACJM,SAAsB,EACtBC,OAAgB,EAChBhB,OAAuB,EACiB;QACxC,MAAMiB,iBAAiB,IAAI,CAAC,WAAW,CAAC,YAAYjB,WAAW,CAAC;QAChE,MAAMQ,iBAA+B;YACnC,aAAaR,SAAS,eAAeH,oBAAoB,WAAW;YACpE,oBACEG,SAAS,sBAAsBH,oBAAoB,kBAAkB;YACvE,GAAIoB,AAA2Bf,WAA3Be,eAAe,OAAO,GACtB;gBAAE,SAASA,eAAe,OAAO;YAAC,IAClC,CAAC,CAAC;QACR;QACA,MAAM,EAAEL,UAAU,EAAEC,gBAAgB,EAAE,GAAGC,YAAYC;QACrD,MAAMG,gBACJ,AAAqB,YAArB,OAAOH,YAAyBA,YAAYA,UAAU,MAAM;QAE9D,IAAI;YACF,MAAM,EAAEN,MAAM,EAAEU,OAAO,EAAE,GACvB,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAC9C,UACAP,YACA,IAAI,CAAC,mBAAmB,IACxBJ,gBACAK,kBACA,IAAI,CAAC,gBAAgB,CAACb,SAAS;YAGnC,MAAMoB,OAAOC,QAAQZ;YACrB,MAAMa,iBAAiBF,OACnBlB,SACA,CAAC,kBAAkB,EAAEc,WAAWE,cAAc,UAAU,EAAEC,WAAW,eAAe;YAExF,IAAInB,SAAS,iBACX,OAAO;gBAAEoB;gBAAMD;gBAAS,SAASG;YAAe;YAElD,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;QAEpB,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,oBAAoB;gBACvC,MAAMN,UAAUK,MAAM,IAAI,EAAE;gBAC5B,MAAME,oBACJF,MAAM,IAAI,EAAE,gBAAgBA,MAAM,KAAK,CAAC,OAAO;gBACjD,MAAMG,SAASR,WAAWO,qBAAqB;gBAC/C,MAAMJ,iBAAiB,CAAC,kBAAkB,EAAEN,WAAWE,cAAc,UAAU,EAAES,QAAQ;gBAEzF,IAAI3B,SAAS,iBACX,OAAO;oBAAE,MAAM;oBAAOmB;oBAAS,SAASG;gBAAe;gBAEzD,MAAM,IAAIC,MAAMD,gBAAgB;oBAAE,OAAOE,MAAM,KAAK;gBAAC;YACvD;YACA,MAAMA;QACR;IACF;IA/KA,YACmBI,YAAiC,EACjCC,mBAAuC,EACvCC,mBAGM,EACNC,YAA8B,CAC/C;;;;;aAPiBH,YAAY,GAAZA;aACAC,mBAAmB,GAAnBA;aACAC,mBAAmB,GAAnBA;aAIAC,YAAY,GAAZA;IAChB;AAwKL"}
+{"version":3,"file":"agent/insight.mjs","sources":["../../../src/agent/insight.ts"],"sourcesContent":["import type { TUserPrompt } from '@/ai-model/index';\nimport type { ModelTask } from '@/ai-model/model-adapter/types';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport type {\n  AgentAssertResult,\n  AssertOptions,\n  InsightAPI,\n  QueryOptions,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport type { AiApiName } from '@midscene/shared/agent-tools/agent-context';\nimport { TaskExecutionError, type TaskExecutor } from './tasks';\nimport { parsePrompt } from './utils';\n\nconst defaultQueryOptions: QueryOptions = {\n  domIncluded: false,\n  screenshotIncluded: true,\n};\n\ntype InsightTaskExecutor = Pick<TaskExecutor, 'createTypeQueryExecution'>;\n\ntype InsightContextOptions = { context?: string };\n\n/** Execute read-only AI operations against either live or fixed UI context. */\nexport class Insight implements InsightAPI {\n  constructor(\n    private readonly taskExecutor: InsightTaskExecutor,\n    private readonly resolveModelRuntime: (task?: ModelTask) => ModelRuntime,\n    private readonly resolveAgentContext: (\n      apiName: AiApiName,\n      callContext?: string,\n    ) => string | undefined,\n    private readonly getUIContext?: () => UIContext,\n  ) {}\n\n  private resolveContext(\n    apiName: AiApiName,\n    options?: InsightContextOptions,\n  ): string | undefined {\n    return this.resolveAgentContext(apiName, options?.context);\n  }\n\n  private withContext<T extends InsightContextOptions>(\n    apiName: AiApiName,\n    options: T,\n  ): T {\n    const resolvedContext = this.resolveContext(apiName, options);\n    if (resolvedContext === undefined) {\n      return options;\n    }\n\n    return {\n      ...options,\n      context: resolvedContext,\n    };\n  }\n\n  private executionOptions(\n    options?: AssertOptions,\n    includeAbortSignal = false,\n  ): { abortSignal?: AbortSignal; uiContext?: UIContext } | undefined {\n    const uiContext = this.getUIContext?.();\n    if (!uiContext && !includeAbortSignal && !options?.abortSignal) {\n      return undefined;\n    }\n    return {\n      ...(includeAbortSignal || options?.abortSignal\n        ? { abortSignal: options?.abortSignal }\n        : {}),\n      ...(uiContext ? { uiContext } : {}),\n    };\n  }\n\n  async aiQuery<ReturnType = any>(\n    demand: ServiceExtractParam,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<ReturnType> {\n    const modelRuntime = this.resolveModelRuntime('extract');\n    const executionOptions = this.executionOptions();\n    const serviceOptions = this.withContext('aiQuery', options);\n    const { output } = executionOptions\n      ? await this.taskExecutor.createTypeQueryExecution(\n          'Query',\n          demand,\n          modelRuntime,\n          serviceOptions,\n          undefined,\n          executionOptions,\n        )\n      : await this.taskExecutor.createTypeQueryExecution(\n          'Query',\n          demand,\n          modelRuntime,\n          serviceOptions,\n        );\n    return output as ReturnType;\n  }\n\n  async aiBoolean(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<boolean> {\n    return this.queryPrimitive('Boolean', 'aiBoolean', prompt, options);\n  }\n\n  async aiNumber(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<number> {\n    return this.queryPrimitive('Number', 'aiNumber', prompt, options);\n  }\n\n  async aiString(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<string> {\n    return this.queryPrimitive('String', 'aiString', prompt, options);\n  }\n\n  async aiAsk(\n    prompt: TUserPrompt,\n    options: QueryOptions = defaultQueryOptions,\n  ): Promise<string> {\n    return this.queryPrimitive('String', 'aiAsk', prompt, options);\n  }\n\n  private async queryPrimitive<ReturnType>(\n    type: 'Boolean' | 'Number' | 'String',\n    apiName: 'aiBoolean' | 'aiNumber' | 'aiString' | 'aiAsk',\n    prompt: TUserPrompt,\n    options: QueryOptions,\n  ): Promise<ReturnType> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(prompt);\n    const serviceOptions = this.withContext(apiName, options);\n    const { output } = await this.taskExecutor.createTypeQueryExecution(\n      type,\n      textPrompt,\n      this.resolveModelRuntime(type === 'Boolean' ? 'boolean' : 'extract'),\n      serviceOptions,\n      multimodalPrompt,\n      this.executionOptions(),\n    );\n    return output as ReturnType;\n  }\n\n  async aiAssert(\n    assertion: TUserPrompt,\n    message?: string,\n    options?: AssertOptions,\n  ): Promise<AgentAssertResult | undefined> {\n    const contextOptions = this.withContext('aiAssert', options ?? {});\n    const serviceOptions: QueryOptions = {\n      domIncluded: options?.domIncluded ?? defaultQueryOptions.domIncluded,\n      screenshotIncluded:\n        options?.screenshotIncluded ?? defaultQueryOptions.screenshotIncluded,\n      ...(contextOptions.context !== undefined\n        ? { context: contextOptions.context }\n        : {}),\n    };\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n    const assertionText =\n      typeof assertion === 'string' ? assertion : assertion.prompt;\n\n    try {\n      const { output, thought } =\n        await this.taskExecutor.createTypeQueryExecution<boolean>(\n          'Assert',\n          textPrompt,\n          this.resolveModelRuntime('boolean'),\n          serviceOptions,\n          multimodalPrompt,\n          this.executionOptions(options, true),\n        );\n\n      const pass = Boolean(output);\n      const failureMessage = pass\n        ? undefined\n        : `Assertion failed: ${message || assertionText}\\nReason: ${thought || '(no_reason)'}`;\n\n      if (options?.keepRawResponse) {\n        return { pass, thought, message: failureMessage };\n      }\n      if (!pass) {\n        throw new Error(failureMessage);\n      }\n    } catch (error) {\n      if (error instanceof TaskExecutionError) {\n        const thought = error.task?.thought;\n        const diagnosticMessage =\n          error.task?.errorMessage || error.cause.message;\n        const reason = thought || diagnosticMessage || '(no_reason)';\n        const failureMessage = `Assertion failed: ${message || assertionText}\\nReason: ${reason}`;\n\n        if (options?.keepRawResponse) {\n          return { pass: false, thought, message: failureMessage };\n        }\n        throw new Error(failureMessage, { cause: error.cause });\n      }\n      throw error;\n    }\n  }\n}\n"],"names":["defaultQueryOptions","Insight","apiName","options","resolvedContext","undefined","includeAbortSignal","uiContext","demand","modelRuntime","executionOptions","serviceOptions","output","prompt","type","textPrompt","multimodalPrompt","parsePrompt","assertion","message","contextOptions","assertionText","thought","pass","Boolean","failureMessage","Error","error","TaskExecutionError","diagnosticMessage","reason","taskExecutor","resolveModelRuntime","resolveAgentContext","getUIContext"],"mappings":";;;;;;;;;;;;AAeA,MAAMA,sBAAoC;IACxC,aAAa;IACb,oBAAoB;AACtB;AAOO,MAAMC;IAWH,eACNC,OAAkB,EAClBC,OAA+B,EACX;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAACD,SAASC,SAAS;IACpD;IAEQ,YACND,OAAkB,EAClBC,OAAU,EACP;QACH,MAAMC,kBAAkB,IAAI,CAAC,cAAc,CAACF,SAASC;QACrD,IAAIC,AAAoBC,WAApBD,iBACF,OAAOD;QAGT,OAAO;YACL,GAAGA,OAAO;YACV,SAASC;QACX;IACF;IAEQ,iBACND,OAAuB,EACvBG,qBAAqB,KAAK,EACwC;QAClE,MAAMC,YAAY,IAAI,CAAC,YAAY;QACnC,IAAI,CAACA,aAAa,CAACD,sBAAsB,CAACH,SAAS,aACjD;QAEF,OAAO;YACL,GAAIG,sBAAsBH,SAAS,cAC/B;gBAAE,aAAaA,SAAS;YAAY,IACpC,CAAC,CAAC;YACN,GAAII,YAAY;gBAAEA;YAAU,IAAI,CAAC,CAAC;QACpC;IACF;IAEA,MAAM,QACJC,MAA2B,EAC3BL,UAAwBH,mBAAmB,EACtB;QACrB,MAAMS,eAAe,IAAI,CAAC,mBAAmB,CAAC;QAC9C,MAAMC,mBAAmB,IAAI,CAAC,gBAAgB;QAC9C,MAAMC,iBAAiB,IAAI,CAAC,WAAW,CAAC,WAAWR;QACnD,MAAM,EAAES,MAAM,EAAE,GAAGF,mBACf,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAC9C,SACAF,QACAC,cACAE,gBACAN,QACAK,oBAEF,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAC9C,SACAF,QACAC,cACAE;QAEN,OAAOC;IACT;IAEA,MAAM,UACJC,MAAmB,EACnBV,UAAwBH,mBAAmB,EACzB;QAClB,OAAO,IAAI,CAAC,cAAc,CAAC,WAAW,aAAaa,QAAQV;IAC7D;IAEA,MAAM,SACJU,MAAmB,EACnBV,UAAwBH,mBAAmB,EAC1B;QACjB,OAAO,IAAI,CAAC,cAAc,CAAC,UAAU,YAAYa,QAAQV;IAC3D;IAEA,MAAM,SACJU,MAAmB,EACnBV,UAAwBH,mBAAmB,EAC1B;QACjB,OAAO,IAAI,CAAC,cAAc,CAAC,UAAU,YAAYa,QAAQV;IAC3D;IAEA,MAAM,MACJU,MAAmB,EACnBV,UAAwBH,mBAAmB,EAC1B;QACjB,OAAO,IAAI,CAAC,cAAc,CAAC,UAAU,SAASa,QAAQV;IACxD;IAEA,MAAc,eACZW,IAAqC,EACrCZ,OAAwD,EACxDW,MAAmB,EACnBV,OAAqB,EACA;QACrB,MAAM,EAAEY,UAAU,EAAEC,gBAAgB,EAAE,GAAGC,YAAYJ;QACrD,MAAMF,iBAAiB,IAAI,CAAC,WAAW,CAACT,SAASC;QACjD,MAAM,EAAES,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CACjEE,MACAC,YACA,IAAI,CAAC,mBAAmB,CAACD,AAAS,cAATA,OAAqB,YAAY,YAC1DH,gBACAK,kBACA,IAAI,CAAC,gBAAgB;QAEvB,OAAOJ;IACT;IAEA,MAAM,SACJM,SAAsB,EACtBC,OAAgB,EAChBhB,OAAuB,EACiB;QACxC,MAAMiB,iBAAiB,IAAI,CAAC,WAAW,CAAC,YAAYjB,WAAW,CAAC;QAChE,MAAMQ,iBAA+B;YACnC,aAAaR,SAAS,eAAeH,oBAAoB,WAAW;YACpE,oBACEG,SAAS,sBAAsBH,oBAAoB,kBAAkB;YACvE,GAAIoB,AAA2Bf,WAA3Be,eAAe,OAAO,GACtB;gBAAE,SAASA,eAAe,OAAO;YAAC,IAClC,CAAC,CAAC;QACR;QACA,MAAM,EAAEL,UAAU,EAAEC,gBAAgB,EAAE,GAAGC,YAAYC;QACrD,MAAMG,gBACJ,AAAqB,YAArB,OAAOH,YAAyBA,YAAYA,UAAU,MAAM;QAE9D,IAAI;YACF,MAAM,EAAEN,MAAM,EAAEU,OAAO,EAAE,GACvB,MAAM,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAC9C,UACAP,YACA,IAAI,CAAC,mBAAmB,CAAC,YACzBJ,gBACAK,kBACA,IAAI,CAAC,gBAAgB,CAACb,SAAS;YAGnC,MAAMoB,OAAOC,QAAQZ;YACrB,MAAMa,iBAAiBF,OACnBlB,SACA,CAAC,kBAAkB,EAAEc,WAAWE,cAAc,UAAU,EAAEC,WAAW,eAAe;YAExF,IAAInB,SAAS,iBACX,OAAO;gBAAEoB;gBAAMD;gBAAS,SAASG;YAAe;YAElD,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;QAEpB,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,oBAAoB;gBACvC,MAAMN,UAAUK,MAAM,IAAI,EAAE;gBAC5B,MAAME,oBACJF,MAAM,IAAI,EAAE,gBAAgBA,MAAM,KAAK,CAAC,OAAO;gBACjD,MAAMG,SAASR,WAAWO,qBAAqB;gBAC/C,MAAMJ,iBAAiB,CAAC,kBAAkB,EAAEN,WAAWE,cAAc,UAAU,EAAES,QAAQ;gBAEzF,IAAI3B,SAAS,iBACX,OAAO;oBAAE,MAAM;oBAAOmB;oBAAS,SAASG;gBAAe;gBAEzD,MAAM,IAAIC,MAAMD,gBAAgB;oBAAE,OAAOE,MAAM,KAAK;gBAAC;YACvD;YACA,MAAMA;QACR;IACF;IA/KA,YACmBI,YAAiC,EACjCC,mBAAuD,EACvDC,mBAGM,EACNC,YAA8B,CAC/C;;;;;aAPiBH,YAAY,GAAZA;aACAC,mBAAmB,GAAnBA;aACAC,mBAAmB,GAAnBA;aAIAC,YAAY,GAAZA;IAChB;AAwKL"}

package/dist/es/agent/task-builder.mjs

@@ -49,13 +49,13 @@ function locatePlanForLocate(param) {
     return locatePlan;
 }
 class TaskBuilder {
-    async build(plans, planningModel, defaultModel, options) {
+    async build(plans, planningModel, locateModel, options) {
         const tasks = [];
         const cacheable = options?.cacheable;
         const context = {
             tasks,
             planningModel,
-            defaultModel,
+            locateModel,
             cacheable,
             multimodalPrompt: options?.multimodalPrompt,
             deepLocate: options?.deepLocate,
@@ -209,7 +209,7 @@ class TaskBuilder {
         context.tasks.push(task);
     }
     createLocateTask(plan, detailedLocateParam, context, onResult) {
-        const { cacheable, defaultModel, deepLocate } = context;
+        const { cacheable, locateModel, deepLocate } = context;
         let locateParam = normalizeLocateParam(detailedLocateParam);
         if (context.multimodalPrompt?.images?.length && locateParam.prompt && !userPromptToMultimodalPrompt(locateParam.prompt)?.images?.length) locateParam = {
             ...locateParam,
@@ -256,9 +256,9 @@ class TaskBuilder {
                         rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,
                         searchAreaRawChoiceMessage: dump.taskInfo?.searchAreaRawChoiceMessage
                     };
-                    task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');
+                    task.usage = withUsageIntent(dump.taskInfo?.usage, 'locate');
                     task.searchArea = dump.taskInfo?.searchArea;
-                    if (dump.taskInfo?.searchAreaUsage) task.searchAreaUsage = withUsageIntent(dump.taskInfo.searchAreaUsage, 'default');
+                    if (dump.taskInfo?.searchAreaUsage) task.searchAreaUsage = withUsageIntent(dump.taskInfo.searchAreaUsage, 'locate');
                     if (dump.taskInfo?.reasoning_content) task.reasoning_content = dump.taskInfo.reasoning_content;
                 };
                 const planLocatedElement = paramWithLocatedPixelResult ? matchElementFromPlan(paramWithLocatedPixelResult) : void 0;
@@ -297,7 +297,7 @@ class TaskBuilder {
                     locateResult = await this.service.locate(locateParam, {
                         context: uiContext,
                         planLocatedElement
-                    }, defaultModel, abortSignal);
+                    }, locateModel, abortSignal);
                     applyDump(locateResult.dump);
                     elementFromAiLocate = locateResult.element;
                 } catch (error) {
@@ -321,9 +321,13 @@ class TaskBuilder {
                         Math.round(element.center[0] / shrunkShotToLogicalRatio),
                         Math.round(element.center[1] / shrunkShotToLogicalRatio)
                     ];
+                    const defaultModel = this.getDefaultModelRuntime();
                     const feature = await this.interface.cacheFeatureForPoint(pointForCache, {
                         targetDescription: 'string' == typeof locateParam.prompt ? locateParam.prompt : locateParam.prompt?.prompt,
-                        modelRuntime: defaultModel
+                        modelRuntime: {
+                            ...defaultModel,
+                            executionId: locateModel.executionId
+                        }
                     });
                     if (hasNonEmptyCache(feature)) {
                         debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
@@ -385,14 +389,16 @@ class TaskBuilder {
         };
         return taskLocator;
     }
-    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction, actionReadiness, operationResource }){
+    constructor({ getDefaultModelRuntime, interfaceInstance, service, taskCache, actionSpace, waitAfterAction, actionReadiness, operationResource }){
         _define_property(this, "interface", void 0);
         _define_property(this, "service", void 0);
+        _define_property(this, "getDefaultModelRuntime", void 0);
         _define_property(this, "taskCache", void 0);
         _define_property(this, "actionSpace", void 0);
         _define_property(this, "waitAfterAction", void 0);
         _define_property(this, "actionReadiness", void 0);
         _define_property(this, "operationResource", void 0);
+        this.getDefaultModelRuntime = getDefaultModelRuntime;
         this.interface = interfaceInstance;
         this.service = service;
         this.taskCache = taskCache;

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n  type TMultimodalPrompt,\n  findActionInActionSpaceOrThrow,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n  validateRequiredLocateFields,\n} from '@/common';\nimport { type AbstractInterface, defineActionSleep } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ActionReadiness } from './action-readiness';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifLocateParamHasLocatedPixelResult,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [element.center?.[0], element.center?.[1]];\n  if (\n    element.center?.length !== 2 ||\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<DetailedLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n  actionReadiness?: ActionReadiness;\n  operationResource?: object;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  multimodalPrompt?: TMultimodalPrompt;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  multimodalPrompt?: TMultimodalPrompt;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n  private readonly actionReadiness?: ActionReadiness;\n  private readonly operationResource: object;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n    actionReadiness,\n    operationResource,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n    this.actionReadiness = actionReadiness;\n    this.operationResource = operationResource ?? interfaceInstance;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      multimodalPrompt: options?.multimodalPrompt,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<DetailedLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      context.abortSignal?.throwIfAborted();\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<DetailedLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    validateRequiredLocateFields(param, action.paramSchema);\n\n    locateFields.forEach((field) => {\n      if (param?.[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelResult is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n      executor: async (taskContext) => {\n        const abortSignal = context.abortSignal ?? taskContext.signal;\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        validateRequiredLocateFields(param, action.paramSchema);\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n        const parsedParam = (() => {\n          if (!action.paramSchema) return param;\n          try {\n            return parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        })();\n\n        const useDefaultWait = !this.actionReadiness;\n        taskContext.task.actionReadiness = useDefaultWait\n          ? 'default'\n          : 'custom';\n        abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = useDefaultWait\n          ? (action.delayBeforeRunner ?? 200)\n          : 0;\n        try {\n          const preparation = await Promise.allSettled([\n            this.interface.beforeInvokeAction?.(\n              action.name,\n              param,\n              abortSignal,\n            ),\n            delayBeforeRunner > 0\n              ? defineActionSleep(abortSignal).call({\n                  timeMs: delayBeforeRunner,\n                })\n              : Promise.resolve(),\n          ]);\n          const rejected = preparation.find(\n            (item) => item.status === 'rejected',\n          );\n          if (rejected?.status === 'rejected') throw rejected.reason;\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n        }\n\n        abortSignal?.throwIfAborted();\n        setTimingFieldOnce(timing, 'callActionStart');\n        debug('calling action', action.name);\n        const actionResult = await action.call.call(\n          this.interface,\n          parsedParam,\n          useDefaultWait\n            ? { ...taskContext, signal: abortSignal }\n            : { ...taskContext, signal: abortSignal, skipDefaultWait: true },\n        );\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n        abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n        setTimingFieldOnce(timing, 'waitForActionReadyStart');\n        try {\n          if (useDefaultWait) {\n            const delayAfterRunner =\n              action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n            if (delayAfterRunner > 0)\n              await defineActionSleep(abortSignal).call({\n                timeMs: delayAfterRunner,\n              });\n            await this.interface.defaultActionWait?.(\n              action.name,\n              parsedParam,\n              abortSignal,\n            );\n          } else {\n            await this.actionReadiness!.wait(\n              {\n                id: taskContext.task.taskId,\n                name: action.name,\n                param: parsedParam,\n              },\n              abortSignal,\n              this.operationResource,\n            );\n          }\n        } finally {\n          setTimingFieldOnce(timing, 'waitForActionReadyEnd');\n        }\n        abortSignal?.throwIfAborted();\n        try {\n          await this.interface.afterInvokeAction?.(\n            action.name,\n            parsedParam,\n            abortSignal,\n          );\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n        }\n        return { output: actionResult };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<DetailedLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    // Planning produces text locators; carry the user's reference images into\n    // independent grounding calls while preserving explicit locator images.\n    if (\n      context.multimodalPrompt?.images?.length &&\n      locateParam.prompt &&\n      !userPromptToMultimodalPrompt(locateParam.prompt)?.images?.length\n    ) {\n      locateParam = {\n        ...locateParam,\n        prompt: {\n          ...context.multimodalPrompt,\n          prompt: userPromptToString(locateParam.prompt),\n        },\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (taskContext) => {\n        const abortSignal = context.abortSignal ?? taskContext.signal;\n        abortSignal?.throwIfAborted();\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n          locateParam,\n        )\n          ? locateParam\n          : undefined;\n\n        assert(\n          locateParam?.prompt || paramWithLocatedPixelResult,\n          `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n            locateParam,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn(abortSignal);\n          abortSignal?.throwIfAborted();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelResult\n          ? matchElementFromPlan(paramWithLocatedPixelResult)\n          : undefined;\n\n        // from locatedPixelResult (direct plan hit)\n        // when deepLocate is enabled, locatedPixelResult should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = locateParam.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          locateParam.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [locateParam.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? transformLogicalElementToScreenshot(\n              {\n                center: [\n                  rectFromXpath.left + rectFromXpath.width / 2,\n                  rectFromXpath.top + rectFromXpath.height / 2,\n                ],\n                description:\n                  typeof locateParam.prompt === 'string'\n                    ? locateParam.prompt\n                    : locateParam.prompt?.prompt || '',\n                rect: rectFromXpath,\n              },\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = locateParam.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                locateParam.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              locateParam,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          locateParam.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              const pointForCache: [number, number] = [\n                Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                Math.round(element.center[1] / shrunkShotToLogicalRatio),\n              ];\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof locateParam.prompt === 'string'\n                      ? locateParam.prompt\n                      : locateParam.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${locateParam.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${locateParam.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelResult) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelResult:\n                paramWithLocatedPixelResult.locatedPixelResult,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: locateParam.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        const promptDisplay = locateParam.promptDisplay;\n        const elementForAction = promptDisplay\n          ? {\n              ...element,\n              description: promptDisplay,\n            }\n          : element;\n\n        if (promptDisplay && locateDump?.matchedElement) {\n          locateDump.matchedElement = locateDump.matchedElement.map(\n            (matchedElement) => ({\n              ...matchedElement,\n              description: promptDisplay,\n            }),\n          );\n        }\n\n        onResult?.(elementForAction);\n\n        return {\n          output: {\n            element: {\n              ...elementForAction,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","abortSignal","timing","uiContext","assert","shrunkShotToLogicalRatio","Error","parsedParam","parseActionParam","error","useDefaultWait","setTimingFieldOnce","delayBeforeRunner","preparation","Promise","defineActionSleep","rejected","item","originalError","String","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","userPromptToMultimodalPrompt","userPromptToString","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction","actionReadiness","operationResource"],"mappings":";;;;;;;;;;;;;;;;;;;AAwCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AA6BO,MAAMC;IA+BX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,kBAAkBF,SAAS;YAC3B,YAAYA,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxBM,QAAQ,WAAW,EAAE;YACrB,MAAMK,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnEI,6BAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,mCAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE5C,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7D,MAAME,SAASF,YAAY,IAAI,CAAC,MAAM;gBAEtC/C,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMG,YAAYH,YAAY,SAAS;gBACvCI,OAAOD,WAAW;gBAElBT,6BAA6B7B,OAAOyB,OAAO,WAAW;gBAEtD,MAAM,EAAEe,wBAAwB,EAAE,GAAGF;gBACrC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAGJ,MAAMC,cAAe,AAAC;oBACpB,IAAI,CAACjB,OAAO,WAAW,EAAE,OAAOzB;oBAChC,IAAI;wBACF,OAAO2C,iBAAiB3C,OAAOyB,OAAO,WAAW,EAAE;4BACjDe;wBACF;oBACF,EAAE,OAAOI,OAAY;wBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAEmB,MAAM,OAAO,CAAC,cAAc,EAAE9C,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAO4C;wBAAM;oBAEnB;gBACF;gBAEA,MAAMC,iBAAiB,CAAC,IAAI,CAAC,eAAe;gBAC5CV,YAAY,IAAI,CAAC,eAAe,GAAGU,iBAC/B,YACA;gBACJT,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3B,MAAMU,oBAAoBF,iBACrBpB,OAAO,iBAAiB,IAAI,MAC7B;gBACJ,IAAI;oBACF,MAAMuB,cAAc,MAAMC,QAAQ,UAAU,CAAC;wBAC3C,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAC/BxB,OAAO,IAAI,EACXzB,OACAoC;wBAEFW,oBAAoB,IAChBG,kBAAkBd,aAAa,IAAI,CAAC;4BAClC,QAAQW;wBACV,KACAE,QAAQ,OAAO;qBACpB;oBACD,MAAME,WAAWH,YAAY,IAAI,CAC/B,CAACI,OAASA,AAAgB,eAAhBA,KAAK,MAAM;oBAEvB,IAAID,UAAU,WAAW,YAAY,MAAMA,SAAS,MAAM;gBAC5D,EAAE,OAAOE,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,wCAAwC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC5G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBAEAD,aAAa;gBACbU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM8B,eAAe,MAAM9B,OAAO,IAAI,CAAC,IAAI,CACzC,IAAI,CAAC,SAAS,EACdiB,aACAG,iBACI;oBAAE,GAAGV,WAAW;oBAAE,QAAQC;gBAAY,IACtC;oBAAE,GAAGD,WAAW;oBAAE,QAAQC;oBAAa,iBAAiB;gBAAK;gBAEnEU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW8B;gBAC/CnB,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3BS,mBAAmBT,QAAQ;gBAC3B,IAAI;oBACF,IAAIQ,gBAAgB;wBAClB,MAAMW,mBACJ/B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;wBACrD,IAAI+B,mBAAmB,GACrB,MAAMN,kBAAkBd,aAAa,IAAI,CAAC;4BACxC,QAAQoB;wBACV;wBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpC/B,OAAO,IAAI,EACXiB,aACAN;oBAEJ,OACE,MAAM,IAAI,CAAC,eAAe,CAAE,IAAI,CAC9B;wBACE,IAAID,YAAY,IAAI,CAAC,MAAM;wBAC3B,MAAMV,OAAO,IAAI;wBACjB,OAAOiB;oBACT,GACAN,aACA,IAAI,CAAC,iBAAiB;gBAG5B,SAAU;oBACRU,mBAAmBT,QAAQ;gBAC7B;gBACAD,aAAa;gBACb,IAAI;oBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpCX,OAAO,IAAI,EACXiB,aACAN;gBAEJ,EAAE,OAAOiB,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,uCAAuC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC3G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBACA,OAAO;oBAAE,QAAQkB;gBAAa;YAChC;QACF;QAEAxC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCuC,mBAAiD,EACjD1C,OAAyB,EACzB2C,QAAgD,EACd;QAClC,MAAM,EAAE5C,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAE,GAAGY;QAEhD,IAAI4C,cAAc5D,qBAAqB0D;QAIvC,IACE1C,QAAQ,gBAAgB,EAAE,QAAQ,UAClC4C,YAAY,MAAM,IAClB,CAACC,6BAA6BD,YAAY,MAAM,GAAG,QAAQ,QAE3DA,cAAc;YACZ,GAAGA,WAAW;YACd,QAAQ;gBACN,GAAG5C,QAAQ,gBAAgB;gBAC3B,QAAQ8C,mBAAmBF,YAAY,MAAM;YAC/C;QACF;QAGF,IAAI7C,AAAcV,WAAdU,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,IAAIX,cAAc,CAACwD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMG,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOH;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7DC,aAAa;gBACb,MAAM,EAAEF,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEG,SAAS,EAAE,GAAGH;gBACpB,MAAM4B,8BAA8BhC,mCAClC4B,eAEEA,cACAvD;gBAEJmC,OACEoB,aAAa,UAAUI,6BACvB,CAAC,mEAAmE,EAAEjE,KAAK,SAAS,CAClF6D,cACC;gBAGL,IAAI,CAACrB,WAAW;oBACdA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAACF;oBAClDA,aAAa;gBACf;gBAEAG,OAAOD,WAAW;gBAElB,MAAM,EAAEE,wBAAwB,EAAE,GAAGF;gBAErC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAIJ,IAAIuB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbjC,KAAK,GAAG,GAAG;wBACTiC;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAjC,KAAK,KAAK,GAAGkC,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnDjC,KAAK,UAAU,GAAGiC,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjBjC,KAAK,eAAe,GAAGkC,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjBjC,KAAK,iBAAiB,GAAGiC,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,qBAAqBP,+BACrB3D;gBAKJ,MAAMmE,kBAAkBZ,YAAY,UAAU,GAC1CvD,SACAiE;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDb,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFc,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACd,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMe,mBAAmBD,gBACrBE,oCACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOd,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMc;gBACR,GACAjC,4BAEFpC;gBAEJ,MAAMwE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAclB,YAAY,MAAM;gBACtC,MAAMmB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAlB,YAAY,SAAS;gBAI7B,MAAMuB,mBAAmBF,yBACrBL,oCACEK,wBACAxC,4BAEFpC;gBAEJ,MAAM+E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM/C,SAASF,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACyC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACF1B,mBAAmBT,QAAQ;oBAC3B4B,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCN,aACA;wBACE,SAASrB;wBACT+B;oBACF,GACA1D,cACAyB;oBAEF8B,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOrB,OAAO;oBACd,IAAIA,iBAAiByC,cACnBnB,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRE,mBAAmBT,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJ6E,mBACAG,oBACAQ,oBACAE;gBAEF,IAAI1F,SAAS;oBACX,MAAM4F,uBAAuB7F,2BAA2BC;oBACxD,IAAI4F,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,aAAaC,sBAAsBtB;wBAE/C,MAAM,IAAIvB,MAAM6C;oBAClB;gBACF;gBAGA,MAAMC,2BAA2BjG,iBAC/BwF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACE9F,WACA,IAAI,CAAC,SAAS,IACd,CAACyF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C5B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM8B,gBAAkC;wBACtCC,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;wBAC/BkD,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;qBAChC;oBAED,MAAMmD,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO9B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAchD;oBAChB;oBAEF,IAAIrB,iBAAiBqG,UAAU;wBAC7BvG,MACE,uCACAyF,aACAc;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQd;4BACR,OAAOc;wBACT,GACAb;oBAEJ,OACE1F,MACE,yDACAyF;gBAGN,EAAE,OAAOjC,OAAO;oBACdxD,MAAM,mCAAmCwD;gBAC3C;qBAEAxD,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIsE,YACF,MAAM,IAAIqB,aACR,CAAC,oBAAoB,EAAE1B,YAAY,MAAM,EAAE,EAC3CK;oBAGJ,MAAM,IAAIvB,MAAM,CAAC,mBAAmB,EAAEkB,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAIiC;gBAEJ,IAAIpB,mBAAmBT,6BACrB6B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE7B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTgB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOjC,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIwB,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPb;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMK,gBAAgBlC,YAAY,aAAa;gBAC/C,MAAMmC,mBAAmBD,gBACrB;oBACE,GAAGnG,OAAO;oBACV,aAAamG;gBACf,IACAnG;gBAEJ,IAAImG,iBAAiB7B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAAC+B,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJnC,WAAWoC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKxD,UAAU,aAAa;wBAC9B;oBACF;oBACAsD;gBACF;YACF;QACF;QAEA,OAAO9B;IACT;IA3nBA,YAAY,EACVkC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACT1E,WAAW,EACX2E,eAAe,EACfC,eAAe,EACfC,iBAAiB,EACD,CAAE;QApBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QACA,uBAAiB,mBAAjB;QACA,uBAAiB,qBAAjB;QAWE,IAAI,CAAC,SAAS,GAAGL;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAG1E;QACnB,IAAI,CAAC,eAAe,GAAG2E;QACvB,IAAI,CAAC,eAAe,GAAGC;QACvB,IAAI,CAAC,iBAAiB,GAAGC,qBAAqBL;IAChD;AA4mBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n  type TMultimodalPrompt,\n  findActionInActionSpaceOrThrow,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n  validateRequiredLocateFields,\n} from '@/common';\nimport { type AbstractInterface, defineActionSleep } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ActionReadiness } from './action-readiness';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifLocateParamHasLocatedPixelResult,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [element.center?.[0], element.center?.[1]];\n  if (\n    element.center?.length !== 2 ||\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<DetailedLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  getDefaultModelRuntime: () => ModelRuntime;\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n  actionReadiness?: ActionReadiness;\n  operationResource?: object;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  multimodalPrompt?: TMultimodalPrompt;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  locateModel: ModelRuntime;\n  cacheable?: boolean;\n  multimodalPrompt?: TMultimodalPrompt;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n  private readonly getDefaultModelRuntime: () => ModelRuntime;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n  private readonly actionReadiness?: ActionReadiness;\n  private readonly operationResource: object;\n\n  constructor({\n    getDefaultModelRuntime,\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n    actionReadiness,\n    operationResource,\n  }: TaskBuilderDeps) {\n    this.getDefaultModelRuntime = getDefaultModelRuntime;\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n    this.actionReadiness = actionReadiness;\n    this.operationResource = operationResource ?? interfaceInstance;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    locateModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      locateModel,\n      cacheable,\n      multimodalPrompt: options?.multimodalPrompt,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<DetailedLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      context.abortSignal?.throwIfAborted();\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<DetailedLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    validateRequiredLocateFields(param, action.paramSchema);\n\n    locateFields.forEach((field) => {\n      if (param?.[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelResult is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n      executor: async (taskContext) => {\n        const abortSignal = context.abortSignal ?? taskContext.signal;\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        validateRequiredLocateFields(param, action.paramSchema);\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n        const parsedParam = (() => {\n          if (!action.paramSchema) return param;\n          try {\n            return parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        })();\n\n        const useDefaultWait = !this.actionReadiness;\n        taskContext.task.actionReadiness = useDefaultWait\n          ? 'default'\n          : 'custom';\n        abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = useDefaultWait\n          ? (action.delayBeforeRunner ?? 200)\n          : 0;\n        try {\n          const preparation = await Promise.allSettled([\n            this.interface.beforeInvokeAction?.(\n              action.name,\n              param,\n              abortSignal,\n            ),\n            delayBeforeRunner > 0\n              ? defineActionSleep(abortSignal).call({\n                  timeMs: delayBeforeRunner,\n                })\n              : Promise.resolve(),\n          ]);\n          const rejected = preparation.find(\n            (item) => item.status === 'rejected',\n          );\n          if (rejected?.status === 'rejected') throw rejected.reason;\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n        }\n\n        abortSignal?.throwIfAborted();\n        setTimingFieldOnce(timing, 'callActionStart');\n        debug('calling action', action.name);\n        const actionResult = await action.call.call(\n          this.interface,\n          parsedParam,\n          useDefaultWait\n            ? { ...taskContext, signal: abortSignal }\n            : { ...taskContext, signal: abortSignal, skipDefaultWait: true },\n        );\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n        abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n        setTimingFieldOnce(timing, 'waitForActionReadyStart');\n        try {\n          if (useDefaultWait) {\n            const delayAfterRunner =\n              action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n            if (delayAfterRunner > 0)\n              await defineActionSleep(abortSignal).call({\n                timeMs: delayAfterRunner,\n              });\n            await this.interface.defaultActionWait?.(\n              action.name,\n              parsedParam,\n              abortSignal,\n            );\n          } else {\n            await this.actionReadiness!.wait(\n              {\n                id: taskContext.task.taskId,\n                name: action.name,\n                param: parsedParam,\n              },\n              abortSignal,\n              this.operationResource,\n            );\n          }\n        } finally {\n          setTimingFieldOnce(timing, 'waitForActionReadyEnd');\n        }\n        abortSignal?.throwIfAborted();\n        try {\n          await this.interface.afterInvokeAction?.(\n            action.name,\n            parsedParam,\n            abortSignal,\n          );\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n        }\n        return { output: actionResult };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<DetailedLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, locateModel, deepLocate } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    // Planning produces text locators; carry the user's reference images into\n    // independent grounding calls while preserving explicit locator images.\n    if (\n      context.multimodalPrompt?.images?.length &&\n      locateParam.prompt &&\n      !userPromptToMultimodalPrompt(locateParam.prompt)?.images?.length\n    ) {\n      locateParam = {\n        ...locateParam,\n        prompt: {\n          ...context.multimodalPrompt,\n          prompt: userPromptToString(locateParam.prompt),\n        },\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (taskContext) => {\n        const abortSignal = context.abortSignal ?? taskContext.signal;\n        abortSignal?.throwIfAborted();\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n          locateParam,\n        )\n          ? locateParam\n          : undefined;\n\n        assert(\n          locateParam?.prompt || paramWithLocatedPixelResult,\n          `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n            locateParam,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn(abortSignal);\n          abortSignal?.throwIfAborted();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'locate');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'locate',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelResult\n          ? matchElementFromPlan(paramWithLocatedPixelResult)\n          : undefined;\n\n        // Separate planning normally returns target descriptions, but may still\n        // return coordinates. Reuse them even with a dedicated Locate model,\n        // preserving the existing direct-plan-hit behavior.\n        // when deepLocate is enabled, locatedPixelResult should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = locateParam.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          locateParam.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [locateParam.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? transformLogicalElementToScreenshot(\n              {\n                center: [\n                  rectFromXpath.left + rectFromXpath.width / 2,\n                  rectFromXpath.top + rectFromXpath.height / 2,\n                ],\n                description:\n                  typeof locateParam.prompt === 'string'\n                    ? locateParam.prompt\n                    : locateParam.prompt?.prompt || '',\n                rect: rectFromXpath,\n              },\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = locateParam.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                locateParam.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              locateParam,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              locateModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          locateParam.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              const pointForCache: [number, number] = [\n                Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                Math.round(element.center[1] / shrunkShotToLogicalRatio),\n              ];\n\n              const defaultModel = this.getDefaultModelRuntime();\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof locateParam.prompt === 'string'\n                      ? locateParam.prompt\n                      : locateParam.prompt?.prompt,\n                  modelRuntime: {\n                    ...defaultModel,\n                    executionId: locateModel.executionId,\n                  },\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${locateParam.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${locateParam.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelResult) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelResult:\n                paramWithLocatedPixelResult.locatedPixelResult,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: locateParam.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        const promptDisplay = locateParam.promptDisplay;\n        const elementForAction = promptDisplay\n          ? {\n              ...element,\n              description: promptDisplay,\n            }\n          : element;\n\n        if (promptDisplay && locateDump?.matchedElement) {\n          locateDump.matchedElement = locateDump.matchedElement.map(\n            (matchedElement) => ({\n              ...matchedElement,\n              description: promptDisplay,\n            }),\n          );\n        }\n\n        onResult?.(elementForAction);\n\n        return {\n          output: {\n            element: {\n              ...elementForAction,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","locateModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","abortSignal","timing","uiContext","assert","shrunkShotToLogicalRatio","Error","parsedParam","parseActionParam","error","useDefaultWait","setTimingFieldOnce","delayBeforeRunner","preparation","Promise","defineActionSleep","rejected","item","originalError","String","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","userPromptToMultimodalPrompt","userPromptToString","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","defaultModel","feature","hitBy","promptDisplay","elementForAction","matchedElement","getDefaultModelRuntime","interfaceInstance","service","taskCache","waitAfterAction","actionReadiness","operationResource"],"mappings":";;;;;;;;;;;;;;;;;;;AAwCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AA8BO,MAAMC;IAkCX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,WAAyB,EACzBC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,kBAAkBF,SAAS;YAC3B,YAAYA,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxBM,QAAQ,WAAW,EAAE;YACrB,MAAMK,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnEI,6BAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,mCAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE5C,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7D,MAAME,SAASF,YAAY,IAAI,CAAC,MAAM;gBAEtC/C,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMG,YAAYH,YAAY,SAAS;gBACvCI,OAAOD,WAAW;gBAElBT,6BAA6B7B,OAAOyB,OAAO,WAAW;gBAEtD,MAAM,EAAEe,wBAAwB,EAAE,GAAGF;gBACrC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAGJ,MAAMC,cAAe,AAAC;oBACpB,IAAI,CAACjB,OAAO,WAAW,EAAE,OAAOzB;oBAChC,IAAI;wBACF,OAAO2C,iBAAiB3C,OAAOyB,OAAO,WAAW,EAAE;4BACjDe;wBACF;oBACF,EAAE,OAAOI,OAAY;wBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAEmB,MAAM,OAAO,CAAC,cAAc,EAAE9C,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAO4C;wBAAM;oBAEnB;gBACF;gBAEA,MAAMC,iBAAiB,CAAC,IAAI,CAAC,eAAe;gBAC5CV,YAAY,IAAI,CAAC,eAAe,GAAGU,iBAC/B,YACA;gBACJT,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3B,MAAMU,oBAAoBF,iBACrBpB,OAAO,iBAAiB,IAAI,MAC7B;gBACJ,IAAI;oBACF,MAAMuB,cAAc,MAAMC,QAAQ,UAAU,CAAC;wBAC3C,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAC/BxB,OAAO,IAAI,EACXzB,OACAoC;wBAEFW,oBAAoB,IAChBG,kBAAkBd,aAAa,IAAI,CAAC;4BAClC,QAAQW;wBACV,KACAE,QAAQ,OAAO;qBACpB;oBACD,MAAME,WAAWH,YAAY,IAAI,CAC/B,CAACI,OAASA,AAAgB,eAAhBA,KAAK,MAAM;oBAEvB,IAAID,UAAU,WAAW,YAAY,MAAMA,SAAS,MAAM;gBAC5D,EAAE,OAAOE,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,wCAAwC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC5G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBAEAD,aAAa;gBACbU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM8B,eAAe,MAAM9B,OAAO,IAAI,CAAC,IAAI,CACzC,IAAI,CAAC,SAAS,EACdiB,aACAG,iBACI;oBAAE,GAAGV,WAAW;oBAAE,QAAQC;gBAAY,IACtC;oBAAE,GAAGD,WAAW;oBAAE,QAAQC;oBAAa,iBAAiB;gBAAK;gBAEnEU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW8B;gBAC/CnB,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3BS,mBAAmBT,QAAQ;gBAC3B,IAAI;oBACF,IAAIQ,gBAAgB;wBAClB,MAAMW,mBACJ/B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;wBACrD,IAAI+B,mBAAmB,GACrB,MAAMN,kBAAkBd,aAAa,IAAI,CAAC;4BACxC,QAAQoB;wBACV;wBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpC/B,OAAO,IAAI,EACXiB,aACAN;oBAEJ,OACE,MAAM,IAAI,CAAC,eAAe,CAAE,IAAI,CAC9B;wBACE,IAAID,YAAY,IAAI,CAAC,MAAM;wBAC3B,MAAMV,OAAO,IAAI;wBACjB,OAAOiB;oBACT,GACAN,aACA,IAAI,CAAC,iBAAiB;gBAG5B,SAAU;oBACRU,mBAAmBT,QAAQ;gBAC7B;gBACAD,aAAa;gBACb,IAAI;oBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpCX,OAAO,IAAI,EACXiB,aACAN;gBAEJ,EAAE,OAAOiB,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,uCAAuC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC3G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBACA,OAAO;oBAAE,QAAQkB;gBAAa;YAChC;QACF;QAEAxC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCuC,mBAAiD,EACjD1C,OAAyB,EACzB2C,QAAgD,EACd;QAClC,MAAM,EAAE5C,SAAS,EAAEH,WAAW,EAAER,UAAU,EAAE,GAAGY;QAE/C,IAAI4C,cAAc5D,qBAAqB0D;QAIvC,IACE1C,QAAQ,gBAAgB,EAAE,QAAQ,UAClC4C,YAAY,MAAM,IAClB,CAACC,6BAA6BD,YAAY,MAAM,GAAG,QAAQ,QAE3DA,cAAc;YACZ,GAAGA,WAAW;YACd,QAAQ;gBACN,GAAG5C,QAAQ,gBAAgB;gBAC3B,QAAQ8C,mBAAmBF,YAAY,MAAM;YAC/C;QACF;QAGF,IAAI7C,AAAcV,WAAdU,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,IAAIX,cAAc,CAACwD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMG,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOH;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7DC,aAAa;gBACb,MAAM,EAAEF,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEG,SAAS,EAAE,GAAGH;gBACpB,MAAM4B,8BAA8BhC,mCAClC4B,eAEEA,cACAvD;gBAEJmC,OACEoB,aAAa,UAAUI,6BACvB,CAAC,mEAAmE,EAAEjE,KAAK,SAAS,CAClF6D,cACC;gBAGL,IAAI,CAACrB,WAAW;oBACdA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAACF;oBAClDA,aAAa;gBACf;gBAEAG,OAAOD,WAAW;gBAElB,MAAM,EAAEE,wBAAwB,EAAE,GAAGF;gBAErC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAIJ,IAAIuB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbjC,KAAK,GAAG,GAAG;wBACTiC;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAjC,KAAK,KAAK,GAAGkC,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnDjC,KAAK,UAAU,GAAGiC,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjBjC,KAAK,eAAe,GAAGkC,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjBjC,KAAK,iBAAiB,GAAGiC,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,qBAAqBP,+BACrB3D;gBAOJ,MAAMmE,kBAAkBZ,YAAY,UAAU,GAC1CvD,SACAiE;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDb,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFc,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACd,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMe,mBAAmBD,gBACrBE,oCACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOd,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMc;gBACR,GACAjC,4BAEFpC;gBAEJ,MAAMwE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAclB,YAAY,MAAM;gBACtC,MAAMmB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAlB,YAAY,SAAS;gBAI7B,MAAMuB,mBAAmBF,yBACrBL,oCACEK,wBACAxC,4BAEFpC;gBAEJ,MAAM+E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM/C,SAASF,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACyC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACF1B,mBAAmBT,QAAQ;oBAC3B4B,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCN,aACA;wBACE,SAASrB;wBACT+B;oBACF,GACA1D,aACAyB;oBAEF8B,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOrB,OAAO;oBACd,IAAIA,iBAAiByC,cACnBnB,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRE,mBAAmBT,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJ6E,mBACAG,oBACAQ,oBACAE;gBAEF,IAAI1F,SAAS;oBACX,MAAM4F,uBAAuB7F,2BAA2BC;oBACxD,IAAI4F,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,aAAaC,sBAAsBtB;wBAE/C,MAAM,IAAIvB,MAAM6C;oBAClB;gBACF;gBAGA,MAAMC,2BAA2BjG,iBAC/BwF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACE9F,WACA,IAAI,CAAC,SAAS,IACd,CAACyF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C5B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM8B,gBAAkC;wBACtCC,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;wBAC/BkD,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;qBAChC;oBAED,MAAMmD,eAAe,IAAI,CAAC,sBAAsB;oBAChD,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDH,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO9B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc;4BACZ,GAAGgC,YAAY;4BACf,aAAahF,YAAY,WAAW;wBACtC;oBACF;oBAEF,IAAIrB,iBAAiBsG,UAAU;wBAC7BxG,MACE,uCACAyF,aACAe;wBAEFJ,oBAAoBI;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACE1F,MACE,yDACAyF;gBAGN,EAAE,OAAOjC,OAAO;oBACdxD,MAAM,mCAAmCwD;gBAC3C;qBAEAxD,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIsE,YACF,MAAM,IAAIqB,aACR,CAAC,oBAAoB,EAAE1B,YAAY,MAAM,EAAE,EAC3CK;oBAGJ,MAAM,IAAIvB,MAAM,CAAC,mBAAmB,EAAEkB,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAIkC;gBAEJ,IAAIrB,mBAAmBT,6BACrB8B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE9B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlC,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIwB,YACTU,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMM,gBAAgBnC,YAAY,aAAa;gBAC/C,MAAMoC,mBAAmBD,gBACrB;oBACE,GAAGpG,OAAO;oBACV,aAAaoG;gBACf,IACApG;gBAEJ,IAAIoG,iBAAiB9B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAACgC,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJpC,WAAWqC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKzD,UAAU,aAAa;wBAC9B;oBACF;oBACAuD;gBACF;YACF;QACF;QAEA,OAAO/B;IACT;IAnoBA,YAAY,EACVmC,sBAAsB,EACtBC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACT5E,WAAW,EACX6E,eAAe,EACfC,eAAe,EACfC,iBAAiB,EACD,CAAE;QAtBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QACA,uBAAiB,0BAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QACA,uBAAiB,mBAAjB;QACA,uBAAiB,qBAAjB;QAYE,IAAI,CAAC,sBAAsB,GAAGN;QAC9B,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAG5E;QACnB,IAAI,CAAC,eAAe,GAAG6E;QACvB,IAAI,CAAC,eAAe,GAAGC;QACvB,IAAI,CAAC,iBAAiB,GAAGC,qBAAqBL;IAChD;AAknBF"}

package/dist/es/agent/tasks.mjs

@@ -88,8 +88,8 @@ class TaskExecutor {
         else warnLog('useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.');
         return getReadableTimeString(format);
     }
-    async convertPlanToExecutable(plans, planningModel, defaultModel, options) {
-        return this.taskBuilder.build(plans, planningModel, defaultModel, options);
+    async convertPlanToExecutable(plans, planningModel, locateModel, options) {
+        return this.taskBuilder.build(plans, planningModel, locateModel, options);
     }
     async loadYamlFlowAsPlanning(userInstruction, yamlString, reportOptions) {
         const session = this.createExecutionSession(taskTitleStr(reportOptions?.type || 'Act', reportOptions?.prompt || userPromptToString(userInstruction)), {
@@ -157,7 +157,7 @@ class TaskExecutor {
             }
         });
     }
-    async runPlans(title, plans, planningModel, defaultModel, options) {
+    async runPlans(title, plans, planningModel, locateModel, options) {
         const abortSignal = options?.abortSignal ?? this.getAbortSignal?.();
         const session = this.createExecutionSession(title, {
             ...options,
@@ -168,11 +168,11 @@ class TaskExecutor {
             ...planningModel,
             executionId: runner.id
         };
-        const executionDefaultModel = {
-            ...defaultModel,
+        const executionLocateModel = {
+            ...locateModel,
             executionId: runner.id
         };
-        const { tasks } = await this.convertPlanToExecutable(plans, executionPlanningModel, executionDefaultModel, {
+        const { tasks } = await this.convertPlanToExecutable(plans, executionPlanningModel, executionLocateModel, {
             abortSignal
         });
         const result = await session.appendAndRun(tasks);
@@ -182,9 +182,9 @@ class TaskExecutor {
             runner
         };
     }
-    async action(userPrompt, planningModel, defaultModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions) {
+    async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions) {
         abortSignal?.throwIfAborted();
-        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, defaultModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions));
+        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions));
     }
     invalidateFailedCacheHitLocates(runner, fromIndex) {
         if (!this.taskCache) return;
@@ -196,7 +196,7 @@ class TaskExecutor {
             }
         }
     }
-    async runAction(userPrompt, planningModel, defaultModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions) {
+    async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions) {
         abortSignal ??= this.getAbortSignal?.();
         const conversationHistory = new ConversationHistory();
         const promptDisplay = reportOptions?.prompt || userPromptToString(userPrompt);
@@ -213,11 +213,11 @@ class TaskExecutor {
             ...planningModel,
             executionId: runner.id
         };
-        defaultModel = {
-            ...defaultModel,
+        locateModel = {
+            ...locateModel,
             executionId: runner.id
         };
-        const noIndividualLocateModel = 'default' === planningModel.config.slot;
+        const noIndividualLocateModel = 'default' === planningModel.config.slot && 'default' === locateModel.config.slot;
         const includeLocateInPlanning = 'deepThink' !== effort && noIndividualLocateModel;
         const imagesIncludeCount = 'deepThink' === effort ? 2 : 1;
         debug('setting includeLocateInPlanning to', includeLocateInPlanning, {
@@ -303,6 +303,8 @@ class TaskExecutor {
                             actionContext: renderAIContext(aiActContext),
                             actionSpace,
                             modelRuntime: planningModel,
+                            locateModelRuntime: locateModel,
+                            getDefaultModelRuntime: this.getDefaultModelRuntime,
                             conversationHistory,
                             includeLocateInPlanning,
                             imagesIncludeCount,
@@ -392,7 +394,7 @@ class TaskExecutor {
             yamlFlow.push(...planResult?.yamlFlow || []);
             let executables;
             try {
-                executables = await this.convertPlanToExecutable(plans, planningModel, defaultModel, {
+                executables = await this.convertPlanToExecutable(plans, planningModel, locateModel, {
                     cacheable,
                     multimodalPrompt: userPromptToMultimodalPrompt(userPrompt),
                     deepLocate,
@@ -625,6 +627,7 @@ class TaskExecutor {
         _define_property(this, "interface", void 0);
         _define_property(this, "service", void 0);
         _define_property(this, "taskCache", void 0);
+        _define_property(this, "getDefaultModelRuntime", void 0);
         _define_property(this, "providedActionSpace", void 0);
         _define_property(this, "taskBuilder", void 0);
         _define_property(this, "onTaskStartCallback", void 0);
@@ -633,6 +636,7 @@ class TaskExecutor {
         _define_property(this, "replanningCycleLimit", void 0);
         _define_property(this, "waitAfterAction", void 0);
         _define_property(this, "useDeviceTime", void 0);
+        this.getDefaultModelRuntime = opts.getDefaultModelRuntime;
         this.interface = interfaceInstance;
         this.service = service;
         this.taskCache = opts.taskCache;
@@ -644,6 +648,7 @@ class TaskExecutor {
         this.getAbortSignal = opts.getAbortSignal;
         this.providedActionSpace = opts.actionSpace;
         this.taskBuilder = new TaskBuilder({
+            getDefaultModelRuntime: opts.getDefaultModelRuntime,
             interfaceInstance,
             service,
             taskCache: opts.taskCache,