@midscene/core 1.0.1-beta-20251028021317.0 → 1.0.1-beta-20251028121806.0

package/dist/es/agent/tasks.mjs

@@ -1,5 +1,5 @@
 import { ConversationHistory, plan, uiTarsPlanning } from "../ai-model/index.mjs";
-import { InsightError } from "../types.mjs";
+import { ServiceError } from "../types.mjs";
 import { MIDSCENE_REPLANNING_CYCLE_LIMIT, globalConfigManager } from "@midscene/shared/env";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
@@ -26,7 +26,7 @@ class TaskExecutor {
         return this.interface;
     }
     createExecutionSession(title, options) {
-        return new ExecutionSession(title, ()=>Promise.resolve(this.insight.contextRetrieverFn()), {
+        return new ExecutionSession(title, ()=>Promise.resolve(this.service.contextRetrieverFn()), {
             onTaskStart: this.onTaskStartCallback,
             tasks: null == options ? void 0 : options.tasks
         });
@@ -227,9 +227,9 @@ class TaskExecutor {
                     extraPageDescription = await descriptionOfTree(tree, 200, false, (null == opt ? void 0 : opt.domIncluded) === 'visible-only');
                 }
                 try {
-                    extractResult = await this.insight.extract(demandInput, modelConfig, opt, extraPageDescription, multimodalPrompt);
+                    extractResult = await this.service.extract(demandInput, modelConfig, opt, extraPageDescription, multimodalPrompt);
                 } catch (error) {
-                    if (error instanceof InsightError) applyDump(error.dump);
+                    if (error instanceof ServiceError) applyDump(error.dump);
                     throw error;
                 }
                 const { data, usage, thought, dump } = extractResult;
@@ -313,23 +313,23 @@ class TaskExecutor {
         }
         return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
     }
-    constructor(interfaceInstance, insight, opts){
+    constructor(interfaceInstance, service, opts){
         _define_property(this, "interface", void 0);
-        _define_property(this, "insight", void 0);
+        _define_property(this, "service", void 0);
         _define_property(this, "taskCache", void 0);
         _define_property(this, "taskBuilder", void 0);
         _define_property(this, "conversationHistory", void 0);
         _define_property(this, "onTaskStartCallback", void 0);
         _define_property(this, "replanningCycleLimit", void 0);
         this.interface = interfaceInstance;
-        this.insight = insight;
+        this.service = service;
         this.taskCache = opts.taskCache;
         this.onTaskStartCallback = null == opts ? void 0 : opts.onTaskStart;
         this.replanningCycleLimit = opts.replanningCycleLimit;
         this.conversationHistory = new ConversationHistory();
         this.taskBuilder = new TaskBuilder({
             interfaceInstance,
-            insight,
+            service,
             taskCache: opts.taskCache
         });
     }

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type Insight from '@/insight';\nimport type { TaskRunner } from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InsightDump,\n  InsightExtractOption,\n  InsightExtractParam,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.conversationHistory = new ConversationHistory();\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      insight,\n      taskCache: opts.taskCache,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.insight.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n      },\n    );\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(plans, modelConfig, options);\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    await session.appendAndRun(task);\n\n    return {\n      runner: session.getRunner(),\n    };\n  }\n\n  private createPlanningTask(\n    userInstruction: string,\n    actionContext: string | undefined,\n    modelConfig: IModelConfig,\n  ): ExecutionTaskPlanningApply {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        const { vlMode } = modelConfig;\n        const uiTarsModelVersion =\n          vlMode === 'vlm-ui-tars' ? modelConfig.uiTarsModelVersion : undefined;\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await (uiTarsModelVersion ? uiTarsPlanning : plan)(\n          param.userInstruction,\n          {\n            context: uiContext,\n            actionContext,\n            interfaceType: this.interface.interfaceType as InterfaceType,\n            actionSpace,\n            modelConfig,\n            conversationHistory: this.conversationHistory,\n          },\n        );\n        debug('planResult', JSON.stringify(planResult, null, 2));\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push(this.sleepPlan(timeRemaining));\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    const result = await session.appendAndRun(tasks);\n    const { output } = result!;\n    return {\n      output,\n      runner: session.getRunner(),\n    };\n  }\n\n  private getReplanningCycleLimit(isVlmUiTars: boolean) {\n    return (\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      (isVlmUiTars\n        ? defaultVlmUiTarsReplanningCycleLimit\n        : defaultReplanningCycleLimit)\n    );\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n    cacheable?: boolean,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit = this.getReplanningCycleLimit(\n      modelConfig.vlMode === 'vlm-ui-tars',\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanning ${replanningCycleLimit} times, which is more than the limit, please split the task into multiple steps`;\n\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      // Create planning task (automatically includes execution history if available)\n      const planningTask = this.createPlanningTask(\n        userPrompt,\n        actionContext,\n        modelConfig,\n      );\n\n      const result = await session.appendAndRun(planningTask);\n      const planResult: PlanningAIResponse = result?.output;\n      if (session.isInErrorState()) {\n        return {\n          output: planResult,\n          runner,\n        };\n      }\n\n      // Execute planned actions\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans, modelConfig, {\n          cacheable,\n          subTask: true,\n        });\n        await session.appendAndRun(executables.tasks);\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (session.isInErrorState()) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      // Check if task is complete\n      if (!planResult.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: InsightDump | undefined;\n        const applyDump = (dump: InsightDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.insight.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof InsightError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner: session.getRunner(),\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n      locate: null,\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        {\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","defaultVlmUiTarsReplanningCycleLimit","TaskExecutor","title","options","ExecutionSession","Promise","plans","modelConfig","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","actionContext","startTime","Date","vlMode","uiTarsModelVersion","undefined","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","finalActions","timeNow","timeRemaining","tasks","result","output","isVlmUiTars","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","userPrompt","cacheable","runner","replanCount","yamlFlow","replanningCycleLimit","errorMsg","planningTask","executables","type","demand","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","InsightError","data","thought","outputResult","Error","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepTask","interfaceInstance","insight","opts","ConversationHistory","TaskBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;;AA0CA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AACpC,MAAMC,uCAAuC;AAEtC,MAAMC;IAgBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAwBQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK;QACvB;IAEJ;IAEA,MAAa,wBACXG,KAAuB,EACvBC,WAAyB,EACzBJ,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACG,OAAOC,aAAaJ;IACpD;IAEA,MAAM,uBAAuBK,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMC,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACL,QAAQF,QAAQ,SAAS;QAC3B;IACF;IAEQ,mBACNF,eAAuB,EACvBS,aAAiC,EACjCV,WAAyB,EACG;QAC5B,MAAMK,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAMI,YAAYC,KAAK,GAAG;gBAC1B,MAAM,EAAEJ,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,MAAM,EAAEK,MAAM,EAAE,GAAGb;gBACnB,MAAMc,qBACJD,AAAW,kBAAXA,SAA2Bb,YAAY,kBAAkB,GAAGe;gBAE9DN,OACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMO,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD1B,MACE,sCACA0B,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDR,OAAOS,MAAM,OAAO,CAACF,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAON,AAAAA,CAAAA,qBAAqBO,iBAAiBC,IAAG,EACjEhB,MAAM,eAAe,EACrB;oBACE,SAASE;oBACTE;oBACA,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CM;oBACAhB;oBACA,qBAAqB,IAAI,CAAC,mBAAmB;gBAC/C;gBAEFV,MAAM,cAAciC,KAAK,SAAS,CAACH,YAAY,MAAM;gBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACN,GAAGV;gBAEJb,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCsB;gBACF;gBACAtB,gBAAgB,IAAI,CAAC,KAAK,GAAGqB;gBAE7B,MAAMG,eAAeP,WAAW,EAAE;gBAElC,IAAIM,OAAO;oBACT,MAAME,UAAUpB,KAAK,GAAG;oBACxB,MAAMqB,gBAAgBH,QAASE,CAAAA,UAAUrB,SAAQ;oBACjD,IAAIsB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;gBAErC;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBtB,OACE,CAACiB,sCAAsCI,OACvCH,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASI;wBACTL;wBACAD;wBACA,UAAUL,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAZ;gBACF;YACF;QACF;QAEA,OAAOH;IACT;IAEA,MAAM,SACJV,KAAa,EACbI,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACR;QAC5C,MAAM,EAAEuC,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACnC,OAAOC;QAC5D,MAAMmC,SAAS,MAAMhC,QAAQ,YAAY,CAAC+B;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD;QACnB,OAAO;YACLC;YACA,QAAQjC,QAAQ,SAAS;QAC3B;IACF;IAEQ,wBAAwBkC,WAAoB,EAAE;QACpD,OACE,IAAI,CAAC,oBAAoB,IACzBC,oBAAoB,oBAAoB,CACtCC,oCAEDF,CAAAA,cACG5C,uCACAD,2BAA0B;IAElC;IAEA,MAAM,OACJgD,UAAkB,EAClBxC,WAAyB,EACzBU,aAAsB,EACtB+B,SAAmB,EAQnB;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMtC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUoC;QAEzB,MAAME,SAASvC,QAAQ,SAAS;QAEhC,IAAIwC,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBAAuB,IAAI,CAAC,uBAAuB,CACvD7C,AAAuB,kBAAvBA,YAAY,MAAM;QAIpB,MAAO,KAAM;YACX,IAAI2C,cAAcE,sBAAsB;gBACtC,MAAMC,WAAW,CAAC,WAAW,EAAED,qBAAqB,+EAA+E,CAAC;gBAEpI,OAAO1C,QAAQ,eAAe,CAAC2C;YACjC;YAGA,MAAMC,eAAe,IAAI,CAAC,kBAAkB,CAC1CP,YACA9B,eACAV;YAGF,MAAMmC,SAAS,MAAMhC,QAAQ,YAAY,CAAC4C;YAC1C,MAAM3B,aAAiCe,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAIhC,QAAQ,cAAc,IACxB,OAAO;gBACL,QAAQiB;gBACRsB;YACF;YAIF,MAAM3C,QAAQqB,WAAW,OAAO,IAAI,EAAE;YACtCwB,SAAS,IAAI,IAAKxB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAI4B;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAACjD,OAAOC,aAAa;oBACnEyC;oBACA,SAAS;gBACX;gBACA,MAAMtC,QAAQ,YAAY,CAAC6C,YAAY,KAAK;YAC9C,EAAE,OAAOrB,OAAO;gBACd,OAAOxB,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEwB,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5ExB,QACC;YAEP;YACA,IAAII,QAAQ,cAAc,IACxB,OAAO;gBACL,QAAQY;gBACR2B;YACF;YAIF,IAAI,CAACtB,WAAW,kCAAkC,EAChD;YAIFuB;QACF;QAEA,OAAO;YACL,QAAQ;gBACNC;YACF;YACAF;QACF;IACF;IAEQ,oBACNO,IAAsE,EACtEC,MAA2B,EAC3BlD,WAAyB,EACzBmD,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBACL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO5C,OAAOgD;gBACtB,MAAM,EAAEjD,IAAI,EAAE,GAAGiD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZpD,KAAK,GAAG,GAAG;wBACToD;oBACF;gBACF;gBAGA,MAAMjD,YAAY8C,YAAY,SAAS;gBACvC7C,OAAOD,WAAW;gBAElB,MAAMkD,mBAAmBT,AAAS,YAATA;gBACzB,IAAIU,cAAcT;gBAClB,IAAIU,cAAc;gBAClB,IAAIF,oBAAqBT,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEW,cAAc;oBACd,MAAMC,gBACJZ,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIS,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGX,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIY;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1D7D,MAAM;oBACN,MAAM0E,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACA3D,aACAmD,KACAY,sBACAX;gBAEJ,EAAE,OAAOzB,OAAO;oBACd,IAAIA,iBAAiBuC,cACnBV,UAAU7B,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEwC,IAAI,EAAEvC,KAAK,EAAEwC,OAAO,EAAEX,IAAI,EAAE,GAAGK;gBACvCN,UAAUC;gBAEV,IAAIY,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAIlB,AAAS,cAATA,MAEPoB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACL5D,OACE0D,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,IAAM,CAACP,YAAY,AAAD,MAAM7C,QACxB;oBAEFsD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,OAAO;oBACL,QAAQS;oBACR,KAAKd;oBACL3B;oBACAwC;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BlD,WAAyB,EACzBmD,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMjD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE6C,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAAS3B,KAAK,SAAS,CAAC2B;QAIzD,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAlD,aACAmD,KACAC;QAGF,MAAMjB,SAAS,MAAMhC,QAAQ,YAAY,CAACkD;QAE1C,IAAI,CAAClB,QACH,MAAM,IAAImC,MACR;QAIJ,MAAM,EAAElC,MAAM,EAAEgC,OAAO,EAAE,GAAGjC;QAE5B,OAAO;YACLC;YACAgC;YACA,QAAQjE,QAAQ,SAAS;QAC3B;IACF;IAEQ,UAAUoE,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;YACA,QAAQ;QACV;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBtB,GAA+B,EAC/BnD,WAAyB,EACO;QAChC,MAAM,EAAE0E,UAAU,EAAEtB,gBAAgB,EAAE,GAAGuB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWwE;QAE1B,MAAMlC,SAASvC,QAAQ,SAAS;QAChC,MAAM,EAAE0E,SAAS,EAAEC,eAAe,EAAE,GAAG3B;QAEvC1C,OAAOgE,WAAW;QAClBhE,OAAOoE,WAAW;QAClBpE,OAAOqE,iBAAiB;QAExBrE,OACEqE,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBnE,KAAK,GAAG;QACjC,IAAID,YAAYC,KAAK,GAAG;QACxB,IAAIoE,eAAe;QACnB,MAAOpE,KAAK,GAAG,KAAKmE,mBAAmBF,UAAW;YAChDlE,YAAYC,KAAK,GAAG;YACpB,MAAMyC,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAqB,YACA1E,aACA;gBACE,iBAAiB;YACnB,GACAoD;YAGF,MAAMjB,SAAU,MAAMhC,QAAQ,YAAY,CAACkD;YAO3C,IAAIlB,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQpB;gBACR2B;YACF;YAGFsC,eACE7C,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACb,CAACA,UAAU,CAAC,0BAA0B,EAAEuC,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMO,MAAMrE,KAAK,GAAG;YACpB,IAAIqE,MAAMtE,YAAYmE,iBAAiB;gBACrC,MAAM7C,gBAAgB6C,kBAAmBG,CAAAA,MAAMtE,SAAQ;gBACvD,MAAMuE,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQjD;gBACV;gBACA,MAAM9B,QAAQ,MAAM,CAAC+E;YACvB;QACF;QAEA,OAAO/E,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAE6E,cAAc;IACnE;IAjiBA,YACEG,iBAAoC,EACpCC,OAAgB,EAChBC,IAIC,CACD;QA3BF;QAEA;QAEA;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA;QAgBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,mBAAmB,GAAG,IAAIC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;QAC3B;IACF;AA8gBF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.conversationHistory = new ConversationHistory();\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n      },\n    );\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(plans, modelConfig, options);\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    await session.appendAndRun(task);\n\n    return {\n      runner: session.getRunner(),\n    };\n  }\n\n  private createPlanningTask(\n    userInstruction: string,\n    actionContext: string | undefined,\n    modelConfig: IModelConfig,\n  ): ExecutionTaskPlanningApply {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        const { vlMode } = modelConfig;\n        const uiTarsModelVersion =\n          vlMode === 'vlm-ui-tars' ? modelConfig.uiTarsModelVersion : undefined;\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await (uiTarsModelVersion ? uiTarsPlanning : plan)(\n          param.userInstruction,\n          {\n            context: uiContext,\n            actionContext,\n            interfaceType: this.interface.interfaceType as InterfaceType,\n            actionSpace,\n            modelConfig,\n            conversationHistory: this.conversationHistory,\n          },\n        );\n        debug('planResult', JSON.stringify(planResult, null, 2));\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push(this.sleepPlan(timeRemaining));\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    const result = await session.appendAndRun(tasks);\n    const { output } = result!;\n    return {\n      output,\n      runner: session.getRunner(),\n    };\n  }\n\n  private getReplanningCycleLimit(isVlmUiTars: boolean) {\n    return (\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      (isVlmUiTars\n        ? defaultVlmUiTarsReplanningCycleLimit\n        : defaultReplanningCycleLimit)\n    );\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n    cacheable?: boolean,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit = this.getReplanningCycleLimit(\n      modelConfig.vlMode === 'vlm-ui-tars',\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanning ${replanningCycleLimit} times, which is more than the limit, please split the task into multiple steps`;\n\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      // Create planning task (automatically includes execution history if available)\n      const planningTask = this.createPlanningTask(\n        userPrompt,\n        actionContext,\n        modelConfig,\n      );\n\n      const result = await session.appendAndRun(planningTask);\n      const planResult: PlanningAIResponse = result?.output;\n      if (session.isInErrorState()) {\n        return {\n          output: planResult,\n          runner,\n        };\n      }\n\n      // Execute planned actions\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans, modelConfig, {\n          cacheable,\n          subTask: true,\n        });\n        await session.appendAndRun(executables.tasks);\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (session.isInErrorState()) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      // Check if task is complete\n      if (!planResult.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner: session.getRunner(),\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n      locate: null,\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        {\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","defaultVlmUiTarsReplanningCycleLimit","TaskExecutor","title","options","ExecutionSession","Promise","plans","modelConfig","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","actionContext","startTime","Date","vlMode","uiTarsModelVersion","undefined","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","finalActions","timeNow","timeRemaining","tasks","result","output","isVlmUiTars","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","userPrompt","cacheable","runner","replanCount","yamlFlow","replanningCycleLimit","errorMsg","planningTask","executables","type","demand","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","thought","outputResult","Error","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepTask","interfaceInstance","service","opts","ConversationHistory","TaskBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;;AA0CA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AACpC,MAAMC,uCAAuC;AAEtC,MAAMC;IAgBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAwBQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK;QACvB;IAEJ;IAEA,MAAa,wBACXG,KAAuB,EACvBC,WAAyB,EACzBJ,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACG,OAAOC,aAAaJ;IACpD;IAEA,MAAM,uBAAuBK,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMC,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACL,QAAQF,QAAQ,SAAS;QAC3B;IACF;IAEQ,mBACNF,eAAuB,EACvBS,aAAiC,EACjCV,WAAyB,EACG;QAC5B,MAAMK,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAMI,YAAYC,KAAK,GAAG;gBAC1B,MAAM,EAAEJ,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,MAAM,EAAEK,MAAM,EAAE,GAAGb;gBACnB,MAAMc,qBACJD,AAAW,kBAAXA,SAA2Bb,YAAY,kBAAkB,GAAGe;gBAE9DN,OACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMO,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD1B,MACE,sCACA0B,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDR,OAAOS,MAAM,OAAO,CAACF,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAON,AAAAA,CAAAA,qBAAqBO,iBAAiBC,IAAG,EACjEhB,MAAM,eAAe,EACrB;oBACE,SAASE;oBACTE;oBACA,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CM;oBACAhB;oBACA,qBAAqB,IAAI,CAAC,mBAAmB;gBAC/C;gBAEFV,MAAM,cAAciC,KAAK,SAAS,CAACH,YAAY,MAAM;gBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACN,GAAGV;gBAEJb,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCsB;gBACF;gBACAtB,gBAAgB,IAAI,CAAC,KAAK,GAAGqB;gBAE7B,MAAMG,eAAeP,WAAW,EAAE;gBAElC,IAAIM,OAAO;oBACT,MAAME,UAAUpB,KAAK,GAAG;oBACxB,MAAMqB,gBAAgBH,QAASE,CAAAA,UAAUrB,SAAQ;oBACjD,IAAIsB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;gBAErC;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBtB,OACE,CAACiB,sCAAsCI,OACvCH,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASI;wBACTL;wBACAD;wBACA,UAAUL,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAZ;gBACF;YACF;QACF;QAEA,OAAOH;IACT;IAEA,MAAM,SACJV,KAAa,EACbI,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACR;QAC5C,MAAM,EAAEuC,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACnC,OAAOC;QAC5D,MAAMmC,SAAS,MAAMhC,QAAQ,YAAY,CAAC+B;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD;QACnB,OAAO;YACLC;YACA,QAAQjC,QAAQ,SAAS;QAC3B;IACF;IAEQ,wBAAwBkC,WAAoB,EAAE;QACpD,OACE,IAAI,CAAC,oBAAoB,IACzBC,oBAAoB,oBAAoB,CACtCC,oCAEDF,CAAAA,cACG5C,uCACAD,2BAA0B;IAElC;IAEA,MAAM,OACJgD,UAAkB,EAClBxC,WAAyB,EACzBU,aAAsB,EACtB+B,SAAmB,EAQnB;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMtC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUoC;QAEzB,MAAME,SAASvC,QAAQ,SAAS;QAEhC,IAAIwC,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBAAuB,IAAI,CAAC,uBAAuB,CACvD7C,AAAuB,kBAAvBA,YAAY,MAAM;QAIpB,MAAO,KAAM;YACX,IAAI2C,cAAcE,sBAAsB;gBACtC,MAAMC,WAAW,CAAC,WAAW,EAAED,qBAAqB,+EAA+E,CAAC;gBAEpI,OAAO1C,QAAQ,eAAe,CAAC2C;YACjC;YAGA,MAAMC,eAAe,IAAI,CAAC,kBAAkB,CAC1CP,YACA9B,eACAV;YAGF,MAAMmC,SAAS,MAAMhC,QAAQ,YAAY,CAAC4C;YAC1C,MAAM3B,aAAiCe,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAIhC,QAAQ,cAAc,IACxB,OAAO;gBACL,QAAQiB;gBACRsB;YACF;YAIF,MAAM3C,QAAQqB,WAAW,OAAO,IAAI,EAAE;YACtCwB,SAAS,IAAI,IAAKxB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAI4B;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAACjD,OAAOC,aAAa;oBACnEyC;oBACA,SAAS;gBACX;gBACA,MAAMtC,QAAQ,YAAY,CAAC6C,YAAY,KAAK;YAC9C,EAAE,OAAOrB,OAAO;gBACd,OAAOxB,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEwB,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5ExB,QACC;YAEP;YACA,IAAII,QAAQ,cAAc,IACxB,OAAO;gBACL,QAAQY;gBACR2B;YACF;YAIF,IAAI,CAACtB,WAAW,kCAAkC,EAChD;YAIFuB;QACF;QAEA,OAAO;YACL,QAAQ;gBACNC;YACF;YACAF;QACF;IACF;IAEQ,oBACNO,IAAsE,EACtEC,MAA2B,EAC3BlD,WAAyB,EACzBmD,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBACL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO5C,OAAOgD;gBACtB,MAAM,EAAEjD,IAAI,EAAE,GAAGiD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZpD,KAAK,GAAG,GAAG;wBACToD;oBACF;gBACF;gBAGA,MAAMjD,YAAY8C,YAAY,SAAS;gBACvC7C,OAAOD,WAAW;gBAElB,MAAMkD,mBAAmBT,AAAS,YAATA;gBACzB,IAAIU,cAAcT;gBAClB,IAAIU,cAAc;gBAClB,IAAIF,oBAAqBT,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEW,cAAc;oBACd,MAAMC,gBACJZ,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIS,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGX,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIY;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1D7D,MAAM;oBACN,MAAM0E,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACA3D,aACAmD,KACAY,sBACAX;gBAEJ,EAAE,OAAOzB,OAAO;oBACd,IAAIA,iBAAiBuC,cACnBV,UAAU7B,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEwC,IAAI,EAAEvC,KAAK,EAAEwC,OAAO,EAAEX,IAAI,EAAE,GAAGK;gBACvCN,UAAUC;gBAEV,IAAIY,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAIlB,AAAS,cAATA,MAEPoB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACL5D,OACE0D,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,IAAM,CAACP,YAAY,AAAD,MAAM7C,QACxB;oBAEFsD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,OAAO;oBACL,QAAQS;oBACR,KAAKd;oBACL3B;oBACAwC;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BlD,WAAyB,EACzBmD,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMjD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE6C,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAAS3B,KAAK,SAAS,CAAC2B;QAIzD,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAlD,aACAmD,KACAC;QAGF,MAAMjB,SAAS,MAAMhC,QAAQ,YAAY,CAACkD;QAE1C,IAAI,CAAClB,QACH,MAAM,IAAImC,MACR;QAIJ,MAAM,EAAElC,MAAM,EAAEgC,OAAO,EAAE,GAAGjC;QAE5B,OAAO;YACLC;YACAgC;YACA,QAAQjE,QAAQ,SAAS;QAC3B;IACF;IAEQ,UAAUoE,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;YACA,QAAQ;QACV;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBtB,GAA+B,EAC/BnD,WAAyB,EACO;QAChC,MAAM,EAAE0E,UAAU,EAAEtB,gBAAgB,EAAE,GAAGuB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWwE;QAE1B,MAAMlC,SAASvC,QAAQ,SAAS;QAChC,MAAM,EAAE0E,SAAS,EAAEC,eAAe,EAAE,GAAG3B;QAEvC1C,OAAOgE,WAAW;QAClBhE,OAAOoE,WAAW;QAClBpE,OAAOqE,iBAAiB;QAExBrE,OACEqE,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBnE,KAAK,GAAG;QACjC,IAAID,YAAYC,KAAK,GAAG;QACxB,IAAIoE,eAAe;QACnB,MAAOpE,KAAK,GAAG,KAAKmE,mBAAmBF,UAAW;YAChDlE,YAAYC,KAAK,GAAG;YACpB,MAAMyC,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAqB,YACA1E,aACA;gBACE,iBAAiB;YACnB,GACAoD;YAGF,MAAMjB,SAAU,MAAMhC,QAAQ,YAAY,CAACkD;YAO3C,IAAIlB,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQpB;gBACR2B;YACF;YAGFsC,eACE7C,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACb,CAACA,UAAU,CAAC,0BAA0B,EAAEuC,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMO,MAAMrE,KAAK,GAAG;YACpB,IAAIqE,MAAMtE,YAAYmE,iBAAiB;gBACrC,MAAM7C,gBAAgB6C,kBAAmBG,CAAAA,MAAMtE,SAAQ;gBACvD,MAAMuE,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQjD;gBACV;gBACA,MAAM9B,QAAQ,MAAM,CAAC+E;YACvB;QACF;QAEA,OAAO/E,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAE6E,cAAc;IACnE;IAjiBA,YACEG,iBAAoC,EACpCC,OAAgB,EAChBC,IAIC,CACD;QA3BF;QAEA;QAEA;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA;QAgBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,mBAAmB,GAAG,IAAIC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;QAC3B;IACF;AA8gBF"}

package/dist/es/agent/ui-utils.mjs

@@ -1,5 +1,4 @@
 function typeStr(task) {
-    if ('Insight' === task.type && ('Query' === task.subType || 'Assert' === task.subType)) return task.type;
     if ('Action' === task.type && task.subType) return `Action Space / ${task.subType}`;
     if (task.subType) return `${task.type} / ${task.subType}`;
     return task.type;

package/dist/es/agent/ui-utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // For Insight tasks with Query or Assert subtypes, show just \"Insight\"\n  if (\n    task.type === 'Insight' &&\n    (task.subType === 'Query' || task.subType === 'Assert')\n  ) {\n    return task.type;\n  }\n\n  // For Action tasks with subType, show \"Action Space / subType\"\n  if (task.type === 'Action' && task.subType) {\n    return `Action Space / ${task.subType}`;\n  }\n\n  // For all other cases with subType, show \"type / subType\"\n  if (task.subType) {\n    return `${task.type} / ${task.subType}`;\n  }\n\n  // No subType, just show type\n  return task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","locateStr","_task_param7","_task_param8","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IAEzC,IACEA,AAAc,cAAdA,KAAK,IAAI,IACRA,CAAAA,AAAiB,YAAjBA,KAAK,OAAO,IAAgBA,AAAiB,aAAjBA,KAAK,OAAO,AAAY,GAErD,OAAOA,KAAK,IAAI;IAIlB,IAAIA,AAAc,aAAdA,KAAK,IAAI,IAAiBA,KAAK,OAAO,EACxC,OAAO,CAAC,eAAe,EAAEA,KAAK,OAAO,EAAE;IAIzC,IAAIA,KAAK,OAAO,EACd,OAAO,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,EAAE;IAIzC,OAAOA,KAAK,IAAI;AAClB;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAGzBkB,cACAC;QAHFH,QACEf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAqC,KAAK,KACzDA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAoC,KAAK,AAAD,IAAzCkB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDlB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAwC,KAAK,AAAD,IAA7CmB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAInB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfoB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMrB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAA8B,KAAK,AAAD,IAAnCoB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXT,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAADyB,CAAAA,eAACzB,KAA8B,KAAK,AAAD,IAAnCyB,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQzB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDL,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACtB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARV,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAR,QADEA,QACM,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOW,KAAK,SAAS,CAACX,OAAOY,QAAW;AAC1C"}
+{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // For Action tasks with subType, show \"Action Space / subType\"\n  if (task.type === 'Action' && task.subType) {\n    return `Action Space / ${task.subType}`;\n  }\n\n  // For all other cases with subType, show \"type / subType\"\n  if (task.subType) {\n    return `${task.type} / ${task.subType}`;\n  }\n\n  // No subType, just show type\n  return task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","locateStr","_task_param7","_task_param8","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IAEzC,IAAIA,AAAc,aAAdA,KAAK,IAAI,IAAiBA,KAAK,OAAO,EACxC,OAAO,CAAC,eAAe,EAAEA,KAAK,OAAO,EAAE;IAIzC,IAAIA,KAAK,OAAO,EACd,OAAO,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,EAAE;IAIzC,OAAOA,KAAK,IAAI;AAClB;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAGzBkB,cACAC;QAHFH,QACEf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAqC,KAAK,KACzDA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAoC,KAAK,AAAD,IAAzCkB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDlB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAwC,KAAK,AAAD,IAA7CmB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAInB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfoB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMrB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAA8B,KAAK,AAAD,IAAnCoB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXT,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAADyB,CAAAA,eAACzB,KAA8B,KAAK,AAAD,IAAnCyB,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQzB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDL,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACtB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARV,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAR,QADEA,QACM,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOW,KAAK,SAAS,CAACX,OAAOY,QAAW;AAC1C"}

package/dist/es/agent/utils.mjs

@@ -99,7 +99,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.0.1-beta-20251028021317.0";
+const getMidsceneVersion = ()=>"1.0.1-beta-20251028121806.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: await describeUserPage(context),\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement =\n        generateElementByPosition(rectCenter);\n      errors = [];\n\n      element.isOrderSensitive =\n        typeof res.content === 'object' &&\n        res.content !== null &&\n        'isOrderSensitive' in res.content\n          ? (res.content as any).isOrderSensitive\n          : undefined;\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: InsightExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","userInstructionPrompt","findElementPrompt","describeUserPage","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","generateElementByPosition","undefined","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;;AAmDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IASC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiB,MAAMC,iBAAiBT;QACxC,0BAA0BX,wBAAwBY;IACpD;IACA,MAAMS,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;YAWXkB,4BACCC;QAXdZ,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,cAAXA,QAAsB;QAC/B,MAAMe,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAON,0BAAuC;QAChD,MAAMoB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMC,MAAM,MAAMpB,SAASR,MAAM6B,aAAa,eAAe,EAAEpB;IAE/D,MAAMqB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAK1DQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAZ;YAGFlB,aAAa,WAAWwC;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMQ,UACJC,0BAA0BF;YAC5BL,SAAS,EAAE;YAEXM,QAAQ,gBAAgB,GACtB,AAAuB,YAAvB,OAAOZ,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCc;YAEN,IAAIF,SACFP,kBAAkB;gBAACO;aAAQ;QAE/B;IACF,EAAE,OAAOG,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACT,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEU,IAAI,CAAC,CAAC;aAFtBV,SAAS;YAACU;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMZ;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;IAClB;AACF;AAEO,eAAekB,gBAAgBzC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEyC,kBAAkB,EAAEtC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAegC,4BAA4BtC;IACjD,MAAMuC,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoBvD,wBAAwBoD;IAC9C;IACA,MAAM/C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMsC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMpB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQkD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA/C,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAGF,IAAI4C;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAajB,gBACjBgB,aACAhD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0B6D;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DzD,aAAa,wBAAwB8D;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASvB,MAAM,OAAO,CAACuB,OAC/B,GAAG,CAAC,CAACA,OACGpB,gBACLoB,MACApD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqB+D;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7D/D,aAAa,iBAAiBiE;QAG9BN,cAAcQ,iBAAiBF,YAAYrD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B2D;IAC1C;IAEA,IAAIS,cAAcnD;IAClB,IAAI0C,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BrD,kBACA0C,aACA3C,AAAW,cAAXA;QAEFoD,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAapB,KAAK,SAAS,CAACoB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwB5D,OAO7C;IACC,MAAM,EAAE6D,SAAS,EAAE5D,OAAO,EAAE6D,aAAa,EAAErE,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAeoD;IACrB,MAAM,EAAEzD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM+D,wBAAwB,MAAMC,uBAClCjE,QAAQ,eAAe,IAAI,IAC3B6D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAK5D;YACL,QAAQ;QACV;IACF;IAGF4D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMrE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASuD;QACX;KACD;IAED,IAAIzE,kBAAkB;QACpB,MAAM6B,SAAS,MAAM9B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAEF,OAAO;QACL,aAAa0C,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: await describeUserPage(context),\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement =\n        generateElementByPosition(rectCenter);\n      errors = [];\n\n      element.isOrderSensitive =\n        typeof res.content === 'object' &&\n        res.content !== null &&\n        'isOrderSensitive' in res.content\n          ? (res.content as any).isOrderSensitive\n          : undefined;\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","userInstructionPrompt","findElementPrompt","describeUserPage","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","generateElementByPosition","undefined","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;;AAmDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IASC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiB,MAAMC,iBAAiBT;QACxC,0BAA0BX,wBAAwBY;IACpD;IACA,MAAMS,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;YAWXkB,4BACCC;QAXdZ,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,cAAXA,QAAsB;QAC/B,MAAMe,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAON,0BAAuC;QAChD,MAAMoB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMC,MAAM,MAAMpB,SAASR,MAAM6B,aAAa,eAAe,EAAEpB;IAE/D,MAAMqB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAK1DQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAZ;YAGFlB,aAAa,WAAWwC;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMQ,UACJC,0BAA0BF;YAC5BL,SAAS,EAAE;YAEXM,QAAQ,gBAAgB,GACtB,AAAuB,YAAvB,OAAOZ,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCc;YAEN,IAAIF,SACFP,kBAAkB;gBAACO;aAAQ;QAE/B;IACF,EAAE,OAAOG,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACT,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEU,IAAI,CAAC,CAAC;aAFtBV,SAAS;YAACU;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMZ;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;IAClB;AACF;AAEO,eAAekB,gBAAgBzC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEyC,kBAAkB,EAAEtC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAegC,4BAA4BtC;IACjD,MAAMuC,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoBvD,wBAAwBoD;IAC9C;IACA,MAAM/C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMsC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMpB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQkD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA/C,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAGF,IAAI4C;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAajB,gBACjBgB,aACAhD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0B6D;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DzD,aAAa,wBAAwB8D;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASvB,MAAM,OAAO,CAACuB,OAC/B,GAAG,CAAC,CAACA,OACGpB,gBACLoB,MACApD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqB+D;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7D/D,aAAa,iBAAiBiE;QAG9BN,cAAcQ,iBAAiBF,YAAYrD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B2D;IAC1C;IAEA,IAAIS,cAAcnD;IAClB,IAAI0C,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BrD,kBACA0C,aACA3C,AAAW,cAAXA;QAEFoD,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAapB,KAAK,SAAS,CAACoB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwB5D,OAO7C;IACC,MAAM,EAAE6D,SAAS,EAAE5D,OAAO,EAAE6D,aAAa,EAAErE,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAeoD;IACrB,MAAM,EAAEzD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM+D,wBAAwB,MAAMC,uBAClCjE,QAAQ,eAAe,IAAI,IAC3B6D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAK5D;YACL,QAAQ;QACV;IACF;IAGF4D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMrE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASuD;QACX;KACD;IAED,IAAIzE,kBAAkB;QACpB,MAAM6B,SAAS,MAAM9B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAEF,OAAO;QACL,aAAa0C,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/index.mjs

@@ -1,12 +1,12 @@
 import { z } from "zod";
-import insight from "./insight/index.mjs";
+import service from "./service/index.mjs";
 import { TaskRunner } from "./task-runner.mjs";
 import { getVersion } from "./utils.mjs";
 import { AiLocateElement, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, describeUserPage, getMidsceneLocationSchema, plan } from "./ai-model/index.mjs";
 import { MIDSCENE_MODEL_NAME } from "@midscene/shared/env";
-import { InsightError } from "./types.mjs";
+import { ServiceError } from "./types.mjs";
 import { Agent, createAgent } from "./agent/index.mjs";
-const src = insight;
-export { Agent, AiLocateElement, insight as Insight, InsightError, MIDSCENE_MODEL_NAME, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, TaskRunner, createAgent, src as default, describeUserPage, getMidsceneLocationSchema, getVersion, plan, z };
+const src = service;
+export { Agent, AiLocateElement, MIDSCENE_MODEL_NAME, PointSchema, RectSchema, service as Service, ServiceError, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, TaskRunner, createAgent, src as default, describeUserPage, getMidsceneLocationSchema, getVersion, plan, z };
 
 //# sourceMappingURL=index.mjs.map

package/dist/es/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sources":["webpack://@midscene/core/./src/index.ts"],"sourcesContent":["import { z } from 'zod';\nimport Insight from './insight/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  getMidsceneLocationSchema,\n  type MidsceneLocationResultType,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport { InsightError } from './types';\n\nexport { z };\n\nexport default Insight;\nexport { TaskRunner, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, createAgent } from './agent';\n"],"names":["Insight"],"mappings":";;;;;;;;AA8BA,YAAeA"}
+{"version":3,"file":"index.mjs","sources":["webpack://@midscene/core/./src/index.ts"],"sourcesContent":["import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  getMidsceneLocationSchema,\n  type MidsceneLocationResultType,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport { ServiceError } from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, createAgent } from './agent';\n"],"names":["Service"],"mappings":";;;;;;;;AA8BA,YAAeA"}

package/dist/es/{insight → service}/index.mjs

@@ -2,12 +2,12 @@ import { AIActionType, expandSearchArea } from "../ai-model/common.mjs";
 import { AiExtractElementInfo, AiLocateElement, callAIWithObjectResponse } from "../ai-model/index.mjs";
 import { AiLocateSection } from "../ai-model/inspect.mjs";
 import { elementDescriberInstruction } from "../ai-model/prompt/describe.mjs";
-import { InsightError } from "../types.mjs";
+import { ServiceError } from "../types.mjs";
 import { MIDSCENE_FORCE_DEEP_THINK, globalConfigManager } from "@midscene/shared/env";
 import { compositeElementInfoImg, cropByRect } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { createInsightDump } from "./utils.mjs";
+import { createServiceDump } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -18,8 +18,8 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const debug = getDebug('ai:insight');
-class Insight {
+const debug = getDebug('ai:service');
+class Service {
     async locate(query, opt, modelConfig) {
         var _parseResult_errors;
         const queryPrompt = 'string' == typeof query ? query : query.prompt;
@@ -84,12 +84,12 @@ class Insight {
             error: errorLog
         };
         const elements = parseResult.elements || [];
-        const dump = createInsightDump({
+        const dump = createServiceDump({
             ...dumpData,
             matchedElement: elements
         });
-        if (errorLog) throw new InsightError(errorLog, dump);
-        if (elements.length > 1) throw new InsightError(`locate: multiple elements found, length = ${elements.length}`, dump);
+        if (errorLog) throw new ServiceError(errorLog, dump);
+        if (elements.length > 1) throw new ServiceError(`locate: multiple elements found, length = ${elements.length}`, dump);
         if (1 === elements.length) return {
             element: {
                 id: elements[0].id,
@@ -138,11 +138,11 @@ class Insight {
             error: errorLog
         };
         const { data, thought } = parseResult || {};
-        const dump = createInsightDump({
+        const dump = createServiceDump({
             ...dumpData,
             data
         });
-        if (errorLog && !data && !(null == opt ? void 0 : opt.doNotThrowError)) throw new InsightError(errorLog, dump);
+        if (errorLog && !data && !(null == opt ? void 0 : opt.doNotThrowError)) throw new ServiceError(errorLog, dump);
         return {
             data,
             thought,
@@ -151,10 +151,10 @@ class Insight {
         };
     }
     async describe(target, modelConfig, opt) {
-        assert(target, 'target is required for insight.describe');
+        assert(target, 'target is required for service.describe');
         const context = await this.contextRetrieverFn();
         const { screenshotBase64, size } = context;
-        assert(screenshotBase64, 'screenshot is required for insight.describe');
+        assert(screenshotBase64, 'screenshot is required for service.describe');
         const { vlMode } = modelConfig;
         const systemPrompt = elementDescriberInstruction();
         const defaultRectSize = 30;
@@ -209,13 +209,13 @@ class Insight {
         _define_property(this, "contextRetrieverFn", void 0);
         _define_property(this, "aiVendorFn", callAIWithObjectResponse);
         _define_property(this, "taskInfo", void 0);
-        assert(context, 'context is required for Insight');
+        assert(context, 'context is required for Service');
         if ('function' == typeof context) this.contextRetrieverFn = context;
         else this.contextRetrieverFn = ()=>Promise.resolve(context);
         if (void 0 !== (null == opt ? void 0 : opt.aiVendorFn)) this.aiVendorFn = opt.aiVendorFn;
         if (void 0 !== (null == opt ? void 0 : opt.taskInfo)) this.taskInfo = opt.taskInfo;
     }
 }
-export { Insight as default };
+export { Service as default };
 
 //# sourceMappingURL=index.mjs.map

package/dist/es/{insight → service}/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"insight/index.mjs","sources":["webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightExtractResult,\n  InsightTaskInfo,\n  LocateResultWithDump,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_FORCE_DEEP_THINK,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { createInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:insight');\nexport default class Insight {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined> =\n    callAIWithObjectResponse;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { vlMode } = modelConfig;\n\n    if (searchAreaPrompt && !vlMode) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage } = await AiLocateElement({\n      callAIFn: this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createInsightDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new InsightError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          isOrderSensitive: elements[0]!.isOrderSensitive,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<InsightExtractResult<T>> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const context = await this.contextRetrieverFn();\n\n    const startTime = Date.now();\n\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelConfig,\n      pageDescription,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    const dump = createInsightDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn();\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { vlMode } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n      debug('describe: set searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        vlMode === 'qwen-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn = this\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Insight","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","undefined","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createInsightDump","InsightError","dataDemand","pageDescription","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AA8CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IA4BnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACM;YAyE3BC;QAxEJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwBC,oBAAoB,qBAAqB,CACrEC;QAEF,IAAIF,uBACFT,MAAM,yBAAyBS;QAEjC,IAAIG;QACJ,IAAIT,MAAM,SAAS,IAAIM,uBACrBG,mBAAmBT,MAAM,MAAM;QAGjC,MAAM,EAAEU,MAAM,EAAE,GAAGR;QAEnB,IAAIO,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,MAAMC,UAAUZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIa;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBJ;gBACpBP;YACF;YACAG,OACEY,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,gBAAgB;YACtE,UAAU,IAAI,CAAC,UAAU;YACzBZ;YACA,0BAA0BT;YAC1B,cAAca;YACdf;QACF;QAEA,MAAMwB,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACL;YAC5B,gBAAgBK,KAAK,SAAS,CAACP;YAC/BG;YACAV;YACAC;YACAC;QACF;QAEA,IAAIa;QACJ,IAAI,QAAA1B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B0B,WAAW,CAAC,6BAA6B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS1B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAakB;YACb,MAAM;YACNK;YACA,WAAW,CAAC,CAACb;YACb,OAAOe;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,kBAAkBA,QAAQ,CAAC,EAAE,CAAE,gBAAgB;YACjD;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BjC,WAAyB,EACzBD,GAA0B,EAC1BmC,eAAwB,EACxBC,gBAAoC,EACF;YA0B9BlC;QAzBJE,OACE,AAAsB,YAAtB,OAAO8B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMtB,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAE7C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEG,KAAK,EAAE,GAAG,MAAMc,qBAAwB;YAC3DzB;YACA,WAAWsB;YACXE;YACA,eAAepC;YACfC;YACAkC;QACF;QAEA,MAAMV,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;QAC9B;QAEA,IAAIQ;QACJ,IAAI,QAAA1B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B0B,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGnB,eAAe,CAAC;QAG1C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXS;QACF;QAEA,IAAIV,YAAY,CAACU,QAAQ,CAACtC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIiC,aAAaL,UAAUG;QAGnC,OAAO;YACLO;YACAC;YACAhB;YACAQ;QACF;IACF;IAEA,MAAM,SACJS,MAA+B,EAC/BvC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOoC,QAAQ;QACf,MAAM5B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAE6B,gBAAgB,EAAEC,IAAI,EAAE,GAAG9B;QACnCR,OAAOqC,kBAAkB;QAEzB,MAAM,EAAEhC,MAAM,EAAE,GAAGR;QACnB,MAAM0C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI9C,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMa,aAAasC,iBAAiBL,YAAYlC,QAAQ,IAAI,EAAEH;YAC9Db,MAAM,4BAA4BiB;YAClC,MAAMuC,gBAAgB,MAAMC,WAC1BJ,cACApC,YACAJ,AAAW,cAAXA;YAEFwC,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,aAAa,gBAAgB,EAC7BxD;QAGF,MAAM,EAAEyD,OAAO,EAAE,GAAGF;QACpBpD,OAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,OAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAxSA,YACE9C,OAA2D,EAC3DZ,GAAoB,CACpB;QAVF;QAEA,qCACE2D;QAEF;QAMEvD,OAAOQ,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMgD,QAAQ,OAAO,CAAChD;QAIlD,IAAI,AAA2B,WAApBZ,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAuRF"}
+{"version":3,"file":"service/index.mjs","sources":["webpack://@midscene/core/./src/service/index.ts"],"sourcesContent":["import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_FORCE_DEEP_THINK,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n  aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  aiVendorFn: Exclude<ServiceOptions['aiVendorFn'], undefined> =\n    callAIWithObjectResponse;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { vlMode } = modelConfig;\n\n    if (searchAreaPrompt && !vlMode) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage } = await AiLocateElement({\n      callAIFn: this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          isOrderSensitive: elements[0]!.isOrderSensitive,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const context = await this.contextRetrieverFn();\n\n    const startTime = Date.now();\n\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelConfig,\n      pageDescription,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { vlMode } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n      debug('describe: set searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        vlMode === 'qwen-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn = this\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","undefined","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AA8CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IA4BnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACM;YAyE3BC;QAxEJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwBC,oBAAoB,qBAAqB,CACrEC;QAEF,IAAIF,uBACFT,MAAM,yBAAyBS;QAEjC,IAAIG;QACJ,IAAIT,MAAM,SAAS,IAAIM,uBACrBG,mBAAmBT,MAAM,MAAM;QAGjC,MAAM,EAAEU,MAAM,EAAE,GAAGR;QAEnB,IAAIO,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,MAAMC,UAAUZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIa;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBJ;gBACpBP;YACF;YACAG,OACEY,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,gBAAgB;YACtE,UAAU,IAAI,CAAC,UAAU;YACzBZ;YACA,0BAA0BT;YAC1B,cAAca;YACdf;QACF;QAEA,MAAMwB,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACL;YAC5B,gBAAgBK,KAAK,SAAS,CAACP;YAC/BG;YACAV;YACAC;YACAC;QACF;QAEA,IAAIa;QACJ,IAAI,QAAA1B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B0B,WAAW,CAAC,6BAA6B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS1B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAakB;YACb,MAAM;YACNK;YACA,WAAW,CAAC,CAACb;YACb,OAAOe;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,kBAAkBA,QAAQ,CAAC,EAAE,CAAE,gBAAgB;YACjD;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BjC,WAAyB,EACzBD,GAA0B,EAC1BmC,eAAwB,EACxBC,gBAAoC,EACF;YA0B9BlC;QAzBJE,OACE,AAAsB,YAAtB,OAAO8B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMtB,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAE7C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEG,KAAK,EAAE,GAAG,MAAMc,qBAAwB;YAC3DzB;YACA,WAAWsB;YACXE;YACA,eAAepC;YACfC;YACAkC;QACF;QAEA,MAAMV,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;QAC9B;QAEA,IAAIQ;QACJ,IAAI,QAAA1B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B0B,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGnB,eAAe,CAAC;QAG1C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXS;QACF;QAEA,IAAIV,YAAY,CAACU,QAAQ,CAACtC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIiC,aAAaL,UAAUG;QAGnC,OAAO;YACLO;YACAC;YACAhB;YACAQ;QACF;IACF;IAEA,MAAM,SACJS,MAA+B,EAC/BvC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOoC,QAAQ;QACf,MAAM5B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAE6B,gBAAgB,EAAEC,IAAI,EAAE,GAAG9B;QACnCR,OAAOqC,kBAAkB;QAEzB,MAAM,EAAEhC,MAAM,EAAE,GAAGR;QACnB,MAAM0C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI9C,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMa,aAAasC,iBAAiBL,YAAYlC,QAAQ,IAAI,EAAEH;YAC9Db,MAAM,4BAA4BiB;YAClC,MAAMuC,gBAAgB,MAAMC,WAC1BJ,cACApC,YACAJ,AAAW,cAAXA;YAEFwC,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,aAAa,gBAAgB,EAC7BxD;QAGF,MAAM,EAAEyD,OAAO,EAAE,GAAGF;QACpBpD,OAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,OAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAxSA,YACE9C,OAA2D,EAC3DZ,GAAoB,CACpB;QAVF;QAEA,qCACE2D;QAEF;QAMEvD,OAAOQ,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMgD,QAAQ,OAAO,CAAChD;QAIlD,IAAI,AAA2B,WAApBZ,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAuRF"}

package/dist/es/{insight → service}/utils.mjs

@@ -1,5 +1,5 @@
 import { uuid } from "@midscene/shared/utils";
-function createInsightDump(data) {
+function createServiceDump(data) {
     const baseData = {
         logTime: Date.now()
     };
@@ -10,6 +10,6 @@ function createInsightDump(data) {
     };
     return finalData;
 }
-export { createInsightDump };
+export { createServiceDump };
 
 //# sourceMappingURL=utils.mjs.map

package/dist/es/service/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"service/utils.mjs","sources":["webpack://@midscene/core/./src/service/utils.ts"],"sourcesContent":["import type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n"],"names":["createServiceDump","data","baseData","Date","finalData","uuid"],"mappings":";AAGO,SAASA,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}

package/dist/es/task-runner.mjs

@@ -105,7 +105,7 @@ class TaskRunner {
                     uiContext
                 };
                 if ('Insight' === task.type) {
-                    assert('Locate' === task.subType || 'Query' === task.subType || 'Assert' === task.subType || 'WaitFor' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported insight subType: ${task.subType}`);
+                    assert('Locate' === task.subType || 'Query' === task.subType || 'Assert' === task.subType || 'WaitFor' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported service subType: ${task.subType}`);
                     returnValue = await task.executor(param, executorContext);
                     if ('Locate' === task.subType) previousFindOutput = null == returnValue ? void 0 : returnValue.output;
                 } else if ('Action' === task.type || 'Planning' === task.type) returnValue = await task.executor(param, executorContext);