@midscene/core 0.28.12-beta-20250923080328.0 → 0.28.12-beta-20250923091649.0

package/dist/lib/agent/tasks.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  ConversationHistory,\n  elementByPositionWithElementInfo,\n  findAllMidsceneLocatorField,\n  vlmPlanning,\n} from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport {\n  type AIUsageInfo,\n  type BaseElement,\n  type DetailedLocateParam,\n  type DumpSubscriber,\n  type ExecutionRecorderItem,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskHitBy,\n  type ExecutionTaskInsightLocateApply,\n  type ExecutionTaskInsightQueryApply,\n  type ExecutionTaskPlanning,\n  type ExecutionTaskPlanningApply,\n  type ExecutionTaskProgressOptions,\n  Executor,\n  type ExecutorContext,\n  type Insight,\n  type InsightDump,\n  type InsightExtractOption,\n  type InsightExtractParam,\n  type InterfaceType,\n  type LocateResultElement,\n  type MidsceneYamlFlowItem,\n  type PlanningAIResponse,\n  type PlanningAction,\n  type PlanningActionParamError,\n  type PlanningActionParamSleep,\n  type PlanningActionParamWaitFor,\n  type PlanningLocateParam,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type UIContext,\n  plan,\n} from '@/index';\nimport { sleep } from '@/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { taskTitleStr } from './ui-utils';\nimport {\n  matchElementFromCache,\n  matchElementFromPlan,\n  parsePrompt,\n} from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  executor: Executor;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.conversationHistory = new ConversationHistory();\n  }\n\n  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {\n    const base64 = await this.interface.screenshotBase64();\n    const item: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot: base64,\n      timing,\n    };\n    return item;\n  }\n\n  private async getElementXpath(\n    uiContext: UIContext<BaseElement>,\n    element: LocateResultElement,\n  ): Promise<string[] | undefined> {\n    if (!(this.interface as any).getXpathsByPoint) {\n      debug('getXpathsByPoint is not supported for this interface');\n      return undefined;\n    }\n\n    let elementId = element?.id;\n    if (element?.isOrderSensitive !== undefined) {\n      try {\n        const xpaths = await (this.interface as any).getXpathsByPoint(\n          {\n            left: element.center[0],\n            top: element.center[1],\n          },\n          element?.isOrderSensitive,\n        );\n\n        return xpaths;\n      } catch (error) {\n        debug('getXpathsByPoint failed: %s', error);\n        return undefined;\n      }\n    }\n\n    // find the nearest xpath for the element\n    if (element?.attributes?.nodeType === NodeType.POSITION) {\n      await this.insight.contextRetrieverFn('locate');\n      const info = elementByPositionWithElementInfo(\n        uiContext.tree,\n        {\n          x: element.center[0],\n          y: element.center[1],\n        },\n        {\n          requireStrictDistance: false,\n          filterPositionElements: true,\n        },\n      );\n      if (info?.id) {\n        elementId = info.id;\n      } else {\n        debug(\n          'no element id found for position node, will not update cache',\n          element,\n        );\n      }\n    }\n\n    if (!elementId) {\n      return undefined;\n    }\n    try {\n      const result = await (this.interface as any).getXpathsById(elementId);\n      return result;\n    } catch (error) {\n      debug('getXpathsById error: ', error);\n    }\n  }\n\n  private prependExecutorWithScreenshot(\n    taskApply: ExecutionTaskApply,\n    appendAfterExecution = false,\n  ): ExecutionTaskApply {\n    const taskWithScreenshot: ExecutionTaskApply = {\n      ...taskApply,\n      executor: async (param, context, ...args) => {\n        const recorder: ExecutionRecorderItem[] = [];\n        const { task } = context;\n        // set the recorder before executor in case of error\n        task.recorder = recorder;\n        const shot = await this.recordScreenshot(`before ${task.type}`);\n        recorder.push(shot);\n\n        const result = await taskApply.executor(param, context, ...args);\n\n        if (appendAfterExecution) {\n          const shot2 = await this.recordScreenshot('after Action');\n          recorder.push(shot2);\n        }\n        return result;\n      },\n    };\n    return taskWithScreenshot;\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ) {\n    const tasks: ExecutionTaskApply[] = [];\n\n    const taskForLocatePlan = (\n      plan: PlanningAction<PlanningLocateParam>,\n      detailedLocateParam: DetailedLocateParam | string,\n      onResult?: (result: LocateResultElement) => void,\n    ): ExecutionTaskInsightLocateApply => {\n      if (typeof detailedLocateParam === 'string') {\n        detailedLocateParam = {\n          prompt: detailedLocateParam,\n        };\n      }\n      const taskFind: ExecutionTaskInsightLocateApply = {\n        type: 'Insight',\n        subType: 'Locate',\n        param: detailedLocateParam,\n        thought: plan.thought,\n        executor: async (param, taskContext) => {\n          const { task } = taskContext;\n          assert(\n            param?.prompt || param?.id || param?.bbox,\n            `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n              param,\n            )}`,\n          );\n          let insightDump: InsightDump | undefined;\n          let usage: AIUsageInfo | undefined;\n          const dumpCollector: DumpSubscriber = (dump) => {\n            insightDump = dump;\n            usage = dump?.taskInfo?.usage;\n\n            task.log = {\n              dump: insightDump,\n            };\n\n            task.usage = usage;\n          };\n          this.insight.onceDumpUpdatedFn = dumpCollector;\n          const shotTime = Date.now();\n\n          // Get context through contextRetrieverFn which handles frozen context\n          const uiContext = await this.insight.contextRetrieverFn('locate');\n          task.uiContext = uiContext;\n\n          const recordItem: ExecutionRecorderItem = {\n            type: 'screenshot',\n            ts: shotTime,\n            screenshot: uiContext.screenshotBase64,\n            timing: 'before Insight',\n          };\n          task.recorder = [recordItem];\n\n          // try matching xpath\n          const elementFromXpath =\n            param.xpath && (this.interface as any).getElementInfoByXpath\n              ? await (this.interface as any).getElementInfoByXpath(param.xpath)\n              : undefined;\n          const userExpectedPathHitFlag = !!elementFromXpath;\n\n          // try matching cache\n          const cachePrompt = param.prompt;\n          const locateCacheRecord =\n            this.taskCache?.matchLocateCache(cachePrompt);\n          const xpaths = locateCacheRecord?.cacheContent?.xpaths;\n          const elementFromCache = userExpectedPathHitFlag\n            ? null\n            : await matchElementFromCache(\n                this,\n                xpaths,\n                cachePrompt,\n                param.cacheable,\n              );\n          const cacheHitFlag = !!elementFromCache;\n\n          // try matching plan\n          const elementFromPlan =\n            !userExpectedPathHitFlag && !cacheHitFlag\n              ? matchElementFromPlan(param, uiContext.tree)\n              : undefined;\n          const planHitFlag = !!elementFromPlan;\n\n          // try ai locate\n          const elementFromAiLocate =\n            !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag\n              ? (\n                  await this.insight.locate(\n                    param,\n                    {\n                      // fallback to ai locate\n                      context: uiContext,\n                    },\n                    modelConfig,\n                  )\n                ).element\n              : undefined;\n          const aiLocateHitFlag = !!elementFromAiLocate;\n\n          const element =\n            elementFromXpath || // highest priority\n            elementFromCache || // second priority\n            elementFromPlan || // third priority\n            elementFromAiLocate;\n\n          // update cache\n          let currentXpaths: string[] | undefined;\n          if (\n            element &&\n            this.taskCache &&\n            !cacheHitFlag &&\n            param?.cacheable !== false\n          ) {\n            const elementXpaths = await this.getElementXpath(\n              uiContext,\n              element,\n            );\n            if (elementXpaths?.length) {\n              debug(\n                'update cache, prompt: %s, xpaths: %s',\n                cachePrompt,\n                elementXpaths,\n              );\n              currentXpaths = elementXpaths;\n              this.taskCache.updateOrAppendCacheRecord(\n                {\n                  type: 'locate',\n                  prompt: cachePrompt,\n                  xpaths: elementXpaths,\n                },\n                locateCacheRecord,\n              );\n            } else {\n              debug(\n                'no xpaths found, will not update cache',\n                cachePrompt,\n                elementXpaths,\n              );\n            }\n          }\n          if (!element) {\n            throw new Error(`Element not found: ${param.prompt}`);\n          }\n\n          let hitBy: ExecutionTaskHitBy | undefined;\n\n          if (userExpectedPathHitFlag) {\n            hitBy = {\n              from: 'User expected path',\n              context: {\n                xpath: param.xpath,\n              },\n            };\n          } else if (cacheHitFlag) {\n            hitBy = {\n              from: 'Cache',\n              context: {\n                xpathsFromCache: xpaths,\n                xpathsToSave: currentXpaths,\n              },\n            };\n          } else if (planHitFlag) {\n            hitBy = {\n              from: 'Planning',\n              context: {\n                id: elementFromPlan?.id,\n                bbox: elementFromPlan?.bbox,\n              },\n            };\n          } else if (aiLocateHitFlag) {\n            hitBy = {\n              from: 'AI model',\n              context: {\n                prompt: param.prompt,\n              },\n            };\n          }\n\n          onResult?.(element);\n\n          return {\n            output: {\n              element,\n            },\n            uiContext,\n            hitBy,\n          };\n        },\n      };\n      return taskFind;\n    };\n\n    for (const plan of plans) {\n      if (plan.type === 'Locate') {\n        if (\n          !plan.locate ||\n          plan.locate === null ||\n          plan.locate?.id === null ||\n          plan.locate?.id === 'null'\n        ) {\n          debug('Locate action with id is null, will be ignored', plan);\n          continue;\n        }\n        const taskLocate = taskForLocatePlan(plan, plan.locate);\n\n        tasks.push(taskLocate);\n      } else if (plan.type === 'Error') {\n        const taskActionError: ExecutionTaskActionApply<PlanningActionParamError> =\n          {\n            type: 'Action',\n            subType: 'Error',\n            param: plan.param,\n            thought: plan.thought || plan.param?.thought,\n            locate: plan.locate,\n            executor: async () => {\n              throw new Error(\n                plan?.thought || plan.param?.thought || 'error without thought',\n              );\n            },\n          };\n        tasks.push(taskActionError);\n      } else if (plan.type === 'Finished') {\n        const taskActionFinished: ExecutionTaskActionApply<null> = {\n          type: 'Action',\n          subType: 'Finished',\n          param: null,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async (param) => {},\n        };\n        tasks.push(taskActionFinished);\n      } else if (plan.type === 'Sleep') {\n        const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =\n          {\n            type: 'Action',\n            subType: 'Sleep',\n            param: plan.param,\n            thought: plan.thought,\n            locate: plan.locate,\n            executor: async (taskParam) => {\n              await sleep(taskParam?.timeMs || 3000);\n            },\n          };\n        tasks.push(taskActionSleep);\n      } else {\n        // action in action space\n        const planType = plan.type;\n        const actionSpace = await this.interface.actionSpace();\n        const action = actionSpace.find((action) => action.name === planType);\n        const param = plan.param;\n\n        if (!action) {\n          throw new Error(`Action type '${planType}' not found`);\n        }\n\n        // find all params that needs location\n        const locateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema)\n          : [];\n\n        const requiredLocateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema, true)\n          : [];\n\n        locateFields.forEach((field) => {\n          if (param[field]) {\n            const locatePlan = locatePlanForLocate(param[field]);\n            debug(\n              'will prepend locate param for field',\n              `action.type=${planType}`,\n              `param=${JSON.stringify(param[field])}`,\n              `locatePlan=${JSON.stringify(locatePlan)}`,\n            );\n            const locateTask = taskForLocatePlan(\n              locatePlan,\n              param[field],\n              (result) => {\n                param[field] = result;\n              },\n            );\n            tasks.push(locateTask);\n          } else {\n            assert(\n              !requiredLocateFields.includes(field),\n              `Required locate field '${field}' is not provided for action ${planType}`,\n            );\n            debug(`field '${field}' is not provided for action ${planType}`);\n          }\n        });\n\n        const task: ExecutionTaskApply<\n          'Action',\n          any,\n          { success: boolean; action: string; param: any },\n          void\n        > = {\n          type: 'Action',\n          subType: planType,\n          thought: plan.thought,\n          param: plan.param,\n          executor: async (param, context) => {\n            debug(\n              'executing action',\n              planType,\n              param,\n              `context.element.center: ${context.element?.center}`,\n            );\n\n            // Get context for actionSpace operations to ensure size info is available\n            const uiContext = await this.insight.contextRetrieverFn('locate');\n            context.task.uiContext = uiContext;\n\n            requiredLocateFields.forEach((field) => {\n              assert(\n                param[field],\n                `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n              );\n            });\n\n            try {\n              await Promise.all([\n                (async () => {\n                  if (this.interface.beforeInvokeAction) {\n                    debug('will call \"beforeInvokeAction\" for interface');\n                    await this.interface.beforeInvokeAction(action.name, param);\n                    debug('called \"beforeInvokeAction\" for interface');\n                  }\n                })(),\n                sleep(200),\n              ]);\n            } catch (originalError: any) {\n              const originalMessage =\n                originalError?.message || String(originalError);\n              throw new Error(\n                `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n                { cause: originalError },\n              );\n            }\n\n            debug('calling action', action.name);\n            const actionFn = action.call.bind(this.interface);\n            await actionFn(param, context);\n            debug('called action', action.name);\n\n            try {\n              if (this.interface.afterInvokeAction) {\n                debug('will call \"afterInvokeAction\" for interface');\n                await this.interface.afterInvokeAction(action.name, param);\n                debug('called \"afterInvokeAction\" for interface');\n              }\n            } catch (originalError: any) {\n              const originalMessage =\n                originalError?.message || String(originalError);\n              throw new Error(\n                `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n                { cause: originalError },\n              );\n            }\n            // Return a proper result for report generation\n            return {\n              output: {\n                success: true,\n                action: planType,\n                param: param,\n              },\n            };\n          },\n        };\n        tasks.push(task);\n      }\n    }\n\n    const wrappedTasks = tasks.map(\n      (task: ExecutionTaskApply, index: number) => {\n        if (task.type === 'Action') {\n          return this.prependExecutorWithScreenshot(\n            task,\n            index === tasks.length - 1,\n          );\n        }\n        return task;\n      },\n    );\n\n    return {\n      tasks: wrappedTasks,\n    };\n  }\n\n  private async setupPlanningContext(executorContext: ExecutorContext) {\n    const shotTime = Date.now();\n    const uiContext = await this.insight.contextRetrieverFn('locate');\n    const recordItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: shotTime,\n      screenshot: uiContext.screenshotBase64,\n      timing: 'before Planning',\n    };\n\n    executorContext.task.recorder = [recordItem];\n    (executorContext.task as ExecutionTaskPlanning).uiContext = uiContext;\n\n    return {\n      uiContext,\n    };\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        await this.setupPlanningContext(executorContext);\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n\n    await taskExecutor.append(task);\n    await taskExecutor.flush();\n\n    return {\n      executor: taskExecutor,\n    };\n  }\n\n  private createPlanningTask(\n    userInstruction: string,\n    actionContext: string | undefined,\n    modelConfig: IModelConfig,\n  ): ExecutionTaskPlanningApply {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = await this.setupPlanningContext(executorContext);\n        const { vlMode } = modelConfig;\n        const uiTarsModelVersion =\n          vlMode === 'vlm-ui-tars' ? modelConfig.uiTarsModelVersion : undefined;\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await (uiTarsModelVersion ? vlmPlanning : plan)(\n          param.userInstruction,\n          {\n            context: uiContext,\n            log: param.log,\n            actionContext,\n            interfaceType: this.interface.interfaceType as InterfaceType,\n            actionSpace,\n            modelConfig,\n            conversationHistory: this.conversationHistory,\n          },\n        );\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push({\n              type: 'Sleep',\n              param: {\n                timeMs: timeRemaining,\n              },\n              locate: null,\n            } as PlanningAction<PlanningActionParamSleep>);\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const taskExecutor = new Executor(title, {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    await taskExecutor.append(tasks);\n    const result = await taskExecutor.flush();\n    const { output } = result!;\n    return {\n      output,\n      executor: taskExecutor,\n    };\n  }\n\n  private getReplanningCycleLimit(isVlmUiTars: boolean) {\n    return (\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      (isVlmUiTars\n        ? defaultVlmUiTarsReplanningCycleLimit\n        : defaultReplanningCycleLimit)\n    );\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit = this.getReplanningCycleLimit(\n      modelConfig.vlMode === 'vlm-ui-tars',\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg =\n          'Replanning too many times, please split the task into multiple steps';\n\n        return this.appendErrorPlan(taskExecutor, errorMsg, modelConfig);\n      }\n\n      // Create planning task (automatically includes execution history if available)\n      const planningTask = this.createPlanningTask(\n        userPrompt,\n        actionContext,\n        modelConfig,\n      );\n\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      const planResult: PlanningAIResponse = result?.output;\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: planResult,\n          executor: taskExecutor,\n        };\n      }\n\n      // Execute planned actions\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans, modelConfig);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          modelConfig,\n        );\n      }\n\n      await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      // Check if task is complete\n      if (!planResult.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        // TODO: display image thumbnail in report\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let insightDump: InsightDump | undefined;\n        const dumpCollector: DumpSubscriber = (dump) => {\n          insightDump = dump;\n        };\n        this.insight.onceDumpUpdatedFn = dumpCollector;\n\n        // Get context for query operations\n        const shotTime = Date.now();\n        const uiContext = await this.insight.contextRetrieverFn('extract');\n        task.uiContext = uiContext;\n\n        const recordItem: ExecutionRecorderItem = {\n          type: 'screenshot',\n          ts: shotTime,\n          screenshot: uiContext.screenshotBase64,\n          timing: 'before Extract',\n        };\n        task.recorder = [recordItem];\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        if (ifTypeRestricted) {\n          const returnType = type === 'Assert' ? 'Boolean' : type;\n          demandInput = {\n            result: `${returnType}, ${demand}`,\n          };\n        }\n\n        const { data, usage, thought } = await this.insight.extract<any>(\n          demandInput,\n          modelConfig,\n          opt,\n          multimodalPrompt,\n        );\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else {\n            assert(data?.result !== undefined, 'No result in query data');\n            outputResult = (data as any).result;\n          }\n        }\n\n        return {\n          output: outputResult,\n          log: { dump: insightDump, isWaitForAssert: opt?.isWaitForAssert },\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const taskExecutor = new Executor(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        onTaskStart: this.onTaskStartCallback,\n      },\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n    const result = await taskExecutor.flush();\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      executor: taskExecutor,\n    };\n  }\n\n  async assert(\n    assertion: TUserPrompt,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n  ): Promise<ExecutionResult<boolean>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n    return await this.createTypeQueryExecution<boolean>(\n      'Assert',\n      textPrompt,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n  }\n\n  private async appendErrorPlan(\n    taskExecutor: Executor,\n    errorMsg: string,\n    modelConfig: IModelConfig,\n  ) {\n    const errorPlan: PlanningAction<PlanningActionParamError> = {\n      type: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      locate: null,\n    };\n    const { tasks } = await this.convertPlanToExecutable(\n      [errorPlan],\n      modelConfig,\n    );\n    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));\n    await taskExecutor.flush();\n\n    return {\n      output: undefined,\n      executor: taskExecutor,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'Assert',\n        textPrompt,\n        modelConfig,\n        {\n          isWaitForAssert: true,\n          returnThought: true,\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n      const result = (await taskExecutor.flush()) as {\n        output: boolean;\n        thought?: string;\n      };\n\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function waitFor',\n        );\n      }\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepPlan: PlanningAction<PlanningActionParamSleep> = {\n          type: 'Sleep',\n          param: {\n            timeMs: timeRemaining,\n          },\n          locate: null,\n        };\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [sleepPlan],\n          modelConfig,\n        );\n        await taskExecutor.append(\n          this.prependExecutorWithScreenshot(sleepTasks[0]),\n        );\n        await taskExecutor.flush();\n      }\n    }\n\n    return this.appendErrorPlan(\n      taskExecutor,\n      `waitFor timeout: ${errorThought}`,\n      modelConfig,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","defaultReplanningCycleLimit","defaultVlmUiTarsReplanningCycleLimit","locatePlanForLocate","param","locate","locatePlan","TaskExecutor","timing","base64","item","Date","uiContext","element","_element_attributes","elementId","undefined","xpaths","error","NodeType","info","elementByPositionWithElementInfo","result","taskApply","appendAfterExecution","taskWithScreenshot","context","args","recorder","task","shot","shot2","plans","modelConfig","tasks","taskForLocatePlan","plan","detailedLocateParam","onResult","taskFind","taskContext","_this_taskCache","_locateCacheRecord_cacheContent","assert","JSON","insightDump","usage","dumpCollector","dump","_dump_taskInfo","shotTime","recordItem","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","aiLocateHitFlag","currentXpaths","elementXpaths","Error","hitBy","_plan_locate","_plan_locate1","taskLocate","_plan_param","taskActionError","taskActionFinished","taskActionSleep","taskParam","sleep","planType","actionSpace","action","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","locateTask","_context_element","Promise","originalError","originalMessage","String","actionFn","wrappedTasks","index","executorContext","userInstruction","yamlString","taskExecutor","Executor","taskTitleStr","actionContext","startTime","vlMode","uiTarsModelVersion","Array","console","planResult","vlmPlanning","actions","log","more_actions_needed_by_instruction","rawResponse","finalActions","timeNow","timeRemaining","title","output","isVlmUiTars","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","userPrompt","replanCount","yamlFlow","replanningCycleLimit","errorMsg","planningTask","executables","type","demand","opt","multimodalPrompt","queryTask","ifTypeRestricted","demandInput","returnType","data","thought","outputResult","assertion","textPrompt","parsePrompt","errorPlan","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepPlan","sleepTasks","interfaceInstance","insight","opts","ConversationHistory"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0DA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,8BAA8B;AACpC,MAAMC,uCAAuC;AAEtC,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAEO,MAAMC;IAcX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAmBA,MAAc,iBAAiBC,MAAuC,EAAE;QACtE,MAAMC,SAAS,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB;QACpD,MAAMC,OAA8B;YAClC,MAAM;YACN,IAAIC,KAAK,GAAG;YACZ,YAAYF;YACZD;QACF;QACA,OAAOE;IACT;IAEA,MAAc,gBACZE,SAAiC,EACjCC,OAA4B,EACG;YAyB3BC;QAxBJ,IAAI,CAAE,IAAI,CAAC,SAAS,CAAS,gBAAgB,EAAE,YAC7Cf,MAAM;QAIR,IAAIgB,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE;QAC3B,IAAIA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB,AAAD,MAAMG,QAChC,IAAI;YACF,MAAMC,SAAS,MAAO,IAAI,CAAC,SAAS,CAAS,gBAAgB,CAC3D;gBACE,MAAMJ,QAAQ,MAAM,CAAC,EAAE;gBACvB,KAAKA,QAAQ,MAAM,CAAC,EAAE;YACxB,GACAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB;YAG3B,OAAOI;QACT,EAAE,OAAOC,OAAO;YACdnB,MAAM,+BAA+BmB;YACrC;QACF;QAIF,IAAIJ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,sBAAAA,QAAS,UAAU,AAAD,IAAlBA,KAAAA,IAAAA,oBAAqB,QAAQ,AAAD,MAAMK,0BAAAA,QAAAA,CAAAA,QAAiB,EAAE;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;YACtC,MAAMC,OAAOC,AAAAA,IAAAA,yBAAAA,gCAAAA,AAAAA,EACXT,UAAU,IAAI,EACd;gBACE,GAAGC,QAAQ,MAAM,CAAC,EAAE;gBACpB,GAAGA,QAAQ,MAAM,CAAC,EAAE;YACtB,GACA;gBACE,uBAAuB;gBACvB,wBAAwB;YAC1B;YAEF,IAAIO,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE,EACVL,YAAYK,KAAK,EAAE;iBAEnBrB,MACE,gEACAc;QAGN;QAEA,IAAI,CAACE,WACH;QAEF,IAAI;YACF,MAAMO,SAAS,MAAO,IAAI,CAAC,SAAS,CAAS,aAAa,CAACP;YAC3D,OAAOO;QACT,EAAE,OAAOJ,OAAO;YACdnB,MAAM,yBAAyBmB;QACjC;IACF;IAEQ,8BACNK,SAA6B,EAC7BC,uBAAuB,KAAK,EACR;QACpB,MAAMC,qBAAyC;YAC7C,GAAGF,SAAS;YACZ,UAAU,OAAOnB,OAAOsB,SAAS,GAAGC;gBAClC,MAAMC,WAAoC,EAAE;gBAC5C,MAAM,EAAEC,IAAI,EAAE,GAAGH;gBAEjBG,KAAK,QAAQ,GAAGD;gBAChB,MAAME,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAED,KAAK,IAAI,EAAE;gBAC9DD,SAAS,IAAI,CAACE;gBAEd,MAAMR,SAAS,MAAMC,UAAU,QAAQ,CAACnB,OAAOsB,YAAYC;gBAE3D,IAAIH,sBAAsB;oBACxB,MAAMO,QAAQ,MAAM,IAAI,CAAC,gBAAgB,CAAC;oBAC1CH,SAAS,IAAI,CAACG;gBAChB;gBACA,OAAOT;YACT;QACF;QACA,OAAOG;IACT;IAEA,MAAa,wBACXO,KAAuB,EACvBC,WAAyB,EACzB;QACA,MAAMC,QAA8B,EAAE;QAEtC,MAAMC,oBAAoB,CACxBC,MACAC,qBACAC;YAEA,IAAI,AAA+B,YAA/B,OAAOD,qBACTA,sBAAsB;gBACpB,QAAQA;YACV;YAEF,MAAME,WAA4C;gBAChD,MAAM;gBACN,SAAS;gBACT,OAAOF;gBACP,SAASD,KAAK,OAAO;gBACrB,UAAU,OAAOhC,OAAOoC;wBA6CpBC,iBACaC;oBA7Cf,MAAM,EAAEb,IAAI,EAAE,GAAGW;oBACjBG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEvC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAEwC,KAAK,SAAS,CACpExC,QACC;oBAEL,IAAIyC;oBACJ,IAAIC;oBACJ,MAAMC,gBAAgC,CAACC;4BAE7BC;wBADRJ,cAAcG;wBACdF,QAAQG,QAAAA,OAAAA,KAAAA,IAAAA,QAAAA,CAAAA,iBAAAA,KAAM,QAAQ,AAAD,IAAbA,KAAAA,IAAAA,eAAgB,KAAK;wBAE7BpB,KAAK,GAAG,GAAG;4BACT,MAAMgB;wBACR;wBAEAhB,KAAK,KAAK,GAAGiB;oBACf;oBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGC;oBACjC,MAAMG,WAAWvC,KAAK,GAAG;oBAGzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxDiB,KAAK,SAAS,GAAGjB;oBAEjB,MAAMuC,aAAoC;wBACxC,MAAM;wBACN,IAAID;wBACJ,YAAYtC,UAAU,gBAAgB;wBACtC,QAAQ;oBACV;oBACAiB,KAAK,QAAQ,GAAG;wBAACsB;qBAAW;oBAG5B,MAAMC,mBACJhD,MAAM,KAAK,IAAK,IAAI,CAAC,SAAS,CAAS,qBAAqB,GACxD,MAAO,IAAI,CAAC,SAAS,CAAS,qBAAqB,CAACA,MAAM,KAAK,IAC/DY;oBACN,MAAMqC,0BAA0B,CAAC,CAACD;oBAGlC,MAAME,cAAclD,MAAM,MAAM;oBAChC,MAAMmD,oBAAAA,QACJd,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACa;oBACnC,MAAMrC,SAASyB,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,MAAM;oBACtD,MAAMc,mBAAmBH,0BACrB,OACA,MAAMI,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ,IAAI,EACJxC,QACAqC,aACAlD,MAAM,SAAS;oBAErB,MAAMsD,eAAe,CAAC,CAACF;oBAGvB,MAAMG,kBACJ,AAACN,2BAA4BK,eAEzB1C,SADA4C,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqBxD,OAAOQ,UAAU,IAAI;oBAEhD,MAAMiD,cAAc,CAAC,CAACF;oBAGtB,MAAMG,sBACJ,AAACT,2BAA4BK,gBAAiBG,cAW1C7C,SATE,OAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACvBZ,OACA;wBAEE,SAASQ;oBACX,GACAqB,YAAW,EAEb,OAAO;oBAEf,MAAM8B,kBAAkB,CAAC,CAACD;oBAE1B,MAAMjD,UACJuC,oBACAI,oBACAG,mBACAG;oBAGF,IAAIE;oBACJ,IACEnD,WACA,IAAI,CAAC,SAAS,IACd,CAAC6C,gBACDtD,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OACrB;wBACA,MAAM6D,gBAAgB,MAAM,IAAI,CAAC,eAAe,CAC9CrD,WACAC;wBAEF,IAAIoD,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,EAAE;4BACzBlE,MACE,wCACAuD,aACAW;4BAEFD,gBAAgBC;4BAChB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gCACE,MAAM;gCACN,QAAQX;gCACR,QAAQW;4BACV,GACAV;wBAEJ,OACExD,MACE,0CACAuD,aACAW;oBAGN;oBACA,IAAI,CAACpD,SACH,MAAM,IAAIqD,MAAM,CAAC,mBAAmB,EAAE9D,MAAM,MAAM,EAAE;oBAGtD,IAAI+D;oBAEJ,IAAId,yBACFc,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,OAAO/D,MAAM,KAAK;wBACpB;oBACF;yBACK,IAAIsD,cACTS,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,iBAAiBlD;4BACjB,cAAc+C;wBAChB;oBACF;yBACK,IAAIH,aACTM,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,IAAIR,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;4BACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;wBAC7B;oBACF;yBACK,IAAII,iBACTI,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,QAAQ/D,MAAM,MAAM;wBACtB;oBACF;oBAGFkC,QAAAA,YAAAA,SAAWzB;oBAEX,OAAO;wBACL,QAAQ;4BACNA;wBACF;wBACAD;wBACAuD;oBACF;gBACF;YACF;YACA,OAAO5B;QACT;QAEA,KAAK,MAAMH,QAAQJ,MACjB,IAAII,AAAc,aAAdA,KAAK,IAAI,EAAe;gBAIxBgC,cACAC;YAJF,IACE,CAACjC,KAAK,MAAM,IACZA,AAAgB,SAAhBA,KAAK,MAAM,IACXgC,AAAAA,SAAAA,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,EAAE,AAAD,MAAM,QACpBC,AAAAA,SAAAA,CAAAA,gBAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,EAAE,AAAD,MAAM,QACpB;gBACAtE,MAAM,kDAAkDqC;gBACxD;YACF;YACA,MAAMkC,aAAanC,kBAAkBC,MAAMA,KAAK,MAAM;YAEtDF,MAAM,IAAI,CAACoC;QACb,OAAO,IAAIlC,AAAc,YAAdA,KAAK,IAAI,EAAc;gBAMHmC;YAL7B,MAAMC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOpC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO,aAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD;gBAC3C,QAAQnC,KAAK,MAAM;gBACnB,UAAU;wBAEWmC;oBADnB,MAAM,IAAIL,MACR9B,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAC,SAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD,KAAK;gBAE5C;YACF;YACFrC,MAAM,IAAI,CAACsC;QACb,OAAO,IAAIpC,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACnC,MAAMqC,qBAAqD;gBACzD,MAAM;gBACN,SAAS;gBACT,OAAO;gBACP,SAASrC,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOhC,SAAW;YAC9B;YACA8B,MAAM,IAAI,CAACuC;QACb,OAAO,IAAIrC,AAAc,YAAdA,KAAK,IAAI,EAAc;YAChC,MAAMsC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOtC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOuC;oBACf,MAAMC,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;gBACnC;YACF;YACFzC,MAAM,IAAI,CAACwC;QACb,OAAO;YAEL,MAAMG,WAAWzC,KAAK,IAAI;YAC1B,MAAM0C,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;YACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACC,SAAWA,OAAO,IAAI,KAAKF;YAC5D,MAAMzE,QAAQgC,KAAK,KAAK;YAExB,IAAI,CAAC2C,QACH,MAAM,IAAIb,MAAM,CAAC,aAAa,EAAEW,SAAS,WAAW,CAAC;YAIvD,MAAMG,eAAeD,SACjBE,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BF,OAAO,WAAW,IAC9C,EAAE;YAEN,MAAMG,uBAAuBH,SACzBE,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BF,OAAO,WAAW,EAAE,QAChD,EAAE;YAENC,aAAa,OAAO,CAAC,CAACG;gBACpB,IAAI/E,KAAK,CAAC+E,MAAM,EAAE;oBAChB,MAAM7E,aAAaH,oBAAoBC,KAAK,CAAC+E,MAAM;oBACnDpF,MACE,uCACA,CAAC,YAAY,EAAE8E,UAAU,EACzB,CAAC,MAAM,EAAEjC,KAAK,SAAS,CAACxC,KAAK,CAAC+E,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEvC,KAAK,SAAS,CAACtC,aAAa;oBAE5C,MAAM8E,aAAajD,kBACjB7B,YACAF,KAAK,CAAC+E,MAAM,EACZ,CAAC7D;wBACClB,KAAK,CAAC+E,MAAM,GAAG7D;oBACjB;oBAEFY,MAAM,IAAI,CAACkD;gBACb,OAAO;oBACLzC,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACuC,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEN,UAAU;oBAE3E9E,MAAM,CAAC,OAAO,EAAEoF,MAAM,6BAA6B,EAAEN,UAAU;gBACjE;YACF;YAEA,MAAMhD,OAKF;gBACF,MAAM;gBACN,SAASgD;gBACT,SAASzC,KAAK,OAAO;gBACrB,OAAOA,KAAK,KAAK;gBACjB,UAAU,OAAOhC,OAAOsB;wBAKO2D;oBAJ7BtF,MACE,oBACA8E,UACAzE,OACA,CAAC,wBAAwB,EAAE,QAAAiF,CAAAA,mBAAAA,QAAQ,OAAO,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,MAAM,EAAE;oBAItD,MAAMzE,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxDc,QAAQ,IAAI,CAAC,SAAS,GAAGd;oBAEzBsE,qBAAqB,OAAO,CAAC,CAACC;wBAC5BxC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEvC,KAAK,CAAC+E,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEN,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;oBAE9G;oBAEA,IAAI;wBACF,MAAMS,QAAQ,GAAG,CAAC;4BACf;gCACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;oCACrCvF,MAAM;oCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACgF,OAAO,IAAI,EAAE3E;oCACrDL,MAAM;gCACR;4BACF;4BACA6E,IAAAA,kCAAAA,KAAAA,AAAAA,EAAM;yBACP;oBACH,EAAE,OAAOW,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,wCAAwC,EAAEa,OAAO,IAAI,CAAC,EAAE,EAAES,iBAAiB,EAC5E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEAxF,MAAM,kBAAkBgF,OAAO,IAAI;oBACnC,MAAMW,WAAWX,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAChD,MAAMW,SAAStF,OAAOsB;oBACtB3B,MAAM,iBAAiBgF,OAAO,IAAI;oBAElC,IAAI;wBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;4BACpChF,MAAM;4BACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACgF,OAAO,IAAI,EAAE3E;4BACpDL,MAAM;wBACR;oBACF,EAAE,OAAOwF,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,uCAAuC,EAAEa,OAAO,IAAI,CAAC,EAAE,EAAES,iBAAiB,EAC3E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEA,OAAO;wBACL,QAAQ;4BACN,SAAS;4BACT,QAAQV;4BACR,OAAOzE;wBACT;oBACF;gBACF;YACF;YACA8B,MAAM,IAAI,CAACL;QACb;QAGF,MAAM8D,eAAezD,MAAM,GAAG,CAC5B,CAACL,MAA0B+D;YACzB,IAAI/D,AAAc,aAAdA,KAAK,IAAI,EACX,OAAO,IAAI,CAAC,6BAA6B,CACvCA,MACA+D,UAAU1D,MAAM,MAAM,GAAG;YAG7B,OAAOL;QACT;QAGF,OAAO;YACL,OAAO8D;QACT;IACF;IAEA,MAAc,qBAAqBE,eAAgC,EAAE;QACnE,MAAM3C,WAAWvC,KAAK,GAAG;QACzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QACxD,MAAMuC,aAAoC;YACxC,MAAM;YACN,IAAID;YACJ,YAAYtC,UAAU,gBAAgB;YACtC,QAAQ;QACV;QAEAiF,gBAAgB,IAAI,CAAC,QAAQ,GAAG;YAAC1C;SAAW;QAC3C0C,gBAAgB,IAAI,CAA2B,SAAS,GAAGjF;QAE5D,OAAO;YACLA;QACF;IACF;IAEA,MAAM,uBAAuBkF,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,eAAe,IAAIC,kCAAAA,QAAQA,CAACC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUJ,kBAAkB;YACzE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,MAAMjE,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLiE;YACF;YACA,UAAU,OAAO1F,OAAOyF;gBACtB,MAAM,IAAI,CAAC,oBAAoB,CAACA;gBAChC,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLE;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QAEA,MAAMC,aAAa,MAAM,CAACnE;QAC1B,MAAMmE,aAAa,KAAK;QAExB,OAAO;YACL,UAAUA;QACZ;IACF;IAEQ,mBACNF,eAAuB,EACvBK,aAAiC,EACjClE,WAAyB,EACG;QAC5B,MAAMJ,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLiE;YACF;YACA,UAAU,OAAO1F,OAAOyF;gBACtB,MAAMO,YAAYzF,KAAK,GAAG;gBAC1B,MAAM,EAAEC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAACiF;gBACtD,MAAM,EAAEQ,MAAM,EAAE,GAAGpE;gBACnB,MAAMqE,qBACJD,AAAW,kBAAXA,SAA2BpE,YAAY,kBAAkB,GAAGjB;gBAE9D2B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMmC,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD/E,MACE,sCACA+E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDpC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4D,MAAM,OAAO,CAACzB,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpB0B,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAOH,AAAAA,CAAAA,qBAAqBI,yBAAAA,WAAWA,GAAGtE,kCAAAA,IAAG,AAAHA,EAC3DhC,MAAM,eAAe,EACrB;oBACE,SAASQ;oBACT,KAAKR,MAAM,GAAG;oBACd+F;oBACA,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CrB;oBACA7C;oBACA,qBAAqB,IAAI,CAAC,mBAAmB;gBAC/C;gBAGF,MAAM,EACJ0E,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClC3F,KAAK,EACL4B,KAAK,EACLgE,WAAW,EACXlC,KAAK,EACN,GAAG6B;gBAEJZ,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCiB;gBACF;gBACAjB,gBAAgB,IAAI,CAAC,KAAK,GAAG/C;gBAE7B,MAAMiE,eAAeJ,WAAW,EAAE;gBAElC,IAAI/B,OAAO;oBACT,MAAMoC,UAAUrG,KAAK,GAAG;oBACxB,MAAMsG,gBAAgBrC,QAASoC,CAAAA,UAAUZ,SAAQ;oBACjD,IAAIa,gBAAgB,GAClBF,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,OAAO;4BACL,QAAQE;wBACV;wBACA,QAAQ;oBACV;gBAEJ;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBpE,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACkE,sCAAsCjC,OACvC1D,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAAS6F;wBACTF;wBACAD;wBACA,UAAUH,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA7F;gBACF;YACF;QACF;QAEA,OAAOiB;IACT;IAEA,MAAM,SACJqF,KAAa,EACblF,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAM+D,eAAe,IAAIC,kCAAAA,QAAQA,CAACiB,OAAO;YACvC,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEhF,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACF,OAAOC;QAC5D,MAAM+D,aAAa,MAAM,CAAC9D;QAC1B,MAAMZ,SAAS,MAAM0E,aAAa,KAAK;QACvC,MAAM,EAAEmB,MAAM,EAAE,GAAG7F;QACnB,OAAO;YACL6F;YACA,UAAUnB;QACZ;IACF;IAEQ,wBAAwBoB,WAAoB,EAAE;QACpD,OACE,IAAI,CAAC,oBAAoB,IACzBC,oBAAAA,mBAAAA,CAAAA,oBAAwC,CACtCC,oBAAAA,+BAA+BA,KAEhCF,CAAAA,cACGlH,uCACAD,2BAA0B;IAElC;IAEA,MAAM,OACJsH,UAAkB,EAClBtF,WAAyB,EACzBkE,aAAsB,EAQtB;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMH,eAAe,IAAIC,kCAAAA,QAAQA,CAACC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,IAAIC,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBAAuB,IAAI,CAAC,uBAAuB,CACvDzF,AAAuB,kBAAvBA,YAAY,MAAM;QAIpB,MAAO,KAAM;YACX,IAAIuF,cAAcE,sBAAsB;gBACtC,MAAMC,WACJ;gBAEF,OAAO,IAAI,CAAC,eAAe,CAAC3B,cAAc2B,UAAU1F;YACtD;YAGA,MAAM2F,eAAe,IAAI,CAAC,kBAAkB,CAC1CL,YACApB,eACAlE;YAGF,MAAM+D,aAAa,MAAM,CAAC4B;YAC1B,MAAMtG,SAAS,MAAM0E,aAAa,KAAK;YACvC,MAAMS,aAAiCnF,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAI0E,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQS;gBACR,UAAUT;YACZ;YAIF,MAAMhE,QAAQyE,WAAW,OAAO,IAAI,EAAE;YACtCgB,SAAS,IAAI,IAAKhB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAIoB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAAC7F,OAAOC;gBACxD+D,aAAa,MAAM,CAAC6B,YAAY,KAAK;YACvC,EAAE,OAAO3G,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzB8E,cACA,CAAC,4CAA4C,EAAE9E,MAAM,SAAS,EAAE0B,KAAK,SAAS,CAC5EZ,QACC,EACHC;YAEJ;YAEA,MAAM+D,aAAa,KAAK;YACxB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQhF;gBACR,UAAUgF;YACZ;YAIF,IAAI,CAACS,WAAW,kCAAkC,EAChD;YAIFe;QACF;QAEA,OAAO;YACL,QAAQ;gBACNC;YACF;YACA,UAAUzB;QACZ;IACF;IAEQ,oBACN8B,IAA0D,EAC1DC,MAA2B,EAC3B9F,WAAyB,EACzB+F,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBAEL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO3H,OAAOoC;gBACtB,MAAM,EAAEX,IAAI,EAAE,GAAGW;gBACjB,IAAIK;gBACJ,MAAME,gBAAgC,CAACC;oBACrCH,cAAcG;gBAChB;gBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGD;gBAGjC,MAAMG,WAAWvC,KAAK,GAAG;gBACzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;gBACxDiB,KAAK,SAAS,GAAGjB;gBAEjB,MAAMuC,aAAoC;oBACxC,MAAM;oBACN,IAAID;oBACJ,YAAYtC,UAAU,gBAAgB;oBACtC,QAAQ;gBACV;gBACAiB,KAAK,QAAQ,GAAG;oBAACsB;iBAAW;gBAE5B,MAAMgF,mBAAmBL,AAAS,YAATA;gBACzB,IAAIM,cAAcL;gBAClB,IAAII,kBAAkB;oBACpB,MAAME,aAAaP,AAAS,aAATA,OAAoB,YAAYA;oBACnDM,cAAc;wBACZ,QAAQ,GAAGC,WAAW,EAAE,EAAEN,QAAQ;oBACpC;gBACF;gBAEA,MAAM,EAAEO,IAAI,EAAExF,KAAK,EAAEyF,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACzDH,aACAnG,aACA+F,KACAC;gBAGF,IAAIO,eAAeF;gBACnB,IAAIH,kBAEF,IAAI,AAAgB,YAAhB,OAAOG,MACTE,eAAeF;qBACV;oBACL3F,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2F,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,MAAMtH,QAAW;oBACnCwH,eAAgBF,KAAa,MAAM;gBACrC;gBAGF,OAAO;oBACL,QAAQE;oBACR,KAAK;wBAAE,MAAM3F;wBAAa,iBAAiBmF,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe;oBAAC;oBAChElF;oBACAyF;gBACF;YACF;QACF;QAEA,OAAOL;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3B9F,WAAyB,EACzB+F,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMjC,eAAe,IAAIC,kCAAAA,QAAQA,CAC/BC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACE4B,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASnF,KAAK,SAAS,CAACmF,UAEvD;YACE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAGF,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACA9F,aACA+F,KACAC;QAGF,MAAMjC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACkC;QAC7D,MAAM5G,SAAS,MAAM0E,aAAa,KAAK;QAEvC,IAAI,CAAC1E,QACH,MAAM,IAAI4C,MACR;QAIJ,MAAM,EAAEiD,MAAM,EAAEoB,OAAO,EAAE,GAAGjH;QAE5B,OAAO;YACL6F;YACAoB;YACA,UAAUvC;QACZ;IACF;IAEA,MAAM,OACJyC,SAAsB,EACtBxG,WAAyB,EACzB+F,GAA0B,EACS;QACnC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,AAAAA,IAAAA,oCAAAA,WAAAA,AAAAA,EAAYF;QACrD,OAAO,MAAM,IAAI,CAAC,wBAAwB,CACxC,UACAC,YACAzG,aACA+F,KACAC;IAEJ;IAEA,MAAc,gBACZjC,YAAsB,EACtB2B,QAAgB,EAChB1F,WAAyB,EACzB;QACA,MAAM2G,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACL,SAASjB;YACX;YACA,QAAQ;QACV;QACA,MAAM,EAAEzF,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClD;YAAC0G;SAAU,EACX3G;QAEF,MAAM+D,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC9D,KAAK,CAAC,EAAE;QACrE,MAAM8D,aAAa,KAAK;QAExB,OAAO;YACL,QAAQhF;YACR,UAAUgF;QACZ;IACF;IAEA,MAAM,QACJyC,SAAsB,EACtBT,GAA+B,EAC/B/F,WAAyB,EACO;QAChC,MAAM,EAAEyG,UAAU,EAAET,gBAAgB,EAAE,GAAGU,AAAAA,IAAAA,oCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMI,cAAc,CAAC,SAAS,EAAEH,YAAY;QAC5C,MAAM1C,eAAe,IAAIC,kCAAAA,QAAQA,CAACC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAW2C,cAAc;YACtE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE,GAAGf;QAEvCrF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO8F,WAAW;QAClB9F,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOmG,WAAW;QAClBnG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOoG,iBAAiB;QAExBpG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEoG,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBrI,KAAK,GAAG;QACjC,IAAIyF,YAAYzF,KAAK,GAAG;QACxB,IAAIsI,eAAe;QACnB,MAAOtI,KAAK,GAAG,KAAKqI,mBAAmBF,UAAW;YAChD1C,YAAYzF,KAAK,GAAG;YACpB,MAAMuH,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,UACAQ,YACAzG,aACA;gBACE,iBAAiB;gBACjB,eAAe;gBACf,iBAAiB;YACnB,GACAgG;YAGF,MAAMjC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACkC;YAC7D,MAAM5G,SAAU,MAAM0E,aAAa,KAAK;YAKxC,IAAI,CAAC1E,QACH,MAAM,IAAI4C,MACR;YAIJ,IAAI5C,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQN;gBACR,UAAUgF;YACZ;YAGFiD,eACE3H,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACd,CAAC,0CAA0C,EAAEoH,YAAY;YAC3D,MAAMQ,MAAMvI,KAAK,GAAG;YACpB,IAAIuI,MAAM9C,YAAY2C,iBAAiB;gBACrC,MAAM9B,gBAAgB8B,kBAAmBG,CAAAA,MAAM9C,SAAQ;gBACvD,MAAM+C,YAAsD;oBAC1D,MAAM;oBACN,OAAO;wBACL,QAAQlC;oBACV;oBACA,QAAQ;gBACV;gBACA,MAAM,EAAE,OAAOmC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAACD;iBAAU,EACXlH;gBAEF,MAAM+D,aAAa,MAAM,CACvB,IAAI,CAAC,6BAA6B,CAACoD,UAAU,CAAC,EAAE;gBAElD,MAAMpD,aAAa,KAAK;YAC1B;QACF;QAEA,OAAO,IAAI,CAAC,eAAe,CACzBA,cACA,CAAC,iBAAiB,EAAEiD,cAAc,EAClChH;IAEJ;IAphCA,YACEoH,iBAAoC,EACpCC,OAAgB,EAChBC,IAIC,CACD;QAzBF;QAEA;QAEA;QAEA,uBAAQ,uBAAR;QAEA;QAEA;QAgBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,mBAAmB,GAAG,IAAIC,yBAAAA,mBAAmBA;IACpD;AAsgCF"}
+{"version":3,"file":"agent/tasks.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  type ChatCompletionMessageParam,\n  elementByPositionWithElementInfo,\n  findAllMidsceneLocatorField,\n  resizeImageForUiTars,\n  vlmPlanning,\n} from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport {\n  type AIUsageInfo,\n  type BaseElement,\n  type DetailedLocateParam,\n  type DumpSubscriber,\n  type ExecutionRecorderItem,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskHitBy,\n  type ExecutionTaskInsightLocateApply,\n  type ExecutionTaskInsightQueryApply,\n  type ExecutionTaskPlanning,\n  type ExecutionTaskPlanningApply,\n  type ExecutionTaskProgressOptions,\n  Executor,\n  type ExecutorContext,\n  type Insight,\n  type InsightDump,\n  type InsightExtractOption,\n  type InsightExtractParam,\n  type InterfaceType,\n  type LocateResultElement,\n  type MidsceneYamlFlowItem,\n  type PlanningAIResponse,\n  type PlanningAction,\n  type PlanningActionParamError,\n  type PlanningActionParamSleep,\n  type PlanningActionParamWaitFor,\n  type PlanningLocateParam,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type UIContext,\n  plan,\n} from '@/index';\nimport { sleep } from '@/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { taskTitleStr } from './ui-utils';\nimport {\n  matchElementFromCache,\n  matchElementFromPlan,\n  parsePrompt,\n} from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  executor: Executor;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  conversationHistory: ChatCompletionMessageParam[] = [];\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n  }\n\n  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {\n    const base64 = await this.interface.screenshotBase64();\n    const item: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot: base64,\n      timing,\n    };\n    return item;\n  }\n\n  private async getElementXpath(\n    uiContext: UIContext<BaseElement>,\n    element: LocateResultElement,\n  ): Promise<string[] | undefined> {\n    if (!(this.interface as any).getXpathsByPoint) {\n      debug('getXpathsByPoint is not supported for this interface');\n      return undefined;\n    }\n\n    let elementId = element?.id;\n    if (element?.isOrderSensitive !== undefined) {\n      try {\n        const xpaths = await (this.interface as any).getXpathsByPoint(\n          {\n            left: element.center[0],\n            top: element.center[1],\n          },\n          element?.isOrderSensitive,\n        );\n\n        return xpaths;\n      } catch (error) {\n        debug('getXpathsByPoint failed: %s', error);\n        return undefined;\n      }\n    }\n\n    // find the nearest xpath for the element\n    if (element?.attributes?.nodeType === NodeType.POSITION) {\n      await this.insight.contextRetrieverFn('locate');\n      const info = elementByPositionWithElementInfo(\n        uiContext.tree,\n        {\n          x: element.center[0],\n          y: element.center[1],\n        },\n        {\n          requireStrictDistance: false,\n          filterPositionElements: true,\n        },\n      );\n      if (info?.id) {\n        elementId = info.id;\n      } else {\n        debug(\n          'no element id found for position node, will not update cache',\n          element,\n        );\n      }\n    }\n\n    if (!elementId) {\n      return undefined;\n    }\n    try {\n      const result = await (this.interface as any).getXpathsById(elementId);\n      return result;\n    } catch (error) {\n      debug('getXpathsById error: ', error);\n    }\n  }\n\n  private prependExecutorWithScreenshot(\n    taskApply: ExecutionTaskApply,\n    appendAfterExecution = false,\n  ): ExecutionTaskApply {\n    const taskWithScreenshot: ExecutionTaskApply = {\n      ...taskApply,\n      executor: async (param, context, ...args) => {\n        const recorder: ExecutionRecorderItem[] = [];\n        const { task } = context;\n        // set the recorder before executor in case of error\n        task.recorder = recorder;\n        const shot = await this.recordScreenshot(`before ${task.type}`);\n        recorder.push(shot);\n\n        const result = await taskApply.executor(param, context, ...args);\n\n        if (appendAfterExecution) {\n          const shot2 = await this.recordScreenshot('after Action');\n          recorder.push(shot2);\n        }\n        return result;\n      },\n    };\n    return taskWithScreenshot;\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ) {\n    const tasks: ExecutionTaskApply[] = [];\n\n    const taskForLocatePlan = (\n      plan: PlanningAction<PlanningLocateParam>,\n      detailedLocateParam: DetailedLocateParam | string,\n      onResult?: (result: LocateResultElement) => void,\n    ): ExecutionTaskInsightLocateApply => {\n      if (typeof detailedLocateParam === 'string') {\n        detailedLocateParam = {\n          prompt: detailedLocateParam,\n        };\n      }\n      const taskFind: ExecutionTaskInsightLocateApply = {\n        type: 'Insight',\n        subType: 'Locate',\n        param: detailedLocateParam,\n        thought: plan.thought,\n        executor: async (param, taskContext) => {\n          const { task } = taskContext;\n          assert(\n            param?.prompt || param?.id || param?.bbox,\n            `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n              param,\n            )}`,\n          );\n          let insightDump: InsightDump | undefined;\n          let usage: AIUsageInfo | undefined;\n          const dumpCollector: DumpSubscriber = (dump) => {\n            insightDump = dump;\n            usage = dump?.taskInfo?.usage;\n\n            task.log = {\n              dump: insightDump,\n            };\n\n            task.usage = usage;\n          };\n          this.insight.onceDumpUpdatedFn = dumpCollector;\n          const shotTime = Date.now();\n\n          // Get context through contextRetrieverFn which handles frozen context\n          const uiContext = await this.insight.contextRetrieverFn('locate');\n          task.uiContext = uiContext;\n\n          const recordItem: ExecutionRecorderItem = {\n            type: 'screenshot',\n            ts: shotTime,\n            screenshot: uiContext.screenshotBase64,\n            timing: 'before Insight',\n          };\n          task.recorder = [recordItem];\n\n          // try matching xpath\n          const elementFromXpath =\n            param.xpath && (this.interface as any).getElementInfoByXpath\n              ? await (this.interface as any).getElementInfoByXpath(param.xpath)\n              : undefined;\n          const userExpectedPathHitFlag = !!elementFromXpath;\n\n          // try matching cache\n          const cachePrompt = param.prompt;\n          const locateCacheRecord =\n            this.taskCache?.matchLocateCache(cachePrompt);\n          const xpaths = locateCacheRecord?.cacheContent?.xpaths;\n          const elementFromCache = userExpectedPathHitFlag\n            ? null\n            : await matchElementFromCache(\n                this,\n                xpaths,\n                cachePrompt,\n                param.cacheable,\n              );\n          const cacheHitFlag = !!elementFromCache;\n\n          // try matching plan\n          const elementFromPlan =\n            !userExpectedPathHitFlag && !cacheHitFlag\n              ? matchElementFromPlan(param, uiContext.tree)\n              : undefined;\n          const planHitFlag = !!elementFromPlan;\n\n          // try ai locate\n          const elementFromAiLocate =\n            !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag\n              ? (\n                  await this.insight.locate(\n                    param,\n                    {\n                      // fallback to ai locate\n                      context: uiContext,\n                    },\n                    modelConfig,\n                  )\n                ).element\n              : undefined;\n          const aiLocateHitFlag = !!elementFromAiLocate;\n\n          const element =\n            elementFromXpath || // highest priority\n            elementFromCache || // second priority\n            elementFromPlan || // third priority\n            elementFromAiLocate;\n\n          // update cache\n          let currentXpaths: string[] | undefined;\n          if (\n            element &&\n            this.taskCache &&\n            !cacheHitFlag &&\n            param?.cacheable !== false\n          ) {\n            const elementXpaths = await this.getElementXpath(\n              uiContext,\n              element,\n            );\n            if (elementXpaths?.length) {\n              debug(\n                'update cache, prompt: %s, xpaths: %s',\n                cachePrompt,\n                elementXpaths,\n              );\n              currentXpaths = elementXpaths;\n              this.taskCache.updateOrAppendCacheRecord(\n                {\n                  type: 'locate',\n                  prompt: cachePrompt,\n                  xpaths: elementXpaths,\n                },\n                locateCacheRecord,\n              );\n            } else {\n              debug(\n                'no xpaths found, will not update cache',\n                cachePrompt,\n                elementXpaths,\n              );\n            }\n          }\n          if (!element) {\n            throw new Error(`Element not found: ${param.prompt}`);\n          }\n\n          let hitBy: ExecutionTaskHitBy | undefined;\n\n          if (userExpectedPathHitFlag) {\n            hitBy = {\n              from: 'User expected path',\n              context: {\n                xpath: param.xpath,\n              },\n            };\n          } else if (cacheHitFlag) {\n            hitBy = {\n              from: 'Cache',\n              context: {\n                xpathsFromCache: xpaths,\n                xpathsToSave: currentXpaths,\n              },\n            };\n          } else if (planHitFlag) {\n            hitBy = {\n              from: 'Planning',\n              context: {\n                id: elementFromPlan?.id,\n                bbox: elementFromPlan?.bbox,\n              },\n            };\n          } else if (aiLocateHitFlag) {\n            hitBy = {\n              from: 'AI model',\n              context: {\n                prompt: param.prompt,\n              },\n            };\n          }\n\n          onResult?.(element);\n\n          return {\n            output: {\n              element,\n            },\n            uiContext,\n            hitBy,\n          };\n        },\n      };\n      return taskFind;\n    };\n\n    for (const plan of plans) {\n      if (plan.type === 'Locate') {\n        if (\n          !plan.locate ||\n          plan.locate === null ||\n          plan.locate?.id === null ||\n          plan.locate?.id === 'null'\n        ) {\n          debug('Locate action with id is null, will be ignored', plan);\n          continue;\n        }\n        const taskLocate = taskForLocatePlan(plan, plan.locate);\n\n        tasks.push(taskLocate);\n      } else if (plan.type === 'Error') {\n        const taskActionError: ExecutionTaskActionApply<PlanningActionParamError> =\n          {\n            type: 'Action',\n            subType: 'Error',\n            param: plan.param,\n            thought: plan.thought || plan.param?.thought,\n            locate: plan.locate,\n            executor: async () => {\n              throw new Error(\n                plan?.thought || plan.param?.thought || 'error without thought',\n              );\n            },\n          };\n        tasks.push(taskActionError);\n      } else if (plan.type === 'Finished') {\n        const taskActionFinished: ExecutionTaskActionApply<null> = {\n          type: 'Action',\n          subType: 'Finished',\n          param: null,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async (param) => {},\n        };\n        tasks.push(taskActionFinished);\n      } else if (plan.type === 'Sleep') {\n        const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =\n          {\n            type: 'Action',\n            subType: 'Sleep',\n            param: plan.param,\n            thought: plan.thought,\n            locate: plan.locate,\n            executor: async (taskParam) => {\n              await sleep(taskParam?.timeMs || 3000);\n            },\n          };\n        tasks.push(taskActionSleep);\n      } else {\n        // action in action space\n        const planType = plan.type;\n        const actionSpace = await this.interface.actionSpace();\n        const action = actionSpace.find((action) => action.name === planType);\n        const param = plan.param;\n\n        if (!action) {\n          throw new Error(`Action type '${planType}' not found`);\n        }\n\n        // find all params that needs location\n        const locateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema)\n          : [];\n\n        const requiredLocateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema, true)\n          : [];\n\n        locateFields.forEach((field) => {\n          if (param[field]) {\n            const locatePlan = locatePlanForLocate(param[field]);\n            debug(\n              'will prepend locate param for field',\n              `action.type=${planType}`,\n              `param=${JSON.stringify(param[field])}`,\n              `locatePlan=${JSON.stringify(locatePlan)}`,\n            );\n            const locateTask = taskForLocatePlan(\n              locatePlan,\n              param[field],\n              (result) => {\n                param[field] = result;\n              },\n            );\n            tasks.push(locateTask);\n          } else {\n            assert(\n              !requiredLocateFields.includes(field),\n              `Required locate field '${field}' is not provided for action ${planType}`,\n            );\n            debug(`field '${field}' is not provided for action ${planType}`);\n          }\n        });\n\n        const task: ExecutionTaskApply<\n          'Action',\n          any,\n          { success: boolean; action: string; param: any },\n          void\n        > = {\n          type: 'Action',\n          subType: planType,\n          thought: plan.thought,\n          param: plan.param,\n          executor: async (param, context) => {\n            debug(\n              'executing action',\n              planType,\n              param,\n              `context.element.center: ${context.element?.center}`,\n            );\n\n            // Get context for actionSpace operations to ensure size info is available\n            const uiContext = await this.insight.contextRetrieverFn('locate');\n            context.task.uiContext = uiContext;\n\n            requiredLocateFields.forEach((field) => {\n              assert(\n                param[field],\n                `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n              );\n            });\n\n            try {\n              await Promise.all([\n                (async () => {\n                  if (this.interface.beforeInvokeAction) {\n                    debug('will call \"beforeInvokeAction\" for interface');\n                    await this.interface.beforeInvokeAction(action.name, param);\n                    debug('called \"beforeInvokeAction\" for interface');\n                  }\n                })(),\n                sleep(200),\n              ]);\n            } catch (originalError: any) {\n              const originalMessage =\n                originalError?.message || String(originalError);\n              throw new Error(\n                `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n                { cause: originalError },\n              );\n            }\n\n            debug('calling action', action.name);\n            const actionFn = action.call.bind(this.interface);\n            await actionFn(param, context);\n            debug('called action', action.name);\n\n            try {\n              if (this.interface.afterInvokeAction) {\n                debug('will call \"afterInvokeAction\" for interface');\n                await this.interface.afterInvokeAction(action.name, param);\n                debug('called \"afterInvokeAction\" for interface');\n              }\n            } catch (originalError: any) {\n              const originalMessage =\n                originalError?.message || String(originalError);\n              throw new Error(\n                `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n                { cause: originalError },\n              );\n            }\n            // Return a proper result for report generation\n            return {\n              output: {\n                success: true,\n                action: planType,\n                param: param,\n              },\n            };\n          },\n        };\n        tasks.push(task);\n      }\n    }\n\n    const wrappedTasks = tasks.map(\n      (task: ExecutionTaskApply, index: number) => {\n        if (task.type === 'Action') {\n          return this.prependExecutorWithScreenshot(\n            task,\n            index === tasks.length - 1,\n          );\n        }\n        return task;\n      },\n    );\n\n    return {\n      tasks: wrappedTasks,\n    };\n  }\n\n  private async setupPlanningContext(executorContext: ExecutorContext) {\n    const shotTime = Date.now();\n    const uiContext = await this.insight.contextRetrieverFn('locate');\n    const recordItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: shotTime,\n      screenshot: uiContext.screenshotBase64,\n      timing: 'before Planning',\n    };\n\n    executorContext.task.recorder = [recordItem];\n    (executorContext.task as ExecutionTaskPlanning).uiContext = uiContext;\n\n    return {\n      uiContext,\n    };\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        await this.setupPlanningContext(executorContext);\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n\n    await taskExecutor.append(task);\n    await taskExecutor.flush();\n\n    return {\n      executor: taskExecutor,\n    };\n  }\n\n  private planningTaskFromPrompt(\n    userInstruction: string,\n    opts: {\n      log?: string;\n      actionContext?: string;\n      modelConfig: IModelConfig;\n    },\n  ) {\n    const { log, actionContext, modelConfig } = opts;\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n        log,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = await this.setupPlanningContext(executorContext);\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await plan(param.userInstruction, {\n          context: uiContext,\n          log: param.log,\n          actionContext,\n          interfaceType: this.interface.interfaceType as InterfaceType,\n          actionSpace,\n          modelConfig,\n        });\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        // TODO: check locate result\n        // let bboxCollected = false;\n        // (actions || []).reduce<PlanningAction[]>(\n        //   (acc, planningAction) => {\n        //     // TODO: magic field \"locate\" is used to indicate the action requires a locate\n        //     if (planningAction.locate) {\n        //       // we only collect bbox once, let qwen re-locate in the following steps\n        //       if (bboxCollected && planningAction.locate.bbox) {\n        //         // biome-ignore lint/performance/noDelete: <explanation>\n        //         delete planningAction.locate.bbox;\n        //       }\n\n        //       if (planningAction.locate.bbox) {\n        //         bboxCollected = true;\n        //       }\n\n        //       acc.push({\n        //         type: 'Locate',\n        //         locate: planningAction.locate,\n        //         param: null,\n        //         // thought is prompt created by ai, always a string\n        //         thought: planningAction.locate.prompt as string,\n        //       });\n        //     }\n        //     acc.push(planningAction);\n        //     return acc;\n        //   },\n        //   [],\n        // );\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push({\n              type: 'Sleep',\n              param: {\n                timeMs: timeRemaining,\n              },\n              locate: null,\n            } as PlanningAction<PlanningActionParamSleep>);\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  private planningTaskToGoal(\n    userInstruction: string,\n    opts: {\n      modelConfig: IModelConfig;\n    },\n  ) {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = await this.setupPlanningContext(executorContext);\n        const { modelConfig } = opts;\n        const { uiTarsModelVersion } = modelConfig;\n\n        const imagePayload = await resizeImageForUiTars(\n          uiContext.screenshotBase64,\n          uiContext.size,\n          uiTarsModelVersion,\n        );\n\n        this.appendConversationHistory({\n          role: 'user',\n          content: [\n            {\n              type: 'image_url',\n              image_url: {\n                url: imagePayload,\n              },\n            },\n          ],\n        });\n        const planResult: {\n          actions: PlanningAction<any>[];\n          action_summary: string;\n          usage?: AIUsageInfo;\n          yamlFlow?: MidsceneYamlFlowItem[];\n          rawResponse?: string;\n        } = await vlmPlanning({\n          userInstruction: param.userInstruction,\n          conversationHistory: this.conversationHistory,\n          size: uiContext.size,\n          modelConfig,\n        });\n\n        const { actions, action_summary, usage } = planResult;\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse: planResult.rawResponse,\n        };\n        executorContext.task.usage = usage;\n        this.appendConversationHistory({\n          role: 'assistant',\n          content: action_summary,\n        });\n        return {\n          output: {\n            actions,\n            thought: actions[0]?.thought,\n            actionType: actions[0].type,\n            more_actions_needed_by_instruction: true,\n            log: '',\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const taskExecutor = new Executor(title, {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    await taskExecutor.append(tasks);\n    const result = await taskExecutor.flush();\n    const { output } = result!;\n    return {\n      output,\n      executor: taskExecutor,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    let planningTask: ExecutionTaskPlanningApply | null =\n      this.planningTaskFromPrompt(userPrompt, {\n        log: undefined,\n        actionContext,\n        modelConfig,\n      });\n    let replanCount = 0;\n    const logList: string[] = [];\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      defaultReplanningCycleLimit;\n    while (planningTask) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg =\n          'Replanning too many times, please split the task into multiple steps';\n\n        return this.appendErrorPlan(taskExecutor, errorMsg, modelConfig);\n      }\n\n      // plan\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      const planResult: PlanningAIResponse = result?.output;\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: planResult,\n          executor: taskExecutor,\n        };\n      }\n\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans, modelConfig);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          modelConfig,\n        );\n      }\n\n      await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (planResult?.log) {\n        logList.push(planResult.log);\n      }\n\n      if (!planResult.more_actions_needed_by_instruction) {\n        planningTask = null;\n        break;\n      }\n      planningTask = this.planningTaskFromPrompt(userPrompt, {\n        log: logList.length > 0 ? `- ${logList.join('\\n- ')}` : undefined,\n        actionContext,\n        modelConfig,\n      });\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  async actionToGoal(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    this.conversationHistory = [];\n    const isCompleted = false;\n    let currentActionCount = 0;\n    const maxActionNumber = 40;\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    while (!isCompleted && currentActionCount < maxActionNumber) {\n      currentActionCount++;\n      debug(\n        'actionToGoal, currentActionCount:',\n        currentActionCount,\n        'userPrompt:',\n        userPrompt,\n      );\n\n      const planningTask: ExecutionTaskPlanningApply = this.planningTaskToGoal(\n        userPrompt,\n        {\n          modelConfig,\n        },\n      );\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function actionToGoal',\n        );\n      }\n      const { output } = result;\n      const plans = output.actions;\n      yamlFlow.push(...(output.yamlFlow || []));\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans, modelConfig);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          modelConfig,\n        );\n      }\n\n      await taskExecutor.flush();\n\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      if (plans[0].type === 'Finished') {\n        break;\n      }\n    }\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        // TODO: display image thumbnail in report\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let insightDump: InsightDump | undefined;\n        const dumpCollector: DumpSubscriber = (dump) => {\n          insightDump = dump;\n        };\n        this.insight.onceDumpUpdatedFn = dumpCollector;\n\n        // Get context for query operations\n        const shotTime = Date.now();\n        const uiContext = await this.insight.contextRetrieverFn('extract');\n        task.uiContext = uiContext;\n\n        const recordItem: ExecutionRecorderItem = {\n          type: 'screenshot',\n          ts: shotTime,\n          screenshot: uiContext.screenshotBase64,\n          timing: 'before Extract',\n        };\n        task.recorder = [recordItem];\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        if (ifTypeRestricted) {\n          const returnType = type === 'Assert' ? 'Boolean' : type;\n          demandInput = {\n            result: `${returnType}, ${demand}`,\n          };\n        }\n\n        const { data, usage, thought } = await this.insight.extract<any>(\n          demandInput,\n          modelConfig,\n          opt,\n          multimodalPrompt,\n        );\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else {\n            assert(data?.result !== undefined, 'No result in query data');\n            outputResult = (data as any).result;\n          }\n        }\n\n        return {\n          output: outputResult,\n          log: { dump: insightDump, isWaitForAssert: opt?.isWaitForAssert },\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const taskExecutor = new Executor(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        onTaskStart: this.onTaskStartCallback,\n      },\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n    const result = await taskExecutor.flush();\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      executor: taskExecutor,\n    };\n  }\n\n  async assert(\n    assertion: TUserPrompt,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n  ): Promise<ExecutionResult<boolean>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n    return await this.createTypeQueryExecution<boolean>(\n      'Assert',\n      textPrompt,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n  }\n\n  /**\n   * Append a message to the conversation history\n   * For user messages with images:\n   * - Keep max 4 user image messages in history\n   * - Remove oldest user image message when limit reached\n   * For assistant messages:\n   * - Simply append to history\n   * @param conversationHistory Message to append\n   */\n  private appendConversationHistory(\n    conversationHistory: ChatCompletionMessageParam,\n  ) {\n    if (conversationHistory.role === 'user') {\n      // Get all existing user messages with images\n      const userImgItems = this.conversationHistory.filter(\n        (item) => item.role === 'user',\n      );\n\n      // If we already have 4 user image messages\n      if (userImgItems.length >= 4 && conversationHistory.role === 'user') {\n        // Remove first user image message when we already have 4, before adding new one\n        const firstUserImgIndex = this.conversationHistory.findIndex(\n          (item) => item.role === 'user',\n        );\n        if (firstUserImgIndex >= 0) {\n          this.conversationHistory.splice(firstUserImgIndex, 1);\n        }\n      }\n    }\n    // For non-user messages, simply append to history\n    this.conversationHistory.push(conversationHistory);\n  }\n\n  private async appendErrorPlan(\n    taskExecutor: Executor,\n    errorMsg: string,\n    modelConfig: IModelConfig,\n  ) {\n    const errorPlan: PlanningAction<PlanningActionParamError> = {\n      type: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      locate: null,\n    };\n    const { tasks } = await this.convertPlanToExecutable(\n      [errorPlan],\n      modelConfig,\n    );\n    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));\n    await taskExecutor.flush();\n\n    return {\n      output: undefined,\n      executor: taskExecutor,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'Assert',\n        textPrompt,\n        modelConfig,\n        {\n          isWaitForAssert: true,\n          returnThought: true,\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n      const result = (await taskExecutor.flush()) as {\n        output: boolean;\n        thought?: string;\n      };\n\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function waitFor',\n        );\n      }\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepPlan: PlanningAction<PlanningActionParamSleep> = {\n          type: 'Sleep',\n          param: {\n            timeMs: timeRemaining,\n          },\n          locate: null,\n        };\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [sleepPlan],\n          modelConfig,\n        );\n        await taskExecutor.append(\n          this.prependExecutorWithScreenshot(sleepTasks[0]),\n        );\n        await taskExecutor.flush();\n      }\n    }\n\n    return this.appendErrorPlan(\n      taskExecutor,\n      `waitFor timeout: ${errorThought}`,\n      modelConfig,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","defaultReplanningCycleLimit","locatePlanForLocate","param","locate","locatePlan","TaskExecutor","timing","base64","item","Date","uiContext","element","_element_attributes","elementId","undefined","xpaths","error","NodeType","info","elementByPositionWithElementInfo","result","taskApply","appendAfterExecution","taskWithScreenshot","context","args","recorder","task","shot","shot2","plans","modelConfig","tasks","taskForLocatePlan","plan","detailedLocateParam","onResult","taskFind","taskContext","_this_taskCache","_locateCacheRecord_cacheContent","assert","JSON","insightDump","usage","dumpCollector","dump","_dump_taskInfo","shotTime","recordItem","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","aiLocateHitFlag","currentXpaths","elementXpaths","Error","hitBy","_plan_locate","_plan_locate1","taskLocate","_plan_param","taskActionError","taskActionFinished","taskActionSleep","taskParam","sleep","planType","actionSpace","action","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","locateTask","_context_element","Promise","originalError","originalMessage","String","actionFn","wrappedTasks","index","executorContext","userInstruction","yamlString","taskExecutor","Executor","taskTitleStr","opts","log","actionContext","startTime","Array","console","planResult","actions","more_actions_needed_by_instruction","rawResponse","finalActions","timeNow","timeRemaining","_actions_","uiTarsModelVersion","imagePayload","resizeImageForUiTars","vlmPlanning","action_summary","title","output","userPrompt","planningTask","replanCount","logList","yamlFlow","replanningCycleLimit","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","errorMsg","executables","isCompleted","currentActionCount","maxActionNumber","type","demand","opt","multimodalPrompt","queryTask","ifTypeRestricted","demandInput","returnType","data","thought","outputResult","assertion","textPrompt","parsePrompt","conversationHistory","userImgItems","firstUserImgIndex","errorPlan","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepPlan","sleepTasks","interfaceInstance","insight"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;AC2DA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,8BAA8B;AAE7B,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAEO,MAAMC;IAcX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAkBA,MAAc,iBAAiBC,MAAuC,EAAE;QACtE,MAAMC,SAAS,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB;QACpD,MAAMC,OAA8B;YAClC,MAAM;YACN,IAAIC,KAAK,GAAG;YACZ,YAAYF;YACZD;QACF;QACA,OAAOE;IACT;IAEA,MAAc,gBACZE,SAAiC,EACjCC,OAA4B,EACG;YAyB3BC;QAxBJ,IAAI,CAAE,IAAI,CAAC,SAAS,CAAS,gBAAgB,EAAE,YAC7Cd,MAAM;QAIR,IAAIe,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE;QAC3B,IAAIA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB,AAAD,MAAMG,QAChC,IAAI;YACF,MAAMC,SAAS,MAAO,IAAI,CAAC,SAAS,CAAS,gBAAgB,CAC3D;gBACE,MAAMJ,QAAQ,MAAM,CAAC,EAAE;gBACvB,KAAKA,QAAQ,MAAM,CAAC,EAAE;YACxB,GACAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB;YAG3B,OAAOI;QACT,EAAE,OAAOC,OAAO;YACdlB,MAAM,+BAA+BkB;YACrC;QACF;QAIF,IAAIJ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,sBAAAA,QAAS,UAAU,AAAD,IAAlBA,KAAAA,IAAAA,oBAAqB,QAAQ,AAAD,MAAMK,0BAAAA,QAAAA,CAAAA,QAAiB,EAAE;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;YACtC,MAAMC,OAAOC,AAAAA,IAAAA,yBAAAA,gCAAAA,AAAAA,EACXT,UAAU,IAAI,EACd;gBACE,GAAGC,QAAQ,MAAM,CAAC,EAAE;gBACpB,GAAGA,QAAQ,MAAM,CAAC,EAAE;YACtB,GACA;gBACE,uBAAuB;gBACvB,wBAAwB;YAC1B;YAEF,IAAIO,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE,EACVL,YAAYK,KAAK,EAAE;iBAEnBpB,MACE,gEACAa;QAGN;QAEA,IAAI,CAACE,WACH;QAEF,IAAI;YACF,MAAMO,SAAS,MAAO,IAAI,CAAC,SAAS,CAAS,aAAa,CAACP;YAC3D,OAAOO;QACT,EAAE,OAAOJ,OAAO;YACdlB,MAAM,yBAAyBkB;QACjC;IACF;IAEQ,8BACNK,SAA6B,EAC7BC,uBAAuB,KAAK,EACR;QACpB,MAAMC,qBAAyC;YAC7C,GAAGF,SAAS;YACZ,UAAU,OAAOnB,OAAOsB,SAAS,GAAGC;gBAClC,MAAMC,WAAoC,EAAE;gBAC5C,MAAM,EAAEC,IAAI,EAAE,GAAGH;gBAEjBG,KAAK,QAAQ,GAAGD;gBAChB,MAAME,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAED,KAAK,IAAI,EAAE;gBAC9DD,SAAS,IAAI,CAACE;gBAEd,MAAMR,SAAS,MAAMC,UAAU,QAAQ,CAACnB,OAAOsB,YAAYC;gBAE3D,IAAIH,sBAAsB;oBACxB,MAAMO,QAAQ,MAAM,IAAI,CAAC,gBAAgB,CAAC;oBAC1CH,SAAS,IAAI,CAACG;gBAChB;gBACA,OAAOT;YACT;QACF;QACA,OAAOG;IACT;IAEA,MAAa,wBACXO,KAAuB,EACvBC,WAAyB,EACzB;QACA,MAAMC,QAA8B,EAAE;QAEtC,MAAMC,oBAAoB,CACxBC,MACAC,qBACAC;YAEA,IAAI,AAA+B,YAA/B,OAAOD,qBACTA,sBAAsB;gBACpB,QAAQA;YACV;YAEF,MAAME,WAA4C;gBAChD,MAAM;gBACN,SAAS;gBACT,OAAOF;gBACP,SAASD,KAAK,OAAO;gBACrB,UAAU,OAAOhC,OAAOoC;wBA6CpBC,iBACaC;oBA7Cf,MAAM,EAAEb,IAAI,EAAE,GAAGW;oBACjBG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEvC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAEwC,KAAK,SAAS,CACpExC,QACC;oBAEL,IAAIyC;oBACJ,IAAIC;oBACJ,MAAMC,gBAAgC,CAACC;4BAE7BC;wBADRJ,cAAcG;wBACdF,QAAQG,QAAAA,OAAAA,KAAAA,IAAAA,QAAAA,CAAAA,iBAAAA,KAAM,QAAQ,AAAD,IAAbA,KAAAA,IAAAA,eAAgB,KAAK;wBAE7BpB,KAAK,GAAG,GAAG;4BACT,MAAMgB;wBACR;wBAEAhB,KAAK,KAAK,GAAGiB;oBACf;oBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGC;oBACjC,MAAMG,WAAWvC,KAAK,GAAG;oBAGzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxDiB,KAAK,SAAS,GAAGjB;oBAEjB,MAAMuC,aAAoC;wBACxC,MAAM;wBACN,IAAID;wBACJ,YAAYtC,UAAU,gBAAgB;wBACtC,QAAQ;oBACV;oBACAiB,KAAK,QAAQ,GAAG;wBAACsB;qBAAW;oBAG5B,MAAMC,mBACJhD,MAAM,KAAK,IAAK,IAAI,CAAC,SAAS,CAAS,qBAAqB,GACxD,MAAO,IAAI,CAAC,SAAS,CAAS,qBAAqB,CAACA,MAAM,KAAK,IAC/DY;oBACN,MAAMqC,0BAA0B,CAAC,CAACD;oBAGlC,MAAME,cAAclD,MAAM,MAAM;oBAChC,MAAMmD,oBAAAA,QACJd,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACa;oBACnC,MAAMrC,SAASyB,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,MAAM;oBACtD,MAAMc,mBAAmBH,0BACrB,OACA,MAAMI,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ,IAAI,EACJxC,QACAqC,aACAlD,MAAM,SAAS;oBAErB,MAAMsD,eAAe,CAAC,CAACF;oBAGvB,MAAMG,kBACJ,AAACN,2BAA4BK,eAEzB1C,SADA4C,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqBxD,OAAOQ,UAAU,IAAI;oBAEhD,MAAMiD,cAAc,CAAC,CAACF;oBAGtB,MAAMG,sBACJ,AAACT,2BAA4BK,gBAAiBG,cAW1C7C,SATE,OAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACvBZ,OACA;wBAEE,SAASQ;oBACX,GACAqB,YAAW,EAEb,OAAO;oBAEf,MAAM8B,kBAAkB,CAAC,CAACD;oBAE1B,MAAMjD,UACJuC,oBACAI,oBACAG,mBACAG;oBAGF,IAAIE;oBACJ,IACEnD,WACA,IAAI,CAAC,SAAS,IACd,CAAC6C,gBACDtD,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OACrB;wBACA,MAAM6D,gBAAgB,MAAM,IAAI,CAAC,eAAe,CAC9CrD,WACAC;wBAEF,IAAIoD,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,EAAE;4BACzBjE,MACE,wCACAsD,aACAW;4BAEFD,gBAAgBC;4BAChB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gCACE,MAAM;gCACN,QAAQX;gCACR,QAAQW;4BACV,GACAV;wBAEJ,OACEvD,MACE,0CACAsD,aACAW;oBAGN;oBACA,IAAI,CAACpD,SACH,MAAM,IAAIqD,MAAM,CAAC,mBAAmB,EAAE9D,MAAM,MAAM,EAAE;oBAGtD,IAAI+D;oBAEJ,IAAId,yBACFc,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,OAAO/D,MAAM,KAAK;wBACpB;oBACF;yBACK,IAAIsD,cACTS,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,iBAAiBlD;4BACjB,cAAc+C;wBAChB;oBACF;yBACK,IAAIH,aACTM,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,IAAIR,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;4BACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;wBAC7B;oBACF;yBACK,IAAII,iBACTI,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,QAAQ/D,MAAM,MAAM;wBACtB;oBACF;oBAGFkC,QAAAA,YAAAA,SAAWzB;oBAEX,OAAO;wBACL,QAAQ;4BACNA;wBACF;wBACAD;wBACAuD;oBACF;gBACF;YACF;YACA,OAAO5B;QACT;QAEA,KAAK,MAAMH,QAAQJ,MACjB,IAAII,AAAc,aAAdA,KAAK,IAAI,EAAe;gBAIxBgC,cACAC;YAJF,IACE,CAACjC,KAAK,MAAM,IACZA,AAAgB,SAAhBA,KAAK,MAAM,IACXgC,AAAAA,SAAAA,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,EAAE,AAAD,MAAM,QACpBC,AAAAA,SAAAA,CAAAA,gBAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,EAAE,AAAD,MAAM,QACpB;gBACArE,MAAM,kDAAkDoC;gBACxD;YACF;YACA,MAAMkC,aAAanC,kBAAkBC,MAAMA,KAAK,MAAM;YAEtDF,MAAM,IAAI,CAACoC;QACb,OAAO,IAAIlC,AAAc,YAAdA,KAAK,IAAI,EAAc;gBAMHmC;YAL7B,MAAMC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOpC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO,aAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD;gBAC3C,QAAQnC,KAAK,MAAM;gBACnB,UAAU;wBAEWmC;oBADnB,MAAM,IAAIL,MACR9B,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAC,SAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD,KAAK;gBAE5C;YACF;YACFrC,MAAM,IAAI,CAACsC;QACb,OAAO,IAAIpC,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACnC,MAAMqC,qBAAqD;gBACzD,MAAM;gBACN,SAAS;gBACT,OAAO;gBACP,SAASrC,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOhC,SAAW;YAC9B;YACA8B,MAAM,IAAI,CAACuC;QACb,OAAO,IAAIrC,AAAc,YAAdA,KAAK,IAAI,EAAc;YAChC,MAAMsC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOtC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOuC;oBACf,MAAMC,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;gBACnC;YACF;YACFzC,MAAM,IAAI,CAACwC;QACb,OAAO;YAEL,MAAMG,WAAWzC,KAAK,IAAI;YAC1B,MAAM0C,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;YACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACC,SAAWA,OAAO,IAAI,KAAKF;YAC5D,MAAMzE,QAAQgC,KAAK,KAAK;YAExB,IAAI,CAAC2C,QACH,MAAM,IAAIb,MAAM,CAAC,aAAa,EAAEW,SAAS,WAAW,CAAC;YAIvD,MAAMG,eAAeD,SACjBE,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BF,OAAO,WAAW,IAC9C,EAAE;YAEN,MAAMG,uBAAuBH,SACzBE,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BF,OAAO,WAAW,EAAE,QAChD,EAAE;YAENC,aAAa,OAAO,CAAC,CAACG;gBACpB,IAAI/E,KAAK,CAAC+E,MAAM,EAAE;oBAChB,MAAM7E,aAAaH,oBAAoBC,KAAK,CAAC+E,MAAM;oBACnDnF,MACE,uCACA,CAAC,YAAY,EAAE6E,UAAU,EACzB,CAAC,MAAM,EAAEjC,KAAK,SAAS,CAACxC,KAAK,CAAC+E,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEvC,KAAK,SAAS,CAACtC,aAAa;oBAE5C,MAAM8E,aAAajD,kBACjB7B,YACAF,KAAK,CAAC+E,MAAM,EACZ,CAAC7D;wBACClB,KAAK,CAAC+E,MAAM,GAAG7D;oBACjB;oBAEFY,MAAM,IAAI,CAACkD;gBACb,OAAO;oBACLzC,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACuC,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEN,UAAU;oBAE3E7E,MAAM,CAAC,OAAO,EAAEmF,MAAM,6BAA6B,EAAEN,UAAU;gBACjE;YACF;YAEA,MAAMhD,OAKF;gBACF,MAAM;gBACN,SAASgD;gBACT,SAASzC,KAAK,OAAO;gBACrB,OAAOA,KAAK,KAAK;gBACjB,UAAU,OAAOhC,OAAOsB;wBAKO2D;oBAJ7BrF,MACE,oBACA6E,UACAzE,OACA,CAAC,wBAAwB,EAAE,QAAAiF,CAAAA,mBAAAA,QAAQ,OAAO,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,MAAM,EAAE;oBAItD,MAAMzE,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxDc,QAAQ,IAAI,CAAC,SAAS,GAAGd;oBAEzBsE,qBAAqB,OAAO,CAAC,CAACC;wBAC5BxC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEvC,KAAK,CAAC+E,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEN,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;oBAE9G;oBAEA,IAAI;wBACF,MAAMS,QAAQ,GAAG,CAAC;4BACf;gCACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;oCACrCtF,MAAM;oCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC+E,OAAO,IAAI,EAAE3E;oCACrDJ,MAAM;gCACR;4BACF;4BACA4E,IAAAA,kCAAAA,KAAAA,AAAAA,EAAM;yBACP;oBACH,EAAE,OAAOW,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,wCAAwC,EAAEa,OAAO,IAAI,CAAC,EAAE,EAAES,iBAAiB,EAC5E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEAvF,MAAM,kBAAkB+E,OAAO,IAAI;oBACnC,MAAMW,WAAWX,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAChD,MAAMW,SAAStF,OAAOsB;oBACtB1B,MAAM,iBAAiB+E,OAAO,IAAI;oBAElC,IAAI;wBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;4BACpC/E,MAAM;4BACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC+E,OAAO,IAAI,EAAE3E;4BACpDJ,MAAM;wBACR;oBACF,EAAE,OAAOuF,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,uCAAuC,EAAEa,OAAO,IAAI,CAAC,EAAE,EAAES,iBAAiB,EAC3E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEA,OAAO;wBACL,QAAQ;4BACN,SAAS;4BACT,QAAQV;4BACR,OAAOzE;wBACT;oBACF;gBACF;YACF;YACA8B,MAAM,IAAI,CAACL;QACb;QAGF,MAAM8D,eAAezD,MAAM,GAAG,CAC5B,CAACL,MAA0B+D;YACzB,IAAI/D,AAAc,aAAdA,KAAK,IAAI,EACX,OAAO,IAAI,CAAC,6BAA6B,CACvCA,MACA+D,UAAU1D,MAAM,MAAM,GAAG;YAG7B,OAAOL;QACT;QAGF,OAAO;YACL,OAAO8D;QACT;IACF;IAEA,MAAc,qBAAqBE,eAAgC,EAAE;QACnE,MAAM3C,WAAWvC,KAAK,GAAG;QACzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QACxD,MAAMuC,aAAoC;YACxC,MAAM;YACN,IAAID;YACJ,YAAYtC,UAAU,gBAAgB;YACtC,QAAQ;QACV;QAEAiF,gBAAgB,IAAI,CAAC,QAAQ,GAAG;YAAC1C;SAAW;QAC3C0C,gBAAgB,IAAI,CAA2B,SAAS,GAAGjF;QAE5D,OAAO;YACLA;QACF;IACF;IAEA,MAAM,uBAAuBkF,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,eAAe,IAAIC,kCAAAA,QAAQA,CAACC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUJ,kBAAkB;YACzE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,MAAMjE,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLiE;YACF;YACA,UAAU,OAAO1F,OAAOyF;gBACtB,MAAM,IAAI,CAAC,oBAAoB,CAACA;gBAChC,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLE;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QAEA,MAAMC,aAAa,MAAM,CAACnE;QAC1B,MAAMmE,aAAa,KAAK;QAExB,OAAO;YACL,UAAUA;QACZ;IACF;IAEQ,uBACNF,eAAuB,EACvBK,IAIC,EACD;QACA,MAAM,EAAEC,GAAG,EAAEC,aAAa,EAAEpE,WAAW,EAAE,GAAGkE;QAC5C,MAAMtE,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLiE;gBACAM;YACF;YACA,UAAU,OAAOhG,OAAOyF;gBACtB,MAAMS,YAAY3F,KAAK,GAAG;gBAC1B,MAAM,EAAEC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAACiF;gBAEtDlD,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMmC,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD9E,MACE,sCACA8E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDpC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4D,MAAM,OAAO,CAACzB,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpB0B,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAMrE,AAAAA,IAAAA,kCAAAA,IAAAA,AAAAA,EAAKhC,MAAM,eAAe,EAAE;oBACnD,SAASQ;oBACT,KAAKR,MAAM,GAAG;oBACdiG;oBACA,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CvB;oBACA7C;gBACF;gBAEA,MAAM,EACJyE,OAAO,EACPN,GAAG,EACHO,kCAAkC,EAClCzF,KAAK,EACL4B,KAAK,EACL8D,WAAW,EACXhC,KAAK,EACN,GAAG6B;gBAEJZ,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCe;gBACF;gBACAf,gBAAgB,IAAI,CAAC,KAAK,GAAG/C;gBAE7B,MAAM+D,eAAeH,WAAW,EAAE;gBAgClC,IAAI9B,OAAO;oBACT,MAAMkC,UAAUnG,KAAK,GAAG;oBACxB,MAAMoG,gBAAgBnC,QAASkC,CAAAA,UAAUR,SAAQ;oBACjD,IAAIS,gBAAgB,GAClBF,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,OAAO;4BACL,QAAQE;wBACV;wBACA,QAAQ;oBACV;gBAEJ;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBlE,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACgE,sCAAsC/B,OACvC1D,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAAS2F;wBACTF;wBACAP;wBACA,UAAUK,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA7F;gBACF;YACF;QACF;QAEA,OAAOiB;IACT;IAEQ,mBACNiE,eAAuB,EACvBK,IAEC,EACD;QACA,MAAMtE,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLiE;YACF;YACA,UAAU,OAAO1F,OAAOyF;oBAgDTmB;gBA/Cb,MAAM,EAAEpG,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAACiF;gBACtD,MAAM,EAAE5D,WAAW,EAAE,GAAGkE;gBACxB,MAAM,EAAEc,kBAAkB,EAAE,GAAGhF;gBAE/B,MAAMiF,eAAe,MAAMC,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EACzBvG,UAAU,gBAAgB,EAC1BA,UAAU,IAAI,EACdqG;gBAGF,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,WAAW;gCACT,KAAKC;4BACP;wBACF;qBACD;gBACH;gBACA,MAAMT,aAMF,MAAMW,AAAAA,IAAAA,yBAAAA,WAAAA,AAAAA,EAAY;oBACpB,iBAAiBhH,MAAM,eAAe;oBACtC,qBAAqB,IAAI,CAAC,mBAAmB;oBAC7C,MAAMQ,UAAU,IAAI;oBACpBqB;gBACF;gBAEA,MAAM,EAAEyE,OAAO,EAAEW,cAAc,EAAEvE,KAAK,EAAE,GAAG2D;gBAC3CZ,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClC,aAAaY,WAAW,WAAW;gBACrC;gBACAZ,gBAAgB,IAAI,CAAC,KAAK,GAAG/C;gBAC7B,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAASuE;gBACX;gBACA,OAAO;oBACL,QAAQ;wBACNX;wBACA,SAAS,QAAAM,CAAAA,YAAAA,OAAO,CAAC,EAAE,AAAD,IAATA,KAAAA,IAAAA,UAAY,OAAO;wBAC5B,YAAYN,OAAO,CAAC,EAAE,CAAC,IAAI;wBAC3B,oCAAoC;wBACpC,KAAK;wBACL,UAAUD,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;gBACF;YACF;QACF;QAEA,OAAO5E;IACT;IAEA,MAAM,SACJyF,KAAa,EACbtF,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAM+D,eAAe,IAAIC,kCAAAA,QAAQA,CAACqB,OAAO;YACvC,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEpF,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACF,OAAOC;QAC5D,MAAM+D,aAAa,MAAM,CAAC9D;QAC1B,MAAMZ,SAAS,MAAM0E,aAAa,KAAK;QACvC,MAAM,EAAEuB,MAAM,EAAE,GAAGjG;QACnB,OAAO;YACLiG;YACA,UAAUvB;QACZ;IACF;IAEA,MAAM,OACJwB,UAAkB,EAClBvF,WAAyB,EACzBoE,aAAsB,EAQtB;QACA,MAAML,eAAe,IAAIC,kCAAAA,QAAQA,CAACC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUsB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,IAAIC,eACF,IAAI,CAAC,sBAAsB,CAACD,YAAY;YACtC,KAAKxG;YACLqF;YACApE;QACF;QACF,IAAIyF,cAAc;QAClB,MAAMC,UAAoB,EAAE;QAE5B,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJ,IAAI,CAAC,oBAAoB,IACzBC,oBAAAA,mBAAAA,CAAAA,oBAAwC,CACtCC,oBAAAA,+BAA+BA,KAEjC7H;QACF,MAAOuH,aAAc;YACnB,IAAIC,cAAcG,sBAAsB;gBACtC,MAAMG,WACJ;gBAEF,OAAO,IAAI,CAAC,eAAe,CAAChC,cAAcgC,UAAU/F;YACtD;YAGA,MAAM+D,aAAa,MAAM,CAACyB;YAC1B,MAAMnG,SAAS,MAAM0E,aAAa,KAAK;YACvC,MAAMS,aAAiCnF,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAI0E,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQS;gBACR,UAAUT;YACZ;YAGF,MAAMhE,QAAQyE,WAAW,OAAO,IAAI,EAAE;YACtCmB,SAAS,IAAI,IAAKnB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAIwB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAACjG,OAAOC;gBACxD+D,aAAa,MAAM,CAACiC,YAAY,KAAK;YACvC,EAAE,OAAO/G,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzB8E,cACA,CAAC,4CAA4C,EAAE9E,MAAM,SAAS,EAAE0B,KAAK,SAAS,CAC5EZ,QACC,EACHC;YAEJ;YAEA,MAAM+D,aAAa,KAAK;YACxB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQhF;gBACR,UAAUgF;YACZ;YAEF,IAAIS,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,GAAG,EACjBkB,QAAQ,IAAI,CAAClB,WAAW,GAAG;YAG7B,IAAI,CAACA,WAAW,kCAAkC,EAAE;gBAClDgB,eAAe;gBACf;YACF;YACAA,eAAe,IAAI,CAAC,sBAAsB,CAACD,YAAY;gBACrD,KAAKG,QAAQ,MAAM,GAAG,IAAI,CAAC,EAAE,EAAEA,QAAQ,IAAI,CAAC,SAAS,GAAG3G;gBACxDqF;gBACApE;YACF;YACAyF;QACF;QAEA,OAAO;YACL,QAAQ;gBACNE;YACF;YACA,UAAU5B;QACZ;IACF;IAEA,MAAM,aACJwB,UAAkB,EAClBvF,WAAyB,EAQzB;QACA,MAAM+D,eAAe,IAAIC,kCAAAA,QAAQA,CAACC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUsB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,IAAI,CAAC,mBAAmB,GAAG,EAAE;QAC7B,MAAMU,cAAc;QACpB,IAAIC,qBAAqB;QACzB,MAAMC,kBAAkB;QAExB,MAAMR,WAAmC,EAAE;QAC3C,MAAO,CAACM,eAAeC,qBAAqBC,gBAAiB;YAC3DD;YACAnI,MACE,qCACAmI,oBACA,eACAX;YAGF,MAAMC,eAA2C,IAAI,CAAC,kBAAkB,CACtED,YACA;gBACEvF;YACF;YAEF,MAAM+D,aAAa,MAAM,CAACyB;YAC1B,MAAMnG,SAAS,MAAM0E,aAAa,KAAK;YACvC,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQhF;gBACR,UAAUgF;YACZ;YAEF,IAAI,CAAC1E,QACH,MAAM,IAAI4C,MACR;YAGJ,MAAM,EAAEqD,MAAM,EAAE,GAAGjG;YACnB,MAAMU,QAAQuF,OAAO,OAAO;YAC5BK,SAAS,IAAI,IAAKL,OAAO,QAAQ,IAAI,EAAE;YACvC,IAAIU;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAACjG,OAAOC;gBACxD+D,aAAa,MAAM,CAACiC,YAAY,KAAK;YACvC,EAAE,OAAO/G,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzB8E,cACA,CAAC,4CAA4C,EAAE9E,MAAM,SAAS,EAAE0B,KAAK,SAAS,CAC5EZ,QACC,EACHC;YAEJ;YAEA,MAAM+D,aAAa,KAAK;YAExB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQhF;gBACR,UAAUgF;YACZ;YAGF,IAAIhE,AAAkB,eAAlBA,KAAK,CAAC,EAAE,CAAC,IAAI,EACf;QAEJ;QACA,OAAO;YACL,QAAQ;gBACN4F;YACF;YACA,UAAU5B;QACZ;IACF;IAEQ,oBACNqC,IAA0D,EAC1DC,MAA2B,EAC3BrG,WAAyB,EACzBsG,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBAEL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAOlI,OAAOoC;gBACtB,MAAM,EAAEX,IAAI,EAAE,GAAGW;gBACjB,IAAIK;gBACJ,MAAME,gBAAgC,CAACC;oBACrCH,cAAcG;gBAChB;gBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGD;gBAGjC,MAAMG,WAAWvC,KAAK,GAAG;gBACzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;gBACxDiB,KAAK,SAAS,GAAGjB;gBAEjB,MAAMuC,aAAoC;oBACxC,MAAM;oBACN,IAAID;oBACJ,YAAYtC,UAAU,gBAAgB;oBACtC,QAAQ;gBACV;gBACAiB,KAAK,QAAQ,GAAG;oBAACsB;iBAAW;gBAE5B,MAAMuF,mBAAmBL,AAAS,YAATA;gBACzB,IAAIM,cAAcL;gBAClB,IAAII,kBAAkB;oBACpB,MAAME,aAAaP,AAAS,aAATA,OAAoB,YAAYA;oBACnDM,cAAc;wBACZ,QAAQ,GAAGC,WAAW,EAAE,EAAEN,QAAQ;oBACpC;gBACF;gBAEA,MAAM,EAAEO,IAAI,EAAE/F,KAAK,EAAEgG,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACzDH,aACA1G,aACAsG,KACAC;gBAGF,IAAIO,eAAeF;gBACnB,IAAIH,kBAEF,IAAI,AAAgB,YAAhB,OAAOG,MACTE,eAAeF;qBACV;oBACLlG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOkG,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,MAAM7H,QAAW;oBACnC+H,eAAgBF,KAAa,MAAM;gBACrC;gBAGF,OAAO;oBACL,QAAQE;oBACR,KAAK;wBAAE,MAAMlG;wBAAa,iBAAiB0F,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe;oBAAC;oBAChEzF;oBACAgG;gBACF;YACF;QACF;QAEA,OAAOL;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BrG,WAAyB,EACzBsG,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMxC,eAAe,IAAIC,kCAAAA,QAAQA,CAC/BC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEmC,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAAS1F,KAAK,SAAS,CAAC0F,UAEvD;YACE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAGF,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACArG,aACAsG,KACAC;QAGF,MAAMxC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACyC;QAC7D,MAAMnH,SAAS,MAAM0E,aAAa,KAAK;QAEvC,IAAI,CAAC1E,QACH,MAAM,IAAI4C,MACR;QAIJ,MAAM,EAAEqD,MAAM,EAAEuB,OAAO,EAAE,GAAGxH;QAE5B,OAAO;YACLiG;YACAuB;YACA,UAAU9C;QACZ;IACF;IAEA,MAAM,OACJgD,SAAsB,EACtB/G,WAAyB,EACzBsG,GAA0B,EACS;QACnC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,AAAAA,IAAAA,oCAAAA,WAAAA,AAAAA,EAAYF;QACrD,OAAO,MAAM,IAAI,CAAC,wBAAwB,CACxC,UACAC,YACAhH,aACAsG,KACAC;IAEJ;IAWQ,0BACNW,mBAA+C,EAC/C;QACA,IAAIA,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;YAEvC,MAAMC,eAAe,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAClD,CAAC1I,OAASA,AAAc,WAAdA,KAAK,IAAI;YAIrB,IAAI0I,aAAa,MAAM,IAAI,KAAKD,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;gBAEnE,MAAME,oBAAoB,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAC1D,CAAC3I,OAASA,AAAc,WAAdA,KAAK,IAAI;gBAErB,IAAI2I,qBAAqB,GACvB,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAACA,mBAAmB;YAEvD;QACF;QAEA,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACF;IAChC;IAEA,MAAc,gBACZnD,YAAsB,EACtBgC,QAAgB,EAChB/F,WAAyB,EACzB;QACA,MAAMqH,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACL,SAAStB;YACX;YACA,QAAQ;QACV;QACA,MAAM,EAAE9F,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClD;YAACoH;SAAU,EACXrH;QAEF,MAAM+D,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC9D,KAAK,CAAC,EAAE;QACrE,MAAM8D,aAAa,KAAK;QAExB,OAAO;YACL,QAAQhF;YACR,UAAUgF;QACZ;IACF;IAEA,MAAM,QACJgD,SAAsB,EACtBT,GAA+B,EAC/BtG,WAAyB,EACO;QAChC,MAAM,EAAEgH,UAAU,EAAET,gBAAgB,EAAE,GAAGU,AAAAA,IAAAA,oCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMO,cAAc,CAAC,SAAS,EAAEN,YAAY;QAC5C,MAAMjD,eAAe,IAAIC,kCAAAA,QAAQA,CAACC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAWqD,cAAc;YACtE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE,GAAGlB;QAEvC5F,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqG,WAAW;QAClBrG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO6G,WAAW;QAClB7G,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO8G,iBAAiB;QAExB9G,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8G,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmB/I,KAAK,GAAG;QACjC,IAAI2F,YAAY3F,KAAK,GAAG;QACxB,IAAIgJ,eAAe;QACnB,MAAOhJ,KAAK,GAAG,KAAK+I,mBAAmBF,UAAW;YAChDlD,YAAY3F,KAAK,GAAG;YACpB,MAAM8H,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,UACAQ,YACAhH,aACA;gBACE,iBAAiB;gBACjB,eAAe;gBACf,iBAAiB;YACnB,GACAuG;YAGF,MAAMxC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACyC;YAC7D,MAAMnH,SAAU,MAAM0E,aAAa,KAAK;YAKxC,IAAI,CAAC1E,QACH,MAAM,IAAI4C,MACR;YAIJ,IAAI5C,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQN;gBACR,UAAUgF;YACZ;YAGF2D,eACErI,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACd,CAAC,0CAA0C,EAAE2H,YAAY;YAC3D,MAAMW,MAAMjJ,KAAK,GAAG;YACpB,IAAIiJ,MAAMtD,YAAYmD,iBAAiB;gBACrC,MAAM1C,gBAAgB0C,kBAAmBG,CAAAA,MAAMtD,SAAQ;gBACvD,MAAMuD,YAAsD;oBAC1D,MAAM;oBACN,OAAO;wBACL,QAAQ9C;oBACV;oBACA,QAAQ;gBACV;gBACA,MAAM,EAAE,OAAO+C,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAACD;iBAAU,EACX5H;gBAEF,MAAM+D,aAAa,MAAM,CACvB,IAAI,CAAC,6BAA6B,CAAC8D,UAAU,CAAC,EAAE;gBAElD,MAAM9D,aAAa,KAAK;YAC1B;QACF;QAEA,OAAO,IAAI,CAAC,eAAe,CACzBA,cACA,CAAC,iBAAiB,EAAE2D,cAAc,EAClC1H;IAEJ;IA7uCA,YACE8H,iBAAoC,EACpCC,OAAgB,EAChB7D,IAIC,CACD;QAzBF;QAEA;QAEA;QAEA,8CAAoD,EAAE;QAEtD;QAEA;QAgBE,IAAI,CAAC,SAAS,GAAG4D;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAG7D,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;IACvD;AAguCF"}

package/dist/lib/agent/utils.js

@@ -186,7 +186,7 @@ function trimContextByViewport(execution) {
         }) : execution.tasks
     };
 }
-const getMidsceneVersion = ()=>"0.28.12-beta-20250923080328.0";
+const getMidsceneVersion = ()=>"0.28.12-beta-20250923091649.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/index.js

@@ -32,20 +32,19 @@ __webpack_require__.d(__webpack_exports__, {
     AiLocateSection: ()=>external_inspect_js_namespaceObject.AiLocateSection,
     AIActionType: ()=>external_common_js_namespaceObject.AIActionType,
     TUserPromptSchema: ()=>external_common_js_namespaceObject.TUserPromptSchema,
-    ConversationHistory: ()=>external_conversation_history_js_namespaceObject.ConversationHistory,
     callAI: ()=>index_js_namespaceObject.callAI,
-    AiExtractElementInfo: ()=>external_inspect_js_namespaceObject.AiExtractElementInfo,
     findAllMidsceneLocatorField: ()=>external_common_js_namespaceObject.findAllMidsceneLocatorField,
+    AiExtractElementInfo: ()=>external_inspect_js_namespaceObject.AiExtractElementInfo,
     generatePlaywrightTest: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTest,
     generateYamlTestStream: ()=>yaml_generator_js_namespaceObject.generateYamlTestStream,
     loadActionParam: ()=>external_common_js_namespaceObject.loadActionParam,
     resizeImageForUiTars: ()=>external_ui_tars_planning_js_namespaceObject.resizeImageForUiTars,
     systemPromptToLocateElement: ()=>llm_locator_js_namespaceObject.systemPromptToLocateElement,
+    vlmPlanning: ()=>external_ui_tars_planning_js_namespaceObject.vlmPlanning,
     RectSchema: ()=>external_common_js_namespaceObject.RectSchema,
     PointSchema: ()=>external_common_js_namespaceObject.PointSchema,
     adaptBboxToRect: ()=>external_common_js_namespaceObject.adaptBboxToRect,
     callAIWithObjectResponse: ()=>index_js_namespaceObject.callAIWithObjectResponse,
-    vlmPlanning: ()=>external_ui_tars_planning_js_namespaceObject.vlmPlanning,
     AiLocateElement: ()=>external_inspect_js_namespaceObject.AiLocateElement,
     getMidsceneLocationSchema: ()=>external_common_js_namespaceObject.getMidsceneLocationSchema,
     callAIWithStringResponse: ()=>index_js_namespaceObject.callAIWithStringResponse,
@@ -62,12 +61,10 @@ const external_inspect_js_namespaceObject = require("./inspect.js");
 const external_llm_planning_js_namespaceObject = require("./llm-planning.js");
 const external_common_js_namespaceObject = require("./common.js");
 const external_ui_tars_planning_js_namespaceObject = require("./ui-tars-planning.js");
-const external_conversation_history_js_namespaceObject = require("./conversation-history.js");
 exports.AIActionType = __webpack_exports__.AIActionType;
 exports.AiExtractElementInfo = __webpack_exports__.AiExtractElementInfo;
 exports.AiLocateElement = __webpack_exports__.AiLocateElement;
 exports.AiLocateSection = __webpack_exports__.AiLocateSection;
-exports.ConversationHistory = __webpack_exports__.ConversationHistory;
 exports.PointSchema = __webpack_exports__.PointSchema;
 exports.RectSchema = __webpack_exports__.RectSchema;
 exports.SizeSchema = __webpack_exports__.SizeSchema;
@@ -96,7 +93,6 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "AiExtractElementInfo",
     "AiLocateElement",
     "AiLocateSection",
-    "ConversationHistory",
     "PointSchema",
     "RectSchema",
     "SizeSchema",

package/dist/lib/ai-model/llm-planning.js

@@ -35,8 +35,8 @@ const util_js_namespaceObject = require("./prompt/util.js");
 const index_js_namespaceObject = require("./service-caller/index.js");
 const debug = (0, logger_namespaceObject.getDebug)('planning');
 async function plan(userInstruction, opts) {
-    var _opts_conversationHistory, _planFromAI_action;
-    const { context, modelConfig, conversationHistory } = opts;
+    var _planFromAI_action;
+    const { context, modelConfig } = opts;
     const { screenshotBase64, size } = context;
     const { modelName, vlMode } = modelConfig;
     const { description: pageDescription, elementById } = await (0, util_js_namespaceObject.describeUserPage)(context, {
@@ -46,9 +46,7 @@ async function plan(userInstruction, opts) {
         actionSpace: opts.actionSpace,
         vlMode: vlMode
     });
-    const historyLog = getLogFromConversationHistory(null == (_opts_conversationHistory = opts.conversationHistory) ? void 0 : _opts_conversationHistory.snapshot());
-    const logForContext = historyLog ?? opts.log;
-    const taskBackgroundContextText = (0, llm_planning_js_namespaceObject.generateTaskBackgroundContext)(userInstruction, logForContext, opts.actionContext);
+    const taskBackgroundContextText = (0, llm_planning_js_namespaceObject.generateTaskBackgroundContext)(userInstruction, opts.log, opts.actionContext);
     const userInstructionPrompt = await (0, llm_planning_js_namespaceObject.automationUserPrompt)(vlMode).format({
         pageDescription,
         taskBackgroundContext: taskBackgroundContextText
@@ -110,26 +108,8 @@ async function plan(userInstruction, opts) {
     });
     (0, utils_namespaceObject.assert)(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);
     if (0 === actions.length && returnValue.more_actions_needed_by_instruction && !returnValue.sleep) console.warn('No actions planned for the prompt, but model said more actions are needed:', userInstruction);
-    null == conversationHistory || conversationHistory.append({
-        role: 'assistant',
-        content: returnValue.log
-    });
     return returnValue;
 }
-function getLogFromConversationHistory(conversationHistory) {
-    if (!(null == conversationHistory ? void 0 : conversationHistory.length)) return;
-    const assistantMessages = conversationHistory.filter((message)=>'assistant' === message.role).map((message)=>{
-        if ('string' == typeof message.content) return message.content.trim();
-        if (Array.isArray(message.content)) return message.content.map((part)=>{
-            if ('text' === part.type && 'string' == typeof part.text) return part.text.trim();
-            if ('string' == typeof part.text) return part.text.trim();
-            return '';
-        }).filter(Boolean).join('\n');
-        return '';
-    }).map((content)=>content.trim()).filter(Boolean);
-    if (0 === assistantMessages.length) return;
-    return `- ${assistantMessages.join('\n- ')}`;
-}
 exports.plan = __webpack_exports__.plan;
 for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "plan"

package/dist/lib/ai-model/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport type { ConversationHistory } from './conversation-history';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    log?: string;\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory?: ConversationHistory;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { modelName, vlMode } = modelConfig;\n\n  const { description: pageDescription, elementById } = await describeUserPage(\n    context,\n    { vlMode },\n  );\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlMode,\n  });\n  const historyLog = getLogFromConversationHistory(\n    opts.conversationHistory?.snapshot(),\n  );\n  const logForContext = historyLog ?? opts.log;\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    logForContext,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(vlMode).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlMode === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlMode) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size, modelName);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const { content, usage } = await callAIWithObjectResponse<PlanningAIResponse>(\n    msgs,\n    AIActionType.PLAN,\n    modelConfig,\n  );\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: use zod.parse to parse the action.param, and then fill the bbox param.\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult) {\n        if (vlMode) {\n          action.param[field] = fillBboxParam(\n            locateResult,\n            size.width,\n            size.height,\n            vlMode,\n          );\n        } else {\n          const element = elementById(locateResult);\n          if (element) {\n            action.param[field].id = element.id;\n          }\n        }\n      }\n    });\n  });\n  // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n  assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  conversationHistory?.append({\n    role: 'assistant',\n    content: returnValue.log,\n  });\n\n  return returnValue;\n}\n\nfunction getLogFromConversationHistory(\n  conversationHistory?: ChatCompletionMessageParam[],\n): string | undefined {\n  if (!conversationHistory?.length) {\n    return undefined;\n  }\n\n  const assistantMessages = conversationHistory\n    .filter((message) => message.role === 'assistant')\n    .map((message) => {\n      if (typeof message.content === 'string') {\n        return message.content.trim();\n      }\n\n      if (Array.isArray(message.content)) {\n        return message.content\n          .map((part) => {\n            if (part.type === 'text' && typeof part.text === 'string') {\n              return part.text.trim();\n            }\n\n            if (typeof (part as any).text === 'string') {\n              return (part as any).text.trim();\n            }\n\n            return '';\n          })\n          .filter(Boolean)\n          .join('\\n');\n      }\n\n      return '';\n    })\n    .map((content) => content.trim())\n    .filter(Boolean);\n\n  if (assistantMessages.length === 0) {\n    return undefined;\n  }\n\n  return `- ${assistantMessages.join('\\n- ')}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","plan","userInstruction","opts","_opts_conversationHistory","_planFromAI_action","context","modelConfig","conversationHistory","screenshotBase64","size","modelName","vlMode","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","historyLog","getLogFromConversationHistory","logForContext","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","content","usage","callAIWithObjectResponse","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","element","console","assistantMessages","message","Array","part","Boolean"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACuBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAQC;QAgBCC,2BAuDCC;IArEH,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGL;IACtD,MAAM,EAAEM,gBAAgB,EAAEC,IAAI,EAAE,GAAGJ;IAEnC,MAAM,EAAEK,SAAS,EAAEC,MAAM,EAAE,GAAGL;IAE9B,MAAM,EAAE,aAAaM,eAAe,EAAEC,WAAW,EAAE,GAAG,MAAMC,AAAAA,IAAAA,wBAAAA,gBAAAA,AAAAA,EAC1DT,SACA;QAAEM;IAAO;IAEX,MAAMI,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAad,KAAK,WAAW;QAC7B,QAAQS;IACV;IACA,MAAMM,aAAaC,8BAA8B,QAC/Cf,CAAAA,4BAAAA,KAAK,mBAAmB,AAAD,IAAvBA,KAAAA,IAAAA,0BAA0B,QAAQ;IAEpC,MAAMgB,gBAAgBF,cAAcf,KAAK,GAAG;IAC5C,MAAMkB,4BAA4BC,AAAAA,IAAAA,gCAAAA,6BAAAA,AAAAA,EAChCpB,iBACAkB,eACAjB,KAAK,aAAa;IAEpB,MAAMoB,wBAAwB,MAAMC,AAAAA,IAAAA,gCAAAA,oBAAAA,AAAAA,EAAqBZ,QAAQ,MAAM,CAAC;QACtEC;QACA,uBAAuBQ;IACzB;IAEA,IAAII,eAAehB;IACnB,IAAIG,AAAW,cAAXA,QACFa,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BD;SAC5C,IAAI,CAACb,QACVa,eAAe,MAAME,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EACnBlB,kBACAH,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBsB,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBlB,MAAMC;IAEzB,MAAMkB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASb;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKS;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAM,EAAEO,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC/BH,MACAI,mCAAAA,YAAAA,CAAAA,IAAiB,EACjB1B;IAEF,MAAM2B,cAAcC,KAAK,SAAS,CAACL,SAASM,QAAW;IACvD,MAAMC,aAAaP;IAEnB,MAAMQ,UACHjC,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAACgC,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAH;QACA,UAAUS,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EACRF,SACAnC,KAAK,WAAW,EAChBkC,WAAW,KAAK;IAEpB;IAEAI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,YAAY;IAGnBC,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBzC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACuC,SAAWA,OAAO,IAAI,KAAKC;QAG9B5C,MAAM,+BAA+B6C;QACrC,MAAMC,eAAeD,sBACjBE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAEN7C,MAAM,gBAAgB8C;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,cACF,IAAIpC,QACF8B,OAAO,KAAK,CAACK,MAAM,GAAGE,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EACpBD,cACAtC,KAAK,KAAK,EACVA,KAAK,MAAM,EACXE;iBAEG;gBACL,MAAMsC,UAAUpC,YAAYkC;gBAC5B,IAAIE,SACFR,OAAO,KAAK,CAACK,MAAM,CAAC,EAAE,GAAGG,QAAQ,EAAE;YAEvC;QAEJ;IACF;IAEAT,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IAEvE,IACEC,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACAjD;IAIJM,QAAAA,uBAAAA,oBAAqB,MAAM,CAAC;QAC1B,MAAM;QACN,SAAS+B,YAAY,GAAG;IAC1B;IAEA,OAAOA;AACT;AAEA,SAASpB,8BACPX,mBAAkD;IAElD,IAAI,CAACA,CAAAA,QAAAA,sBAAAA,KAAAA,IAAAA,oBAAqB,MAAM,AAAD,GAC7B;IAGF,MAAM4C,oBAAoB5C,oBACvB,MAAM,CAAC,CAAC6C,UAAYA,AAAiB,gBAAjBA,QAAQ,IAAI,EAChC,GAAG,CAAC,CAACA;QACJ,IAAI,AAA2B,YAA3B,OAAOA,QAAQ,OAAO,EACxB,OAAOA,QAAQ,OAAO,CAAC,IAAI;QAG7B,IAAIC,MAAM,OAAO,CAACD,QAAQ,OAAO,GAC/B,OAAOA,QAAQ,OAAO,CACnB,GAAG,CAAC,CAACE;YACJ,IAAIA,AAAc,WAAdA,KAAK,IAAI,IAAe,AAAqB,YAArB,OAAOA,KAAK,IAAI,EAC1C,OAAOA,KAAK,IAAI,CAAC,IAAI;YAGvB,IAAI,AAA8B,YAA9B,OAAQA,KAAa,IAAI,EAC3B,OAAQA,KAAa,IAAI,CAAC,IAAI;YAGhC,OAAO;QACT,GACC,MAAM,CAACC,SACP,IAAI,CAAC;QAGV,OAAO;IACT,GACC,GAAG,CAAC,CAAC1B,UAAYA,QAAQ,IAAI,IAC7B,MAAM,CAAC0B;IAEV,IAAIJ,AAA6B,MAA7BA,kBAAkB,MAAM,EAC1B;IAGF,OAAO,CAAC,EAAE,EAAEA,kBAAkB,IAAI,CAAC,SAAS;AAC9C"}
+{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    log?: string;\n    actionContext?: string;\n    modelConfig: IModelConfig;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { modelName, vlMode } = modelConfig;\n\n  const { description: pageDescription, elementById } = await describeUserPage(\n    context,\n    { vlMode },\n  );\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlMode,\n  });\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    opts.log,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(vlMode).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlMode === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlMode) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size, modelName);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const { content, usage } = await callAIWithObjectResponse<PlanningAIResponse>(\n    msgs,\n    AIActionType.PLAN,\n    modelConfig,\n  );\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: use zod.parse to parse the action.param, and then fill the bbox param.\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult) {\n        if (vlMode) {\n          action.param[field] = fillBboxParam(\n            locateResult,\n            size.width,\n            size.height,\n            vlMode,\n          );\n        } else {\n          const element = elementById(locateResult);\n          if (element) {\n            action.param[field].id = element.id;\n          }\n        }\n      }\n    });\n  });\n  // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n  assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","plan","userInstruction","opts","_planFromAI_action","context","modelConfig","screenshotBase64","size","modelName","vlMode","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","content","usage","callAIWithObjectResponse","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","element","console"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACqBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAOC;QAmEEC;IAjEH,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGH;IACjC,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGH;IAEnC,MAAM,EAAEI,SAAS,EAAEC,MAAM,EAAE,GAAGJ;IAE9B,MAAM,EAAE,aAAaK,eAAe,EAAEC,WAAW,EAAE,GAAG,MAAMC,AAAAA,IAAAA,wBAAAA,gBAAAA,AAAAA,EAC1DR,SACA;QAAEK;IAAO;IAEX,MAAMI,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaZ,KAAK,WAAW;QAC7B,QAAQO;IACV;IACA,MAAMM,4BAA4BC,AAAAA,IAAAA,gCAAAA,6BAAAA,AAAAA,EAChCf,iBACAC,KAAK,GAAG,EACRA,KAAK,aAAa;IAEpB,MAAMe,wBAAwB,MAAMC,AAAAA,IAAAA,gCAAAA,oBAAAA,AAAAA,EAAqBT,QAAQ,MAAM,CAAC;QACtEC;QACA,uBAAuBK;IACzB;IAEA,IAAII,eAAeb;IACnB,IAAIG,AAAW,cAAXA,QACFU,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BD;SAC5C,IAAI,CAACV,QACVU,eAAe,MAAME,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EACnBf,kBACAF,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBkB,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBf,MAAMC;IAEzB,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASV;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKM;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAM,EAAEO,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC/BH,MACAI,mCAAAA,YAAAA,CAAAA,IAAiB,EACjBtB;IAEF,MAAMuB,cAAcC,KAAK,SAAS,CAACL,SAASM,QAAW;IACvD,MAAMC,aAAaP;IAEnB,MAAMQ,UACH7B,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAAC4B,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAH;QACA,UAAUS,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EACRF,SACA9B,KAAK,WAAW,EAChB6B,WAAW,KAAK;IAEpB;IAEAI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,YAAY;IAGnBC,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBpC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACkC,SAAWA,OAAO,IAAI,KAAKC;QAG9BvC,MAAM,+BAA+BwC;QACrC,MAAMC,eAAeD,sBACjBE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENxC,MAAM,gBAAgByC;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,cACF,IAAIjC,QACF2B,OAAO,KAAK,CAACK,MAAM,GAAGE,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EACpBD,cACAnC,KAAK,KAAK,EACVA,KAAK,MAAM,EACXE;iBAEG;gBACL,MAAMmC,UAAUjC,YAAY+B;gBAC5B,IAAIE,SACFR,OAAO,KAAK,CAACK,MAAM,CAAC,EAAE,GAAGG,QAAQ,EAAE;YAEvC;QAEJ;IACF;IAEAT,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IAEvE,IACEC,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACA5C;IAIJ,OAAOgC;AACT"}

package/dist/lib/ai-model/ui-tars-planning.js

@@ -44,31 +44,18 @@ const pointToBbox = (point, width, height)=>[
         Math.round(Math.min(point.x + bboxSize / 2, width)),
         Math.round(Math.min(point.y + bboxSize / 2, height))
     ];
-async function vlmPlanning(userInstruction, options) {
-    const { conversationHistory, context, modelConfig } = options;
+async function vlmPlanning(options) {
+    const { conversationHistory, userInstruction, size, modelConfig } = options;
     const { uiTarsModelVersion } = modelConfig;
     const systemPrompt = (0, ui_tars_planning_js_namespaceObject.getUiTarsPlanningPrompt)() + userInstruction;
-    const imagePayload = await resizeImageForUiTars(context.screenshotBase64, context.size, uiTarsModelVersion);
-    conversationHistory.append({
-        role: 'user',
-        content: [
-            {
-                type: 'image_url',
-                image_url: {
-                    url: imagePayload
-                }
-            }
-        ]
-    });
     const res = await (0, index_js_namespaceObject.callAIWithStringResponse)([
         {
             role: 'user',
             content: systemPrompt
         },
-        ...conversationHistory.snapshot()
+        ...conversationHistory
     ], external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig);
     const convertedText = convertBboxToCoordinates(res.content);
-    const { size } = context;
     const { parsed } = (0, action_parser_namespaceObject.actionParser)({
         prediction: convertedText,
         factor: [
@@ -83,7 +70,6 @@ async function vlmPlanning(userInstruction, options) {
     });
     debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
     const transformActions = [];
-    let shouldContinue = true;
     parsed.forEach((action)=>{
         if ('click' === action.action_type) {
             (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
@@ -161,7 +147,6 @@ async function vlmPlanning(userInstruction, options) {
             },
             thought: action.thought || ''
         });
-        else if ('finished' === action.action_type) shouldContinue = false;
     });
     if (0 === transformActions.length) throw new Error(`No actions found, response: ${res.content}`, {
         cause: {
@@ -170,17 +155,12 @@ async function vlmPlanning(userInstruction, options) {
         }
     });
     debug('transformActions', JSON.stringify(transformActions, null, 2));
-    const log = (0, ui_tars_planning_js_namespaceObject.getSummary)(res.content);
-    conversationHistory.append({
-        role: 'assistant',
-        content: log
-    });
     return {
         actions: transformActions,
-        log,
+        actionsFromModel: parsed,
+        action_summary: (0, ui_tars_planning_js_namespaceObject.getSummary)(res.content),
         usage: res.usage,
-        rawResponse: JSON.stringify(res.content, void 0, 2),
-        more_actions_needed_by_instruction: shouldContinue
+        rawResponse: JSON.stringify(res.content, void 0, 2)
     };
 }
 function convertBboxToCoordinates(text) {