@midscene/core 0.27.4 → 0.27.5-beta-20250827031507.0

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import {\n  type ChatCompletionMessageParam,\n  elementByPositionWithElementInfo,\n  findAllMidsceneLocatorField,\n  resizeImageForUiTars,\n  vlmPlanning,\n} from '@/ai-model';\nimport type { AbstractDevice } from '@/device';\nimport {\n  type AIUsageInfo,\n  type BaseElement,\n  type DetailedLocateParam,\n  type DumpSubscriber,\n  type ExecutionRecorderItem,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskHitBy,\n  type ExecutionTaskInsightLocateApply,\n  type ExecutionTaskInsightQueryApply,\n  type ExecutionTaskPlanning,\n  type ExecutionTaskPlanningApply,\n  type ExecutionTaskProgressOptions,\n  Executor,\n  type ExecutorContext,\n  type Insight,\n  type InsightDump,\n  type InsightExtractOption,\n  type InsightExtractParam,\n  type LocateResultElement,\n  type MidsceneYamlFlowItem,\n  type PageType,\n  type PlanningAIResponse,\n  type PlanningAction,\n  type PlanningActionParamError,\n  type PlanningActionParamSleep,\n  type PlanningActionParamWaitFor,\n  type PlanningLocateParam,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type UIContext,\n  plan,\n} from '@/index';\nimport { sleep } from '@/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelPreferences,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { taskTitleStr } from './ui-utils';\nimport {\n  matchElementFromCache,\n  matchElementFromPlan,\n  parsePrompt,\n} from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  executor: Executor;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\nexport class PageTaskExecutor {\n  page: AbstractDevice;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  conversationHistory: ChatCompletionMessageParam[] = [];\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    page: AbstractDevice,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n    },\n  ) {\n    this.page = page;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n  }\n\n  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {\n    const base64 = await this.page.screenshotBase64();\n    const item: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot: base64,\n      timing,\n    };\n    return item;\n  }\n\n  private async getElementXpath(\n    pageContext: UIContext<BaseElement>,\n    element: LocateResultElement,\n  ): Promise<string[] | undefined> {\n    if (!(this.page as any).getXpathsByPoint) {\n      debug('getXpathsByPoint is not supported for this page');\n      return undefined;\n    }\n\n    let elementId = element?.id;\n    if (element?.isOrderSensitive !== undefined) {\n      const xpaths = await (this.page as any).getXpathsByPoint(\n        {\n          left: element.center[0],\n          top: element.center[1],\n        },\n        element?.isOrderSensitive,\n      );\n\n      return xpaths;\n    }\n\n    // find the nearest xpath for the element\n    if (element?.attributes?.nodeType === NodeType.POSITION) {\n      await this.insight.contextRetrieverFn('locate');\n      const info = elementByPositionWithElementInfo(\n        pageContext.tree,\n        {\n          x: element.center[0],\n          y: element.center[1],\n        },\n        {\n          requireStrictDistance: false,\n          filterPositionElements: true,\n        },\n      );\n      if (info?.id) {\n        elementId = info.id;\n      } else {\n        debug(\n          'no element id found for position node, will not update cache',\n          element,\n        );\n      }\n    }\n\n    if (!elementId) {\n      return undefined;\n    }\n    try {\n      const result = await (this.page as any).getXpathsById(elementId);\n      return result;\n    } catch (error) {\n      debug('getXpathsById error: ', error);\n    }\n  }\n\n  private prependExecutorWithScreenshot(\n    taskApply: ExecutionTaskApply,\n    appendAfterExecution = false,\n  ): ExecutionTaskApply {\n    const taskWithScreenshot: ExecutionTaskApply = {\n      ...taskApply,\n      executor: async (param, context, ...args) => {\n        const recorder: ExecutionRecorderItem[] = [];\n        const { task } = context;\n        // set the recorder before executor in case of error\n        task.recorder = recorder;\n        const shot = await this.recordScreenshot(`before ${task.type}`);\n        recorder.push(shot);\n        const result = await taskApply.executor(param, context, ...args);\n        if (taskApply.type === 'Action') {\n          await Promise.all([\n            (async () => {\n              await sleep(100);\n              if (this.page.beforeAction) {\n                debug('will call \"beforeAction\" for page');\n                await this.page.beforeAction();\n              }\n            })(),\n            sleep(200),\n          ]);\n        }\n        if (appendAfterExecution) {\n          const shot2 = await this.recordScreenshot('after Action');\n          recorder.push(shot2);\n        }\n        return result;\n      },\n    };\n    return taskWithScreenshot;\n  }\n\n  public async convertPlanToExecutable(plans: PlanningAction[]) {\n    const tasks: ExecutionTaskApply[] = [];\n\n    const taskForLocatePlan = (\n      plan: PlanningAction<PlanningLocateParam>,\n      detailedLocateParam: DetailedLocateParam | string,\n      onResult?: (result: LocateResultElement) => void,\n    ): ExecutionTaskInsightLocateApply => {\n      if (typeof detailedLocateParam === 'string') {\n        detailedLocateParam = {\n          prompt: detailedLocateParam,\n        };\n      }\n      const taskFind: ExecutionTaskInsightLocateApply = {\n        type: 'Insight',\n        subType: 'Locate',\n        param: detailedLocateParam,\n        thought: plan.thought,\n        executor: async (param, taskContext) => {\n          const { task } = taskContext;\n          assert(\n            param?.prompt || param?.id || param?.bbox,\n            `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n              param,\n            )}`,\n          );\n          let insightDump: InsightDump | undefined;\n          let usage: AIUsageInfo | undefined;\n          const dumpCollector: DumpSubscriber = (dump) => {\n            insightDump = dump;\n            usage = dump?.taskInfo?.usage;\n\n            task.log = {\n              dump: insightDump,\n            };\n\n            task.usage = usage;\n          };\n          this.insight.onceDumpUpdatedFn = dumpCollector;\n          const shotTime = Date.now();\n\n          // Get context through contextRetrieverFn which handles frozen context\n          const pageContext = await this.insight.contextRetrieverFn('locate');\n          task.pageContext = pageContext;\n\n          const recordItem: ExecutionRecorderItem = {\n            type: 'screenshot',\n            ts: shotTime,\n            screenshot: pageContext.screenshotBase64,\n            timing: 'before Insight',\n          };\n          task.recorder = [recordItem];\n\n          // try matching xpath\n          const elementFromXpath =\n            param.xpath && (this.page as any).getElementInfoByXpath\n              ? await (this.page as any).getElementInfoByXpath(param.xpath)\n              : undefined;\n          const userExpectedPathHitFlag = !!elementFromXpath;\n\n          // try matching cache\n          const cachePrompt = param.prompt;\n          const locateCacheRecord =\n            this.taskCache?.matchLocateCache(cachePrompt);\n          const xpaths = locateCacheRecord?.cacheContent?.xpaths;\n          const elementFromCache = userExpectedPathHitFlag\n            ? null\n            : await matchElementFromCache(\n                this,\n                xpaths,\n                cachePrompt,\n                param.cacheable,\n              );\n          const cacheHitFlag = !!elementFromCache;\n\n          // try matching plan\n          const elementFromPlan =\n            !userExpectedPathHitFlag && !cacheHitFlag\n              ? matchElementFromPlan(param, pageContext.tree)\n              : undefined;\n          const planHitFlag = !!elementFromPlan;\n\n          // try ai locate\n          const elementFromAiLocate =\n            !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag\n              ? (\n                  await this.insight.locate(param, {\n                    // fallback to ai locate\n                    context: pageContext,\n                  })\n                ).element\n              : undefined;\n          const aiLocateHitFlag = !!elementFromAiLocate;\n\n          const element =\n            elementFromXpath || // highest priority\n            elementFromCache || // second priority\n            elementFromPlan || // third priority\n            elementFromAiLocate;\n\n          // update cache\n          let currentXpaths: string[] | undefined;\n          if (\n            element &&\n            this.taskCache &&\n            !cacheHitFlag &&\n            param?.cacheable !== false\n          ) {\n            const elementXpaths = await this.getElementXpath(\n              pageContext,\n              element,\n            );\n            if (elementXpaths?.length) {\n              currentXpaths = elementXpaths;\n              this.taskCache.updateOrAppendCacheRecord(\n                {\n                  type: 'locate',\n                  prompt: cachePrompt,\n                  xpaths: elementXpaths,\n                },\n                locateCacheRecord,\n              );\n            } else {\n              debug(\n                'no xpaths found, will not update cache',\n                cachePrompt,\n                elementXpaths,\n              );\n            }\n          }\n          if (!element) {\n            throw new Error(`Element not found: ${param.prompt}`);\n          }\n\n          let hitBy: ExecutionTaskHitBy | undefined;\n\n          if (userExpectedPathHitFlag) {\n            hitBy = {\n              from: 'User expected path',\n              context: {\n                xpath: param.xpath,\n              },\n            };\n          } else if (cacheHitFlag) {\n            hitBy = {\n              from: 'Cache',\n              context: {\n                xpathsFromCache: xpaths,\n                xpathsToSave: currentXpaths,\n              },\n            };\n          } else if (planHitFlag) {\n            hitBy = {\n              from: 'Planning',\n              context: {\n                id: elementFromPlan?.id,\n                bbox: elementFromPlan?.bbox,\n              },\n            };\n          } else if (aiLocateHitFlag) {\n            hitBy = {\n              from: 'AI model',\n              context: {\n                prompt: param.prompt,\n              },\n            };\n          }\n\n          onResult?.(element);\n\n          return {\n            output: {\n              element,\n            },\n            pageContext,\n            hitBy,\n          };\n        },\n      };\n      return taskFind;\n    };\n\n    for (const plan of plans) {\n      if (plan.type === 'Locate') {\n        if (\n          !plan.locate ||\n          plan.locate === null ||\n          plan.locate?.id === null ||\n          plan.locate?.id === 'null'\n        ) {\n          debug('Locate action with id is null, will be ignored', plan);\n          continue;\n        }\n        const taskLocate = taskForLocatePlan(plan, plan.locate);\n\n        tasks.push(taskLocate);\n      } else if (plan.type === 'Error') {\n        const taskActionError: ExecutionTaskActionApply<PlanningActionParamError> =\n          {\n            type: 'Action',\n            subType: 'Error',\n            param: plan.param,\n            thought: plan.thought || plan.param?.thought,\n            locate: plan.locate,\n            executor: async () => {\n              throw new Error(\n                plan?.thought || plan.param?.thought || 'error without thought',\n              );\n            },\n          };\n        tasks.push(taskActionError);\n      } else if (plan.type === 'Finished') {\n        const taskActionFinished: ExecutionTaskActionApply<null> = {\n          type: 'Action',\n          subType: 'Finished',\n          param: null,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async (param) => {},\n        };\n        tasks.push(taskActionFinished);\n      } else if (plan.type === 'Sleep') {\n        const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =\n          {\n            type: 'Action',\n            subType: 'Sleep',\n            param: plan.param,\n            thought: plan.thought,\n            locate: plan.locate,\n            executor: async (taskParam) => {\n              await sleep(taskParam?.timeMs || 3000);\n            },\n          };\n        tasks.push(taskActionSleep);\n      } else {\n        // action in action space\n        const planType = plan.type;\n        const actionSpace = await this.page.actionSpace();\n        const action = actionSpace.find((action) => action.name === planType);\n        const param = plan.param;\n\n        if (!action) {\n          throw new Error(`Action type '${planType}' not found`);\n        }\n\n        // find all params that needs location\n        const locateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema)\n          : [];\n\n        const requiredLocateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema, true)\n          : [];\n\n        locateFields.forEach((field) => {\n          if (param[field]) {\n            const locatePlan = locatePlanForLocate(param[field]);\n            debug(\n              'will prepend locate param for field',\n              `action.type=${planType}`,\n              `param=${JSON.stringify(param[field])}`,\n              `locatePlan=${JSON.stringify(locatePlan)}`,\n            );\n            const locateTask = taskForLocatePlan(\n              locatePlan,\n              param[field],\n              (result) => {\n                param[field] = result;\n              },\n            );\n            tasks.push(locateTask);\n          } else {\n            assert(\n              !requiredLocateFields.includes(field),\n              `Required locate field '${field}' is not provided for action ${planType}`,\n            );\n            debug(`field '${field}' is not provided for action ${planType}`);\n          }\n        });\n\n        const task: ExecutionTaskApply<\n          'Action',\n          any,\n          { success: boolean; action: string; param: any },\n          void\n        > = {\n          type: 'Action',\n          subType: planType,\n          thought: plan.thought,\n          param: plan.param,\n          executor: async (param, context) => {\n            debug(\n              'executing action',\n              planType,\n              param,\n              `context.element.center: ${context.element?.center}`,\n            );\n\n            // Get page context for actionSpace operations to ensure size info is available\n            const pageContext = await this.insight.contextRetrieverFn('locate');\n            context.task.pageContext = pageContext;\n\n            requiredLocateFields.forEach((field) => {\n              assert(\n                param[field],\n                `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n              );\n            });\n\n            const actionFn = action.call.bind(this.page);\n            await actionFn(param, context);\n            // Return a proper result for report generation\n            return {\n              output: {\n                success: true,\n                action: planType,\n                param: param,\n              },\n            };\n          },\n        };\n        tasks.push(task);\n      }\n    }\n\n    const wrappedTasks = tasks.map(\n      (task: ExecutionTaskApply, index: number) => {\n        if (task.type === 'Action') {\n          return this.prependExecutorWithScreenshot(\n            task,\n            index === tasks.length - 1,\n          );\n        }\n        return task;\n      },\n    );\n\n    return {\n      tasks: wrappedTasks,\n    };\n  }\n\n  private async setupPlanningContext(executorContext: ExecutorContext) {\n    const shotTime = Date.now();\n    const pageContext = await this.insight.contextRetrieverFn('locate');\n    const recordItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: shotTime,\n      screenshot: pageContext.screenshotBase64,\n      timing: 'before Planning',\n    };\n\n    executorContext.task.recorder = [recordItem];\n    (executorContext.task as ExecutionTaskPlanning).pageContext = pageContext;\n\n    return {\n      pageContext,\n    };\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        await this.setupPlanningContext(executorContext);\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n\n    await taskExecutor.append(task);\n    await taskExecutor.flush();\n\n    return {\n      executor: taskExecutor,\n    };\n  }\n\n  private planningTaskFromPrompt(\n    userInstruction: string,\n    log?: string,\n    actionContext?: string,\n  ) {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n        log,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { pageContext } =\n          await this.setupPlanningContext(executorContext);\n\n        assert(\n          this.page.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.page.actionSpace();\n        debug(\n          'actionSpace for page is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.page.pageType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await plan(param.userInstruction, {\n          context: pageContext,\n          log: param.log,\n          actionContext,\n          pageType: this.page.pageType as PageType,\n          actionSpace,\n        });\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        // TODO: check locate result\n        // let bboxCollected = false;\n        // (actions || []).reduce<PlanningAction[]>(\n        //   (acc, planningAction) => {\n        //     // TODO: magic field \"locate\" is used to indicate the action requires a locate\n        //     if (planningAction.locate) {\n        //       // we only collect bbox once, let qwen re-locate in the following steps\n        //       if (bboxCollected && planningAction.locate.bbox) {\n        //         // biome-ignore lint/performance/noDelete: <explanation>\n        //         delete planningAction.locate.bbox;\n        //       }\n\n        //       if (planningAction.locate.bbox) {\n        //         bboxCollected = true;\n        //       }\n\n        //       acc.push({\n        //         type: 'Locate',\n        //         locate: planningAction.locate,\n        //         param: null,\n        //         // thought is prompt created by ai, always a string\n        //         thought: planningAction.locate.prompt as string,\n        //       });\n        //     }\n        //     acc.push(planningAction);\n        //     return acc;\n        //   },\n        //   [],\n        // );\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push({\n              type: 'Sleep',\n              param: {\n                timeMs: timeRemaining,\n              },\n              locate: null,\n            } as PlanningAction<PlanningActionParamSleep>);\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          pageContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  private planningTaskToGoal(\n    userInstruction: string,\n    modelPreferences: IModelPreferences,\n  ) {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { pageContext } =\n          await this.setupPlanningContext(executorContext);\n\n        const imagePayload = await resizeImageForUiTars(\n          pageContext.screenshotBase64,\n          pageContext.size,\n          modelPreferences,\n        );\n\n        this.appendConversationHistory({\n          role: 'user',\n          content: [\n            {\n              type: 'image_url',\n              image_url: {\n                url: imagePayload,\n              },\n            },\n          ],\n        });\n        const planResult: {\n          actions: PlanningAction<any>[];\n          action_summary: string;\n          usage?: AIUsageInfo;\n          yamlFlow?: MidsceneYamlFlowItem[];\n          rawResponse?: string;\n        } = await vlmPlanning({\n          userInstruction: param.userInstruction,\n          conversationHistory: this.conversationHistory,\n          size: pageContext.size,\n          modelPreferences,\n        });\n\n        const { actions, action_summary, usage } = planResult;\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse: planResult.rawResponse,\n        };\n        executorContext.task.usage = usage;\n        this.appendConversationHistory({\n          role: 'assistant',\n          content: action_summary,\n        });\n        return {\n          output: {\n            actions,\n            thought: actions[0]?.thought,\n            actionType: actions[0].type,\n            more_actions_needed_by_instruction: true,\n            log: '',\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n  ): Promise<ExecutionResult> {\n    const taskExecutor = new Executor(title, {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { tasks } = await this.convertPlanToExecutable(plans);\n    await taskExecutor.append(tasks);\n    const result = await taskExecutor.flush();\n    const { output } = result!;\n    return {\n      output,\n      executor: taskExecutor,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    actionContext?: string,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    let planningTask: ExecutionTaskPlanningApply | null =\n      this.planningTaskFromPrompt(userPrompt, undefined, actionContext);\n    let replanCount = 0;\n    const logList: string[] = [];\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) || defaultReplanningCycleLimit;\n    while (planningTask) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg =\n          'Replanning too many times, please split the task into multiple steps';\n\n        return this.appendErrorPlan(taskExecutor, errorMsg);\n      }\n\n      // plan\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      const planResult: PlanningAIResponse = result?.output;\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: planResult,\n          executor: taskExecutor,\n        };\n      }\n\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n\n      await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (planResult?.log) {\n        logList.push(planResult.log);\n      }\n\n      if (!planResult.more_actions_needed_by_instruction) {\n        planningTask = null;\n        break;\n      }\n      planningTask = this.planningTaskFromPrompt(\n        userPrompt,\n        logList.length > 0 ? `- ${logList.join('\\n- ')}` : undefined,\n        actionContext,\n      );\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  async actionToGoal(userPrompt: string): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    this.conversationHistory = [];\n    const isCompleted = false;\n    let currentActionCount = 0;\n    const maxActionNumber = 40;\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    while (!isCompleted && currentActionCount < maxActionNumber) {\n      currentActionCount++;\n      debug(\n        'actionToGoal, currentActionCount:',\n        currentActionCount,\n        'userPrompt:',\n        userPrompt,\n      );\n      const planningTask: ExecutionTaskPlanningApply = this.planningTaskToGoal(\n        userPrompt,\n        {\n          intent: 'planning',\n        },\n      );\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function actionToGoal',\n        );\n      }\n      const { output } = result;\n      const plans = output.actions;\n      yamlFlow.push(...(output.yamlFlow || []));\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n\n      await taskExecutor.flush();\n\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      if (plans[0].type === 'Finished') {\n        break;\n      }\n    }\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        // TODO: display image thumbnail in report\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let insightDump: InsightDump | undefined;\n        const dumpCollector: DumpSubscriber = (dump) => {\n          insightDump = dump;\n        };\n        this.insight.onceDumpUpdatedFn = dumpCollector;\n\n        // Get page context for query operations\n        const shotTime = Date.now();\n        const pageContext = await this.insight.contextRetrieverFn('extract');\n        task.pageContext = pageContext;\n\n        const recordItem: ExecutionRecorderItem = {\n          type: 'screenshot',\n          ts: shotTime,\n          screenshot: pageContext.screenshotBase64,\n          timing: 'before Extract',\n        };\n        task.recorder = [recordItem];\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        if (ifTypeRestricted) {\n          const returnType = type === 'Assert' ? 'Boolean' : type;\n          demandInput = {\n            result: `${returnType}, ${demand}`,\n          };\n        }\n\n        const { data, usage, thought } = await this.insight.extract<any>(\n          demandInput,\n          opt,\n          multimodalPrompt,\n        );\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          assert(data?.result !== undefined, 'No result in query data');\n          outputResult = (data as any).result;\n        }\n\n        return {\n          output: outputResult,\n          log: { dump: insightDump, isWaitForAssert: opt?.isWaitForAssert },\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const taskExecutor = new Executor(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        onTaskStart: this.onTaskStartCallback,\n      },\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      opt,\n      multimodalPrompt,\n    );\n\n    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n    const result = await taskExecutor.flush();\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      executor: taskExecutor,\n    };\n  }\n\n  async assert(\n    assertion: TUserPrompt,\n    opt?: InsightExtractOption,\n  ): Promise<ExecutionResult<boolean>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n    return await this.createTypeQueryExecution<boolean>(\n      'Assert',\n      textPrompt,\n      opt,\n      multimodalPrompt,\n    );\n  }\n\n  /**\n   * Append a message to the conversation history\n   * For user messages with images:\n   * - Keep max 4 user image messages in history\n   * - Remove oldest user image message when limit reached\n   * For assistant messages:\n   * - Simply append to history\n   * @param conversationHistory Message to append\n   */\n  private appendConversationHistory(\n    conversationHistory: ChatCompletionMessageParam,\n  ) {\n    if (conversationHistory.role === 'user') {\n      // Get all existing user messages with images\n      const userImgItems = this.conversationHistory.filter(\n        (item) => item.role === 'user',\n      );\n\n      // If we already have 4 user image messages\n      if (userImgItems.length >= 4 && conversationHistory.role === 'user') {\n        // Remove first user image message when we already have 4, before adding new one\n        const firstUserImgIndex = this.conversationHistory.findIndex(\n          (item) => item.role === 'user',\n        );\n        if (firstUserImgIndex >= 0) {\n          this.conversationHistory.splice(firstUserImgIndex, 1);\n        }\n      }\n    }\n    // For non-user messages, simply append to history\n    this.conversationHistory.push(conversationHistory);\n  }\n\n  private async appendErrorPlan(taskExecutor: Executor, errorMsg: string) {\n    const errorPlan: PlanningAction<PlanningActionParamError> = {\n      type: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      locate: null,\n    };\n    const { tasks } = await this.convertPlanToExecutable([errorPlan]);\n    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));\n    await taskExecutor.flush();\n\n    return {\n      output: undefined,\n      executor: taskExecutor,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'Assert',\n        textPrompt,\n        {\n          isWaitForAssert: true,\n          returnThought: true,\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n      const result = (await taskExecutor.flush()) as {\n        output: boolean;\n        thought?: string;\n      };\n\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function waitFor',\n        );\n      }\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepPlan: PlanningAction<PlanningActionParamSleep> = {\n          type: 'Sleep',\n          param: {\n            timeMs: timeRemaining,\n          },\n          locate: null,\n        };\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable([\n          sleepPlan,\n        ]);\n        await taskExecutor.append(\n          this.prependExecutorWithScreenshot(sleepTasks[0]),\n        );\n        await taskExecutor.flush();\n      }\n    }\n\n    return this.appendErrorPlan(\n      taskExecutor,\n      `waitFor timeout: ${errorThought}`,\n    );\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","locatePlanForLocate","param","locate","locatePlan","PageTaskExecutor","timing","base64","item","Date","pageContext","element","_element_attributes","elementId","undefined","xpaths","NodeType","info","elementByPositionWithElementInfo","result","error","taskApply","appendAfterExecution","taskWithScreenshot","context","args","recorder","task","shot","Promise","sleep","shot2","plans","tasks","taskForLocatePlan","plan","detailedLocateParam","onResult","taskFind","taskContext","_this_taskCache","_locateCacheRecord_cacheContent","assert","JSON","insightDump","usage","dumpCollector","dump","_dump_taskInfo","shotTime","recordItem","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","aiLocateHitFlag","currentXpaths","elementXpaths","Error","hitBy","_plan_locate","_plan_locate1","taskLocate","_plan_param","taskActionError","taskActionFinished","taskActionSleep","taskParam","planType","actionSpace","action","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","locateTask","_context_element","actionFn","wrappedTasks","index","executorContext","userInstruction","yamlString","taskExecutor","Executor","taskTitleStr","log","actionContext","startTime","Array","console","planResult","actions","more_actions_needed_by_instruction","rawResponse","finalActions","timeNow","timeRemaining","modelPreferences","_actions_","imagePayload","resizeImageForUiTars","vlmPlanning","action_summary","title","output","userPrompt","planningTask","replanCount","logList","yamlFlow","replanningCycleLimit","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","errorMsg","executables","isCompleted","currentActionCount","maxActionNumber","type","demand","opt","multimodalPrompt","queryTask","ifTypeRestricted","demandInput","returnType","data","thought","outputResult","assertion","textPrompt","parsePrompt","conversationHistory","userImgItems","firstUserImgIndex","errorPlan","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepPlan","sleepTasks","page","insight","opts"],"mappings":";;;;;;;;;;;;;;;;;;;AAiEA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AAE7B,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAEO,MAAMC;IAyBX,MAAc,iBAAiBC,MAAuC,EAAE;QACtE,MAAMC,SAAS,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAC/C,MAAMC,OAA8B;YAClC,MAAM;YACN,IAAIC,KAAK,GAAG;YACZ,YAAYF;YACZD;QACF;QACA,OAAOE;IACT;IAEA,MAAc,gBACZE,WAAmC,EACnCC,OAA4B,EACG;YAoB3BC;QAnBJ,IAAI,CAAE,IAAI,CAAC,IAAI,CAAS,gBAAgB,EAAE,YACxCd,MAAM;QAIR,IAAIe,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE;QAC3B,IAAIA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB,AAAD,MAAMG,QAAW;YAC3C,MAAMC,SAAS,MAAO,IAAI,CAAC,IAAI,CAAS,gBAAgB,CACtD;gBACE,MAAMJ,QAAQ,MAAM,CAAC,EAAE;gBACvB,KAAKA,QAAQ,MAAM,CAAC,EAAE;YACxB,GACAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB;YAG3B,OAAOI;QACT;QAGA,IAAIH,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,sBAAAA,QAAS,UAAU,AAAD,IAAlBA,KAAAA,IAAAA,oBAAqB,QAAQ,AAAD,MAAMI,SAAS,QAAQ,EAAE;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;YACtC,MAAMC,OAAOC,iCACXR,YAAY,IAAI,EAChB;gBACE,GAAGC,QAAQ,MAAM,CAAC,EAAE;gBACpB,GAAGA,QAAQ,MAAM,CAAC,EAAE;YACtB,GACA;gBACE,uBAAuB;gBACvB,wBAAwB;YAC1B;YAEF,IAAIM,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE,EACVJ,YAAYI,KAAK,EAAE;iBAEnBnB,MACE,gEACAa;QAGN;QAEA,IAAI,CAACE,WACH;QAEF,IAAI;YACF,MAAMM,SAAS,MAAO,IAAI,CAAC,IAAI,CAAS,aAAa,CAACN;YACtD,OAAOM;QACT,EAAE,OAAOC,OAAO;YACdtB,MAAM,yBAAyBsB;QACjC;IACF;IAEQ,8BACNC,SAA6B,EAC7BC,uBAAuB,KAAK,EACR;QACpB,MAAMC,qBAAyC;YAC7C,GAAGF,SAAS;YACZ,UAAU,OAAOnB,OAAOsB,SAAS,GAAGC;gBAClC,MAAMC,WAAoC,EAAE;gBAC5C,MAAM,EAAEC,IAAI,EAAE,GAAGH;gBAEjBG,KAAK,QAAQ,GAAGD;gBAChB,MAAME,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAED,KAAK,IAAI,EAAE;gBAC9DD,SAAS,IAAI,CAACE;gBACd,MAAMT,SAAS,MAAME,UAAU,QAAQ,CAACnB,OAAOsB,YAAYC;gBAC3D,IAAIJ,AAAmB,aAAnBA,UAAU,IAAI,EAChB,MAAMQ,QAAQ,GAAG,CAAC;oBACf;wBACC,MAAMC,yBAAM;wBACZ,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;4BAC1BhC,MAAM;4BACN,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY;wBAC9B;oBACF;oBACAgC,yBAAM;iBACP;gBAEH,IAAIR,sBAAsB;oBACxB,MAAMS,QAAQ,MAAM,IAAI,CAAC,gBAAgB,CAAC;oBAC1CL,SAAS,IAAI,CAACK;gBAChB;gBACA,OAAOZ;YACT;QACF;QACA,OAAOI;IACT;IAEA,MAAa,wBAAwBS,KAAuB,EAAE;QAC5D,MAAMC,QAA8B,EAAE;QAEtC,MAAMC,oBAAoB,CACxBC,MACAC,qBACAC;YAEA,IAAI,AAA+B,YAA/B,OAAOD,qBACTA,sBAAsB;gBACpB,QAAQA;YACV;YAEF,MAAME,WAA4C;gBAChD,MAAM;gBACN,SAAS;gBACT,OAAOF;gBACP,SAASD,KAAK,OAAO;gBACrB,UAAU,OAAOjC,OAAOqC;wBA6CpBC,iBACaC;oBA7Cf,MAAM,EAAEd,IAAI,EAAE,GAAGY;oBACjBG,OACExC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAEyC,KAAK,SAAS,CACpEzC,QACC;oBAEL,IAAI0C;oBACJ,IAAIC;oBACJ,MAAMC,gBAAgC,CAACC;4BAE7BC;wBADRJ,cAAcG;wBACdF,QAAQG,QAAAA,OAAAA,KAAAA,IAAAA,QAAAA,CAAAA,iBAAAA,KAAM,QAAQ,AAAD,IAAbA,KAAAA,IAAAA,eAAgB,KAAK;wBAE7BrB,KAAK,GAAG,GAAG;4BACT,MAAMiB;wBACR;wBAEAjB,KAAK,KAAK,GAAGkB;oBACf;oBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGC;oBACjC,MAAMG,WAAWxC,KAAK,GAAG;oBAGzB,MAAMC,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBAC1DiB,KAAK,WAAW,GAAGjB;oBAEnB,MAAMwC,aAAoC;wBACxC,MAAM;wBACN,IAAID;wBACJ,YAAYvC,YAAY,gBAAgB;wBACxC,QAAQ;oBACV;oBACAiB,KAAK,QAAQ,GAAG;wBAACuB;qBAAW;oBAG5B,MAAMC,mBACJjD,MAAM,KAAK,IAAK,IAAI,CAAC,IAAI,CAAS,qBAAqB,GACnD,MAAO,IAAI,CAAC,IAAI,CAAS,qBAAqB,CAACA,MAAM,KAAK,IAC1DY;oBACN,MAAMsC,0BAA0B,CAAC,CAACD;oBAGlC,MAAME,cAAcnD,MAAM,MAAM;oBAChC,MAAMoD,oBAAAA,QACJd,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACa;oBACnC,MAAMtC,SAAS0B,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,MAAM;oBACtD,MAAMc,mBAAmBH,0BACrB,OACA,MAAMI,sBACJ,IAAI,EACJzC,QACAsC,aACAnD,MAAM,SAAS;oBAErB,MAAMuD,eAAe,CAAC,CAACF;oBAGvB,MAAMG,kBACJ,AAACN,2BAA4BK,eAEzB3C,SADA6C,qBAAqBzD,OAAOQ,YAAY,IAAI;oBAElD,MAAMkD,cAAc,CAAC,CAACF;oBAGtB,MAAMG,sBACJ,AAACT,2BAA4BK,gBAAiBG,cAO1C9C,SALE,OAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAACZ,OAAO;wBAE/B,SAASQ;oBACX,EAAC,EACD,OAAO;oBAEf,MAAMoD,kBAAkB,CAAC,CAACD;oBAE1B,MAAMlD,UACJwC,oBACAI,oBACAG,mBACAG;oBAGF,IAAIE;oBACJ,IACEpD,WACA,IAAI,CAAC,SAAS,IACd,CAAC8C,gBACDvD,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OACrB;wBACA,MAAM8D,gBAAgB,MAAM,IAAI,CAAC,eAAe,CAC9CtD,aACAC;wBAEF,IAAIqD,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,EAAE;4BACzBD,gBAAgBC;4BAChB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gCACE,MAAM;gCACN,QAAQX;gCACR,QAAQW;4BACV,GACAV;wBAEJ,OACExD,MACE,0CACAuD,aACAW;oBAGN;oBACA,IAAI,CAACrD,SACH,MAAM,IAAIsD,MAAM,CAAC,mBAAmB,EAAE/D,MAAM,MAAM,EAAE;oBAGtD,IAAIgE;oBAEJ,IAAId,yBACFc,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,OAAOhE,MAAM,KAAK;wBACpB;oBACF;yBACK,IAAIuD,cACTS,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,iBAAiBnD;4BACjB,cAAcgD;wBAChB;oBACF;yBACK,IAAIH,aACTM,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,IAAIR,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;4BACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;wBAC7B;oBACF;yBACK,IAAII,iBACTI,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,QAAQhE,MAAM,MAAM;wBACtB;oBACF;oBAGFmC,QAAAA,YAAAA,SAAW1B;oBAEX,OAAO;wBACL,QAAQ;4BACNA;wBACF;wBACAD;wBACAwD;oBACF;gBACF;YACF;YACA,OAAO5B;QACT;QAEA,KAAK,MAAMH,QAAQH,MACjB,IAAIG,AAAc,aAAdA,KAAK,IAAI,EAAe;gBAIxBgC,cACAC;YAJF,IACE,CAACjC,KAAK,MAAM,IACZA,AAAgB,SAAhBA,KAAK,MAAM,IACXgC,AAAAA,SAAAA,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,EAAE,AAAD,MAAM,QACpBC,AAAAA,SAAAA,CAAAA,gBAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,EAAE,AAAD,MAAM,QACpB;gBACAtE,MAAM,kDAAkDqC;gBACxD;YACF;YACA,MAAMkC,aAAanC,kBAAkBC,MAAMA,KAAK,MAAM;YAEtDF,MAAM,IAAI,CAACoC;QACb,OAAO,IAAIlC,AAAc,YAAdA,KAAK,IAAI,EAAc;gBAMHmC;YAL7B,MAAMC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOpC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO,aAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD;gBAC3C,QAAQnC,KAAK,MAAM;gBACnB,UAAU;wBAEWmC;oBADnB,MAAM,IAAIL,MACR9B,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAC,SAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD,KAAK;gBAE5C;YACF;YACFrC,MAAM,IAAI,CAACsC;QACb,OAAO,IAAIpC,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACnC,MAAMqC,qBAAqD;gBACzD,MAAM;gBACN,SAAS;gBACT,OAAO;gBACP,SAASrC,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOjC,SAAW;YAC9B;YACA+B,MAAM,IAAI,CAACuC;QACb,OAAO,IAAIrC,AAAc,YAAdA,KAAK,IAAI,EAAc;YAChC,MAAMsC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOtC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOuC;oBACf,MAAM5C,yBAAM4C,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;gBACnC;YACF;YACFzC,MAAM,IAAI,CAACwC;QACb,OAAO;YAEL,MAAME,WAAWxC,KAAK,IAAI;YAC1B,MAAMyC,cAAc,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW;YAC/C,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACC,SAAWA,OAAO,IAAI,KAAKF;YAC5D,MAAMzE,QAAQiC,KAAK,KAAK;YAExB,IAAI,CAAC0C,QACH,MAAM,IAAIZ,MAAM,CAAC,aAAa,EAAEU,SAAS,WAAW,CAAC;YAIvD,MAAMG,eAAeD,SACjBE,4BAA4BF,OAAO,WAAW,IAC9C,EAAE;YAEN,MAAMG,uBAAuBH,SACzBE,4BAA4BF,OAAO,WAAW,EAAE,QAChD,EAAE;YAENC,aAAa,OAAO,CAAC,CAACG;gBACpB,IAAI/E,KAAK,CAAC+E,MAAM,EAAE;oBAChB,MAAM7E,aAAaH,oBAAoBC,KAAK,CAAC+E,MAAM;oBACnDnF,MACE,uCACA,CAAC,YAAY,EAAE6E,UAAU,EACzB,CAAC,MAAM,EAAEhC,KAAK,SAAS,CAACzC,KAAK,CAAC+E,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEtC,KAAK,SAAS,CAACvC,aAAa;oBAE5C,MAAM8E,aAAahD,kBACjB9B,YACAF,KAAK,CAAC+E,MAAM,EACZ,CAAC9D;wBACCjB,KAAK,CAAC+E,MAAM,GAAG9D;oBACjB;oBAEFc,MAAM,IAAI,CAACiD;gBACb,OAAO;oBACLxC,OACE,CAACsC,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEN,UAAU;oBAE3E7E,MAAM,CAAC,OAAO,EAAEmF,MAAM,6BAA6B,EAAEN,UAAU;gBACjE;YACF;YAEA,MAAMhD,OAKF;gBACF,MAAM;gBACN,SAASgD;gBACT,SAASxC,KAAK,OAAO;gBACrB,OAAOA,KAAK,KAAK;gBACjB,UAAU,OAAOjC,OAAOsB;wBAKO2D;oBAJ7BrF,MACE,oBACA6E,UACAzE,OACA,CAAC,wBAAwB,EAAE,QAAAiF,CAAAA,mBAAAA,QAAQ,OAAO,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,MAAM,EAAE;oBAItD,MAAMzE,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBAC1Dc,QAAQ,IAAI,CAAC,WAAW,GAAGd;oBAE3BsE,qBAAqB,OAAO,CAAC,CAACC;wBAC5BvC,OACExC,KAAK,CAAC+E,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEN,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;oBAE9G;oBAEA,MAAMS,WAAWP,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;oBAC3C,MAAMO,SAASlF,OAAOsB;oBAEtB,OAAO;wBACL,QAAQ;4BACN,SAAS;4BACT,QAAQmD;4BACR,OAAOzE;wBACT;oBACF;gBACF;YACF;YACA+B,MAAM,IAAI,CAACN;QACb;QAGF,MAAM0D,eAAepD,MAAM,GAAG,CAC5B,CAACN,MAA0B2D;YACzB,IAAI3D,AAAc,aAAdA,KAAK,IAAI,EACX,OAAO,IAAI,CAAC,6BAA6B,CACvCA,MACA2D,UAAUrD,MAAM,MAAM,GAAG;YAG7B,OAAON;QACT;QAGF,OAAO;YACL,OAAO0D;QACT;IACF;IAEA,MAAc,qBAAqBE,eAAgC,EAAE;QACnE,MAAMtC,WAAWxC,KAAK,GAAG;QACzB,MAAMC,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAC1D,MAAMwC,aAAoC;YACxC,MAAM;YACN,IAAID;YACJ,YAAYvC,YAAY,gBAAgB;YACxC,QAAQ;QACV;QAEA6E,gBAAgB,IAAI,CAAC,QAAQ,GAAG;YAACrC;SAAW;QAC3CqC,gBAAgB,IAAI,CAA2B,WAAW,GAAG7E;QAE9D,OAAO;YACLA;QACF;IACF;IAEA,MAAM,uBAAuB8E,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,eAAe,IAAIC,SAASC,aAAa,UAAUJ,kBAAkB;YACzE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,MAAM7D,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL6D;YACF;YACA,UAAU,OAAOtF,OAAOqF;gBACtB,MAAM,IAAI,CAAC,oBAAoB,CAACA;gBAChC,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLE;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QAEA,MAAMC,aAAa,MAAM,CAAC/D;QAC1B,MAAM+D,aAAa,KAAK;QAExB,OAAO;YACL,UAAUA;QACZ;IACF;IAEQ,uBACNF,eAAuB,EACvBK,GAAY,EACZC,aAAsB,EACtB;QACA,MAAMnE,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL6D;gBACAK;YACF;YACA,UAAU,OAAO3F,OAAOqF;gBACtB,MAAMQ,YAAYtF,KAAK,GAAG;gBAC1B,MAAM,EAAEC,WAAW,EAAE,GACnB,MAAM,IAAI,CAAC,oBAAoB,CAAC6E;gBAElC7C,OACE,IAAI,CAAC,IAAI,CAAC,WAAW,EACrB;gBAEF,MAAMkC,cAAc,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW;gBAC/C9E,MACE,4BACA8E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDnC,OAAOsD,MAAM,OAAO,CAACpB,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBqB,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gDAAgD,CAAC;gBAI3F,MAAMC,aAAa,MAAM/D,wBAAKjC,MAAM,eAAe,EAAE;oBACnD,SAASQ;oBACT,KAAKR,MAAM,GAAG;oBACd4F;oBACA,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAC5BlB;gBACF;gBAEA,MAAM,EACJuB,OAAO,EACPN,GAAG,EACHO,kCAAkC,EAClChF,KAAK,EACLyB,KAAK,EACLwD,WAAW,EACXvE,KAAK,EACN,GAAGoE;gBAEJX,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCc;gBACF;gBACAd,gBAAgB,IAAI,CAAC,KAAK,GAAG1C;gBAE7B,MAAMyD,eAAeH,WAAW,EAAE;gBAgClC,IAAIrE,OAAO;oBACT,MAAMyE,UAAU9F,KAAK,GAAG;oBACxB,MAAM+F,gBAAgB1E,QAASyE,CAAAA,UAAUR,SAAQ;oBACjD,IAAIS,gBAAgB,GAClBF,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,OAAO;4BACL,QAAQE;wBACV;wBACA,QAAQ;oBACV;gBAEJ;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrB5D,OACE,CAAC0D,sCAAsCtE,OACvCV,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASkF;wBACTF;wBACAP;wBACA,UAAUK,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAxF;gBACF;YACF;QACF;QAEA,OAAOiB;IACT;IAEQ,mBACN6D,eAAuB,EACvBiB,gBAAmC,EACnC;QACA,MAAM9E,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL6D;YACF;YACA,UAAU,OAAOtF,OAAOqF;oBA+CTmB;gBA9Cb,MAAM,EAAEhG,WAAW,EAAE,GACnB,MAAM,IAAI,CAAC,oBAAoB,CAAC6E;gBAElC,MAAMoB,eAAe,MAAMC,qBACzBlG,YAAY,gBAAgB,EAC5BA,YAAY,IAAI,EAChB+F;gBAGF,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,WAAW;gCACT,KAAKE;4BACP;wBACF;qBACD;gBACH;gBACA,MAAMT,aAMF,MAAMW,YAAY;oBACpB,iBAAiB3G,MAAM,eAAe;oBACtC,qBAAqB,IAAI,CAAC,mBAAmB;oBAC7C,MAAMQ,YAAY,IAAI;oBACtB+F;gBACF;gBAEA,MAAM,EAAEN,OAAO,EAAEW,cAAc,EAAEjE,KAAK,EAAE,GAAGqD;gBAC3CX,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClC,aAAaW,WAAW,WAAW;gBACrC;gBACAX,gBAAgB,IAAI,CAAC,KAAK,GAAG1C;gBAC7B,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAASiE;gBACX;gBACA,OAAO;oBACL,QAAQ;wBACNX;wBACA,SAAS,QAAAO,CAAAA,YAAAA,OAAO,CAAC,EAAE,AAAD,IAATA,KAAAA,IAAAA,UAAY,OAAO;wBAC5B,YAAYP,OAAO,CAAC,EAAE,CAAC,IAAI;wBAC3B,oCAAoC;wBACpC,KAAK;wBACL,UAAUD,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;gBACF;YACF;QACF;QAEA,OAAOvE;IACT;IAEA,MAAM,SACJoF,KAAa,EACb/E,KAAuB,EACG;QAC1B,MAAM0D,eAAe,IAAIC,SAASoB,OAAO;YACvC,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAE9E,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACD;QACrD,MAAM0D,aAAa,MAAM,CAACzD;QAC1B,MAAMd,SAAS,MAAMuE,aAAa,KAAK;QACvC,MAAM,EAAEsB,MAAM,EAAE,GAAG7F;QACnB,OAAO;YACL6F;YACA,UAAUtB;QACZ;IACF;IAEA,MAAM,OACJuB,UAAkB,EAClBnB,aAAsB,EAQtB;QACA,MAAMJ,eAAe,IAAIC,SAASC,aAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,IAAIC,eACF,IAAI,CAAC,sBAAsB,CAACD,YAAYnG,QAAWgF;QACrD,IAAIqB,cAAc;QAClB,MAAMC,UAAoB,EAAE;QAE5B,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJC,oBAAoB,oBAAoB,CACtCC,oCACGxH;QACP,MAAOkH,aAAc;YACnB,IAAIC,cAAcG,sBAAsB;gBACtC,MAAMG,WACJ;gBAEF,OAAO,IAAI,CAAC,eAAe,CAAC/B,cAAc+B;YAC5C;YAGA,MAAM/B,aAAa,MAAM,CAACwB;YAC1B,MAAM/F,SAAS,MAAMuE,aAAa,KAAK;YACvC,MAAMQ,aAAiC/E,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAIuE,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQQ;gBACR,UAAUR;YACZ;YAGF,MAAM1D,QAAQkE,WAAW,OAAO,IAAI,EAAE;YACtCmB,SAAS,IAAI,IAAKnB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAIwB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAAC1F;gBACjD0D,aAAa,MAAM,CAACgC,YAAY,KAAK;YACvC,EAAE,OAAOtG,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzBsE,cACA,CAAC,4CAA4C,EAAEtE,MAAM,SAAS,EAAEuB,KAAK,SAAS,CAC5EX,QACC;YAEP;YAEA,MAAM0D,aAAa,KAAK;YACxB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ5E;gBACR,UAAU4E;YACZ;YAEF,IAAIQ,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,GAAG,EACjBkB,QAAQ,IAAI,CAAClB,WAAW,GAAG;YAG7B,IAAI,CAACA,WAAW,kCAAkC,EAAE;gBAClDgB,eAAe;gBACf;YACF;YACAA,eAAe,IAAI,CAAC,sBAAsB,CACxCD,YACAG,QAAQ,MAAM,GAAG,IAAI,CAAC,EAAE,EAAEA,QAAQ,IAAI,CAAC,SAAS,GAAGtG,QACnDgF;YAEFqB;QACF;QAEA,OAAO;YACL,QAAQ;gBACNE;YACF;YACA,UAAU3B;QACZ;IACF;IAEA,MAAM,aAAauB,UAAkB,EAOnC;QACA,MAAMvB,eAAe,IAAIC,SAASC,aAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,IAAI,CAAC,mBAAmB,GAAG,EAAE;QAC7B,MAAMU,cAAc;QACpB,IAAIC,qBAAqB;QACzB,MAAMC,kBAAkB;QAExB,MAAMR,WAAmC,EAAE;QAC3C,MAAO,CAACM,eAAeC,qBAAqBC,gBAAiB;YAC3DD;YACA9H,MACE,qCACA8H,oBACA,eACAX;YAEF,MAAMC,eAA2C,IAAI,CAAC,kBAAkB,CACtED,YACA;gBACE,QAAQ;YACV;YAEF,MAAMvB,aAAa,MAAM,CAACwB;YAC1B,MAAM/F,SAAS,MAAMuE,aAAa,KAAK;YACvC,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ5E;gBACR,UAAU4E;YACZ;YAEF,IAAI,CAACvE,QACH,MAAM,IAAI8C,MACR;YAGJ,MAAM,EAAE+C,MAAM,EAAE,GAAG7F;YACnB,MAAMa,QAAQgF,OAAO,OAAO;YAC5BK,SAAS,IAAI,IAAKL,OAAO,QAAQ,IAAI,EAAE;YACvC,IAAIU;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAAC1F;gBACjD0D,aAAa,MAAM,CAACgC,YAAY,KAAK;YACvC,EAAE,OAAOtG,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzBsE,cACA,CAAC,4CAA4C,EAAEtE,MAAM,SAAS,EAAEuB,KAAK,SAAS,CAC5EX,QACC;YAEP;YAEA,MAAM0D,aAAa,KAAK;YAExB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ5E;gBACR,UAAU4E;YACZ;YAGF,IAAI1D,AAAkB,eAAlBA,KAAK,CAAC,EAAE,CAAC,IAAI,EACf;QAEJ;QACA,OAAO;YACL,QAAQ;gBACNqF;YACF;YACA,UAAU3B;QACZ;IACF;IAEQ,oBACNoC,IAA0D,EAC1DC,MAA2B,EAC3BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBAEL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO7H,OAAOqC;gBACtB,MAAM,EAAEZ,IAAI,EAAE,GAAGY;gBACjB,IAAIK;gBACJ,MAAME,gBAAgC,CAACC;oBACrCH,cAAcG;gBAChB;gBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGD;gBAGjC,MAAMG,WAAWxC,KAAK,GAAG;gBACzB,MAAMC,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;gBAC1DiB,KAAK,WAAW,GAAGjB;gBAEnB,MAAMwC,aAAoC;oBACxC,MAAM;oBACN,IAAID;oBACJ,YAAYvC,YAAY,gBAAgB;oBACxC,QAAQ;gBACV;gBACAiB,KAAK,QAAQ,GAAG;oBAACuB;iBAAW;gBAE5B,MAAMiF,mBAAmBL,AAAS,YAATA;gBACzB,IAAIM,cAAcL;gBAClB,IAAII,kBAAkB;oBACpB,MAAME,aAAaP,AAAS,aAATA,OAAoB,YAAYA;oBACnDM,cAAc;wBACZ,QAAQ,GAAGC,WAAW,EAAE,EAAEN,QAAQ;oBACpC;gBACF;gBAEA,MAAM,EAAEO,IAAI,EAAEzF,KAAK,EAAE0F,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACzDH,aACAJ,KACAC;gBAGF,IAAIO,eAAeF;gBACnB,IAAIH,kBAAkB;oBACpBzF,OAAO4F,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,MAAMxH,QAAW;oBACnC0H,eAAgBF,KAAa,MAAM;gBACrC;gBAEA,OAAO;oBACL,QAAQE;oBACR,KAAK;wBAAE,MAAM5F;wBAAa,iBAAiBoF,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe;oBAAC;oBAChEnF;oBACA0F;gBACF;YACF;QACF;QAEA,OAAOL;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvC,eAAe,IAAIC,SACvBC,aACEkC,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpF,KAAK,SAAS,CAACoF,UAEvD;YACE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAGF,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAC,KACAC;QAGF,MAAMvC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACwC;QAC7D,MAAM/G,SAAS,MAAMuE,aAAa,KAAK;QAEvC,IAAI,CAACvE,QACH,MAAM,IAAI8C,MACR;QAIJ,MAAM,EAAE+C,MAAM,EAAEuB,OAAO,EAAE,GAAGpH;QAE5B,OAAO;YACL6F;YACAuB;YACA,UAAU7C;QACZ;IACF;IAEA,MAAM,OACJ+C,SAAsB,EACtBT,GAA0B,EACS;QACnC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,YAAYF;QACrD,OAAO,MAAM,IAAI,CAAC,wBAAwB,CACxC,UACAC,YACAV,KACAC;IAEJ;IAWQ,0BACNW,mBAA+C,EAC/C;QACA,IAAIA,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;YAEvC,MAAMC,eAAe,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAClD,CAACrI,OAASA,AAAc,WAAdA,KAAK,IAAI;YAIrB,IAAIqI,aAAa,MAAM,IAAI,KAAKD,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;gBAEnE,MAAME,oBAAoB,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAC1D,CAACtI,OAASA,AAAc,WAAdA,KAAK,IAAI;gBAErB,IAAIsI,qBAAqB,GACvB,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAACA,mBAAmB;YAEvD;QACF;QAEA,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACF;IAChC;IAEA,MAAc,gBAAgBlD,YAAsB,EAAE+B,QAAgB,EAAE;QACtE,MAAMsB,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACL,SAAStB;YACX;YACA,QAAQ;QACV;QACA,MAAM,EAAExF,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC;YAAC8G;SAAU;QAChE,MAAMrD,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACzD,KAAK,CAAC,EAAE;QACrE,MAAMyD,aAAa,KAAK;QAExB,OAAO;YACL,QAAQ5E;YACR,UAAU4E;QACZ;IACF;IAEA,MAAM,QACJ+C,SAAsB,EACtBT,GAA+B,EACC;QAChC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,YAAYF;QAErD,MAAMO,cAAc,CAAC,SAAS,EAAEN,YAAY;QAC5C,MAAMhD,eAAe,IAAIC,SAASC,aAAa,WAAWoD,cAAc;YACtE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE,GAAGlB;QAEvCtF,OAAO+F,WAAW;QAClB/F,OAAOuG,WAAW;QAClBvG,OAAOwG,iBAAiB;QAExBxG,OACEwG,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmB1I,KAAK,GAAG;QACjC,IAAIsF,YAAYtF,KAAK,GAAG;QACxB,IAAI2I,eAAe;QACnB,MAAO3I,KAAK,GAAG,KAAK0I,mBAAmBF,UAAW;YAChDlD,YAAYtF,KAAK,GAAG;YACpB,MAAMyH,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,UACAQ,YACA;gBACE,iBAAiB;gBACjB,eAAe;gBACf,iBAAiB;YACnB,GACAT;YAGF,MAAMvC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACwC;YAC7D,MAAM/G,SAAU,MAAMuE,aAAa,KAAK;YAKxC,IAAI,CAACvE,QACH,MAAM,IAAI8C,MACR;YAIJ,IAAI9C,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQL;gBACR,UAAU4E;YACZ;YAGF0D,eACEjI,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACd,CAAC,0CAA0C,EAAEuH,YAAY;YAC3D,MAAMW,MAAM5I,KAAK,GAAG;YACpB,IAAI4I,MAAMtD,YAAYmD,iBAAiB;gBACrC,MAAM1C,gBAAgB0C,kBAAmBG,CAAAA,MAAMtD,SAAQ;gBACvD,MAAMuD,YAAsD;oBAC1D,MAAM;oBACN,OAAO;wBACL,QAAQ9C;oBACV;oBACA,QAAQ;gBACV;gBACA,MAAM,EAAE,OAAO+C,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC;oBAC/DD;iBACD;gBACD,MAAM5D,aAAa,MAAM,CACvB,IAAI,CAAC,6BAA6B,CAAC6D,UAAU,CAAC,EAAE;gBAElD,MAAM7D,aAAa,KAAK;YAC1B;QACF;QAEA,OAAO,IAAI,CAAC,eAAe,CACzBA,cACA,CAAC,iBAAiB,EAAE0D,cAAc;IAEtC;IAppCA,YACEI,IAAoB,EACpBC,OAAgB,EAChBC,IAGC,CACD;QAjBF;QAEA;QAEA;QAEA,8CAAoD,EAAE;QAEtD;QAUE,IAAI,CAAC,IAAI,GAAGF;QACZ,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;IAC9C;AAyoCF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import {\n  type ChatCompletionMessageParam,\n  elementByPositionWithElementInfo,\n  findAllMidsceneLocatorField,\n  resizeImageForUiTars,\n  vlmPlanning,\n} from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport {\n  type AIUsageInfo,\n  type BaseElement,\n  type DetailedLocateParam,\n  type DumpSubscriber,\n  type ExecutionRecorderItem,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskHitBy,\n  type ExecutionTaskInsightLocateApply,\n  type ExecutionTaskInsightQueryApply,\n  type ExecutionTaskPlanning,\n  type ExecutionTaskPlanningApply,\n  type ExecutionTaskProgressOptions,\n  Executor,\n  type ExecutorContext,\n  type Insight,\n  type InsightDump,\n  type InsightExtractOption,\n  type InsightExtractParam,\n  type InterfaceType,\n  type LocateResultElement,\n  type MidsceneYamlFlowItem,\n  type PlanningAIResponse,\n  type PlanningAction,\n  type PlanningActionParamError,\n  type PlanningActionParamSleep,\n  type PlanningActionParamWaitFor,\n  type PlanningLocateParam,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type UIContext,\n  plan,\n} from '@/index';\nimport { sleep } from '@/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelPreferences,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { taskTitleStr } from './ui-utils';\nimport {\n  matchElementFromCache,\n  matchElementFromPlan,\n  parsePrompt,\n} from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  executor: Executor;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  conversationHistory: ChatCompletionMessageParam[] = [];\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n  }\n\n  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {\n    const base64 = await this.interface.screenshotBase64();\n    const item: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot: base64,\n      timing,\n    };\n    return item;\n  }\n\n  private async getElementXpath(\n    uiContext: UIContext<BaseElement>,\n    element: LocateResultElement,\n  ): Promise<string[] | undefined> {\n    if (!(this.interface as any).getXpathsByPoint) {\n      debug('getXpathsByPoint is not supported for this interface');\n      return undefined;\n    }\n\n    let elementId = element?.id;\n    if (element?.isOrderSensitive !== undefined) {\n      const xpaths = await (this.interface as any).getXpathsByPoint(\n        {\n          left: element.center[0],\n          top: element.center[1],\n        },\n        element?.isOrderSensitive,\n      );\n\n      return xpaths;\n    }\n\n    // find the nearest xpath for the element\n    if (element?.attributes?.nodeType === NodeType.POSITION) {\n      await this.insight.contextRetrieverFn('locate');\n      const info = elementByPositionWithElementInfo(\n        uiContext.tree,\n        {\n          x: element.center[0],\n          y: element.center[1],\n        },\n        {\n          requireStrictDistance: false,\n          filterPositionElements: true,\n        },\n      );\n      if (info?.id) {\n        elementId = info.id;\n      } else {\n        debug(\n          'no element id found for position node, will not update cache',\n          element,\n        );\n      }\n    }\n\n    if (!elementId) {\n      return undefined;\n    }\n    try {\n      const result = await (this.interface as any).getXpathsById(elementId);\n      return result;\n    } catch (error) {\n      debug('getXpathsById error: ', error);\n    }\n  }\n\n  private prependExecutorWithScreenshot(\n    taskApply: ExecutionTaskApply,\n    appendAfterExecution = false,\n  ): ExecutionTaskApply {\n    const taskWithScreenshot: ExecutionTaskApply = {\n      ...taskApply,\n      executor: async (param, context, ...args) => {\n        const recorder: ExecutionRecorderItem[] = [];\n        const { task } = context;\n        // set the recorder before executor in case of error\n        task.recorder = recorder;\n        const shot = await this.recordScreenshot(`before ${task.type}`);\n        recorder.push(shot);\n        const result = await taskApply.executor(param, context, ...args);\n        if (taskApply.type === 'Action') {\n          await Promise.all([\n            (async () => {\n              await sleep(100);\n              if (this.interface.beforeAction) {\n                debug('will call \"beforeAction\" for interface');\n                await this.interface.beforeAction();\n              }\n            })(),\n            sleep(200),\n          ]);\n        }\n        if (appendAfterExecution) {\n          const shot2 = await this.recordScreenshot('after Action');\n          recorder.push(shot2);\n        }\n        return result;\n      },\n    };\n    return taskWithScreenshot;\n  }\n\n  public async convertPlanToExecutable(plans: PlanningAction[]) {\n    const tasks: ExecutionTaskApply[] = [];\n\n    const taskForLocatePlan = (\n      plan: PlanningAction<PlanningLocateParam>,\n      detailedLocateParam: DetailedLocateParam | string,\n      onResult?: (result: LocateResultElement) => void,\n    ): ExecutionTaskInsightLocateApply => {\n      if (typeof detailedLocateParam === 'string') {\n        detailedLocateParam = {\n          prompt: detailedLocateParam,\n        };\n      }\n      const taskFind: ExecutionTaskInsightLocateApply = {\n        type: 'Insight',\n        subType: 'Locate',\n        param: detailedLocateParam,\n        thought: plan.thought,\n        executor: async (param, taskContext) => {\n          const { task } = taskContext;\n          assert(\n            param?.prompt || param?.id || param?.bbox,\n            `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n              param,\n            )}`,\n          );\n          let insightDump: InsightDump | undefined;\n          let usage: AIUsageInfo | undefined;\n          const dumpCollector: DumpSubscriber = (dump) => {\n            insightDump = dump;\n            usage = dump?.taskInfo?.usage;\n\n            task.log = {\n              dump: insightDump,\n            };\n\n            task.usage = usage;\n          };\n          this.insight.onceDumpUpdatedFn = dumpCollector;\n          const shotTime = Date.now();\n\n          // Get context through contextRetrieverFn which handles frozen context\n          const uiContext = await this.insight.contextRetrieverFn('locate');\n          task.uiContext = uiContext;\n\n          const recordItem: ExecutionRecorderItem = {\n            type: 'screenshot',\n            ts: shotTime,\n            screenshot: uiContext.screenshotBase64,\n            timing: 'before Insight',\n          };\n          task.recorder = [recordItem];\n\n          // try matching xpath\n          const elementFromXpath =\n            param.xpath && (this.interface as any).getElementInfoByXpath\n              ? await (this.interface as any).getElementInfoByXpath(param.xpath)\n              : undefined;\n          const userExpectedPathHitFlag = !!elementFromXpath;\n\n          // try matching cache\n          const cachePrompt = param.prompt;\n          const locateCacheRecord =\n            this.taskCache?.matchLocateCache(cachePrompt);\n          const xpaths = locateCacheRecord?.cacheContent?.xpaths;\n          const elementFromCache = userExpectedPathHitFlag\n            ? null\n            : await matchElementFromCache(\n                this,\n                xpaths,\n                cachePrompt,\n                param.cacheable,\n              );\n          const cacheHitFlag = !!elementFromCache;\n\n          // try matching plan\n          const elementFromPlan =\n            !userExpectedPathHitFlag && !cacheHitFlag\n              ? matchElementFromPlan(param, uiContext.tree)\n              : undefined;\n          const planHitFlag = !!elementFromPlan;\n\n          // try ai locate\n          const elementFromAiLocate =\n            !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag\n              ? (\n                  await this.insight.locate(param, {\n                    // fallback to ai locate\n                    context: uiContext,\n                  })\n                ).element\n              : undefined;\n          const aiLocateHitFlag = !!elementFromAiLocate;\n\n          const element =\n            elementFromXpath || // highest priority\n            elementFromCache || // second priority\n            elementFromPlan || // third priority\n            elementFromAiLocate;\n\n          // update cache\n          let currentXpaths: string[] | undefined;\n          if (\n            element &&\n            this.taskCache &&\n            !cacheHitFlag &&\n            param?.cacheable !== false\n          ) {\n            const elementXpaths = await this.getElementXpath(\n              uiContext,\n              element,\n            );\n            if (elementXpaths?.length) {\n              currentXpaths = elementXpaths;\n              this.taskCache.updateOrAppendCacheRecord(\n                {\n                  type: 'locate',\n                  prompt: cachePrompt,\n                  xpaths: elementXpaths,\n                },\n                locateCacheRecord,\n              );\n            } else {\n              debug(\n                'no xpaths found, will not update cache',\n                cachePrompt,\n                elementXpaths,\n              );\n            }\n          }\n          if (!element) {\n            throw new Error(`Element not found: ${param.prompt}`);\n          }\n\n          let hitBy: ExecutionTaskHitBy | undefined;\n\n          if (userExpectedPathHitFlag) {\n            hitBy = {\n              from: 'User expected path',\n              context: {\n                xpath: param.xpath,\n              },\n            };\n          } else if (cacheHitFlag) {\n            hitBy = {\n              from: 'Cache',\n              context: {\n                xpathsFromCache: xpaths,\n                xpathsToSave: currentXpaths,\n              },\n            };\n          } else if (planHitFlag) {\n            hitBy = {\n              from: 'Planning',\n              context: {\n                id: elementFromPlan?.id,\n                bbox: elementFromPlan?.bbox,\n              },\n            };\n          } else if (aiLocateHitFlag) {\n            hitBy = {\n              from: 'AI model',\n              context: {\n                prompt: param.prompt,\n              },\n            };\n          }\n\n          onResult?.(element);\n\n          return {\n            output: {\n              element,\n            },\n            uiContext,\n            hitBy,\n          };\n        },\n      };\n      return taskFind;\n    };\n\n    for (const plan of plans) {\n      if (plan.type === 'Locate') {\n        if (\n          !plan.locate ||\n          plan.locate === null ||\n          plan.locate?.id === null ||\n          plan.locate?.id === 'null'\n        ) {\n          debug('Locate action with id is null, will be ignored', plan);\n          continue;\n        }\n        const taskLocate = taskForLocatePlan(plan, plan.locate);\n\n        tasks.push(taskLocate);\n      } else if (plan.type === 'Error') {\n        const taskActionError: ExecutionTaskActionApply<PlanningActionParamError> =\n          {\n            type: 'Action',\n            subType: 'Error',\n            param: plan.param,\n            thought: plan.thought || plan.param?.thought,\n            locate: plan.locate,\n            executor: async () => {\n              throw new Error(\n                plan?.thought || plan.param?.thought || 'error without thought',\n              );\n            },\n          };\n        tasks.push(taskActionError);\n      } else if (plan.type === 'Finished') {\n        const taskActionFinished: ExecutionTaskActionApply<null> = {\n          type: 'Action',\n          subType: 'Finished',\n          param: null,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async (param) => {},\n        };\n        tasks.push(taskActionFinished);\n      } else if (plan.type === 'Sleep') {\n        const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =\n          {\n            type: 'Action',\n            subType: 'Sleep',\n            param: plan.param,\n            thought: plan.thought,\n            locate: plan.locate,\n            executor: async (taskParam) => {\n              await sleep(taskParam?.timeMs || 3000);\n            },\n          };\n        tasks.push(taskActionSleep);\n      } else {\n        // action in action space\n        const planType = plan.type;\n        const actionSpace = await this.interface.actionSpace();\n        const action = actionSpace.find((action) => action.name === planType);\n        const param = plan.param;\n\n        if (!action) {\n          throw new Error(`Action type '${planType}' not found`);\n        }\n\n        // find all params that needs location\n        const locateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema)\n          : [];\n\n        const requiredLocateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema, true)\n          : [];\n\n        locateFields.forEach((field) => {\n          if (param[field]) {\n            const locatePlan = locatePlanForLocate(param[field]);\n            debug(\n              'will prepend locate param for field',\n              `action.type=${planType}`,\n              `param=${JSON.stringify(param[field])}`,\n              `locatePlan=${JSON.stringify(locatePlan)}`,\n            );\n            const locateTask = taskForLocatePlan(\n              locatePlan,\n              param[field],\n              (result) => {\n                param[field] = result;\n              },\n            );\n            tasks.push(locateTask);\n          } else {\n            assert(\n              !requiredLocateFields.includes(field),\n              `Required locate field '${field}' is not provided for action ${planType}`,\n            );\n            debug(`field '${field}' is not provided for action ${planType}`);\n          }\n        });\n\n        const task: ExecutionTaskApply<\n          'Action',\n          any,\n          { success: boolean; action: string; param: any },\n          void\n        > = {\n          type: 'Action',\n          subType: planType,\n          thought: plan.thought,\n          param: plan.param,\n          executor: async (param, context) => {\n            debug(\n              'executing action',\n              planType,\n              param,\n              `context.element.center: ${context.element?.center}`,\n            );\n\n            // Get context for actionSpace operations to ensure size info is available\n            const uiContext = await this.insight.contextRetrieverFn('locate');\n            context.task.uiContext = uiContext;\n\n            requiredLocateFields.forEach((field) => {\n              assert(\n                param[field],\n                `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n              );\n            });\n\n            const actionFn = action.call.bind(this.interface);\n            await actionFn(param, context);\n            // Return a proper result for report generation\n            return {\n              output: {\n                success: true,\n                action: planType,\n                param: param,\n              },\n            };\n          },\n        };\n        tasks.push(task);\n      }\n    }\n\n    const wrappedTasks = tasks.map(\n      (task: ExecutionTaskApply, index: number) => {\n        if (task.type === 'Action') {\n          return this.prependExecutorWithScreenshot(\n            task,\n            index === tasks.length - 1,\n          );\n        }\n        return task;\n      },\n    );\n\n    return {\n      tasks: wrappedTasks,\n    };\n  }\n\n  private async setupPlanningContext(executorContext: ExecutorContext) {\n    const shotTime = Date.now();\n    const uiContext = await this.insight.contextRetrieverFn('locate');\n    const recordItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: shotTime,\n      screenshot: uiContext.screenshotBase64,\n      timing: 'before Planning',\n    };\n\n    executorContext.task.recorder = [recordItem];\n    (executorContext.task as ExecutionTaskPlanning).uiContext = uiContext;\n\n    return {\n      uiContext,\n    };\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        await this.setupPlanningContext(executorContext);\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n\n    await taskExecutor.append(task);\n    await taskExecutor.flush();\n\n    return {\n      executor: taskExecutor,\n    };\n  }\n\n  private planningTaskFromPrompt(\n    userInstruction: string,\n    log?: string,\n    actionContext?: string,\n  ) {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n        log,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = await this.setupPlanningContext(executorContext);\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await plan(param.userInstruction, {\n          context: uiContext,\n          log: param.log,\n          actionContext,\n          interfaceType: this.interface.interfaceType as InterfaceType,\n          actionSpace,\n        });\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        // TODO: check locate result\n        // let bboxCollected = false;\n        // (actions || []).reduce<PlanningAction[]>(\n        //   (acc, planningAction) => {\n        //     // TODO: magic field \"locate\" is used to indicate the action requires a locate\n        //     if (planningAction.locate) {\n        //       // we only collect bbox once, let qwen re-locate in the following steps\n        //       if (bboxCollected && planningAction.locate.bbox) {\n        //         // biome-ignore lint/performance/noDelete: <explanation>\n        //         delete planningAction.locate.bbox;\n        //       }\n\n        //       if (planningAction.locate.bbox) {\n        //         bboxCollected = true;\n        //       }\n\n        //       acc.push({\n        //         type: 'Locate',\n        //         locate: planningAction.locate,\n        //         param: null,\n        //         // thought is prompt created by ai, always a string\n        //         thought: planningAction.locate.prompt as string,\n        //       });\n        //     }\n        //     acc.push(planningAction);\n        //     return acc;\n        //   },\n        //   [],\n        // );\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push({\n              type: 'Sleep',\n              param: {\n                timeMs: timeRemaining,\n              },\n              locate: null,\n            } as PlanningAction<PlanningActionParamSleep>);\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  private planningTaskToGoal(\n    userInstruction: string,\n    modelPreferences: IModelPreferences,\n  ) {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = await this.setupPlanningContext(executorContext);\n\n        const imagePayload = await resizeImageForUiTars(\n          uiContext.screenshotBase64,\n          uiContext.size,\n          modelPreferences,\n        );\n\n        this.appendConversationHistory({\n          role: 'user',\n          content: [\n            {\n              type: 'image_url',\n              image_url: {\n                url: imagePayload,\n              },\n            },\n          ],\n        });\n        const planResult: {\n          actions: PlanningAction<any>[];\n          action_summary: string;\n          usage?: AIUsageInfo;\n          yamlFlow?: MidsceneYamlFlowItem[];\n          rawResponse?: string;\n        } = await vlmPlanning({\n          userInstruction: param.userInstruction,\n          conversationHistory: this.conversationHistory,\n          size: uiContext.size,\n          modelPreferences,\n        });\n\n        const { actions, action_summary, usage } = planResult;\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse: planResult.rawResponse,\n        };\n        executorContext.task.usage = usage;\n        this.appendConversationHistory({\n          role: 'assistant',\n          content: action_summary,\n        });\n        return {\n          output: {\n            actions,\n            thought: actions[0]?.thought,\n            actionType: actions[0].type,\n            more_actions_needed_by_instruction: true,\n            log: '',\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n  ): Promise<ExecutionResult> {\n    const taskExecutor = new Executor(title, {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { tasks } = await this.convertPlanToExecutable(plans);\n    await taskExecutor.append(tasks);\n    const result = await taskExecutor.flush();\n    const { output } = result!;\n    return {\n      output,\n      executor: taskExecutor,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    actionContext?: string,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    let planningTask: ExecutionTaskPlanningApply | null =\n      this.planningTaskFromPrompt(userPrompt, undefined, actionContext);\n    let replanCount = 0;\n    const logList: string[] = [];\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) || defaultReplanningCycleLimit;\n    while (planningTask) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg =\n          'Replanning too many times, please split the task into multiple steps';\n\n        return this.appendErrorPlan(taskExecutor, errorMsg);\n      }\n\n      // plan\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      const planResult: PlanningAIResponse = result?.output;\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: planResult,\n          executor: taskExecutor,\n        };\n      }\n\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n\n      await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (planResult?.log) {\n        logList.push(planResult.log);\n      }\n\n      if (!planResult.more_actions_needed_by_instruction) {\n        planningTask = null;\n        break;\n      }\n      planningTask = this.planningTaskFromPrompt(\n        userPrompt,\n        logList.length > 0 ? `- ${logList.join('\\n- ')}` : undefined,\n        actionContext,\n      );\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  async actionToGoal(userPrompt: string): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    this.conversationHistory = [];\n    const isCompleted = false;\n    let currentActionCount = 0;\n    const maxActionNumber = 40;\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    while (!isCompleted && currentActionCount < maxActionNumber) {\n      currentActionCount++;\n      debug(\n        'actionToGoal, currentActionCount:',\n        currentActionCount,\n        'userPrompt:',\n        userPrompt,\n      );\n      const planningTask: ExecutionTaskPlanningApply = this.planningTaskToGoal(\n        userPrompt,\n        {\n          intent: 'planning',\n        },\n      );\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function actionToGoal',\n        );\n      }\n      const { output } = result;\n      const plans = output.actions;\n      yamlFlow.push(...(output.yamlFlow || []));\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n\n      await taskExecutor.flush();\n\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      if (plans[0].type === 'Finished') {\n        break;\n      }\n    }\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        // TODO: display image thumbnail in report\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let insightDump: InsightDump | undefined;\n        const dumpCollector: DumpSubscriber = (dump) => {\n          insightDump = dump;\n        };\n        this.insight.onceDumpUpdatedFn = dumpCollector;\n\n        // Get context for query operations\n        const shotTime = Date.now();\n        const uiContext = await this.insight.contextRetrieverFn('extract');\n        task.uiContext = uiContext;\n\n        const recordItem: ExecutionRecorderItem = {\n          type: 'screenshot',\n          ts: shotTime,\n          screenshot: uiContext.screenshotBase64,\n          timing: 'before Extract',\n        };\n        task.recorder = [recordItem];\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        if (ifTypeRestricted) {\n          const returnType = type === 'Assert' ? 'Boolean' : type;\n          demandInput = {\n            result: `${returnType}, ${demand}`,\n          };\n        }\n\n        const { data, usage, thought } = await this.insight.extract<any>(\n          demandInput,\n          opt,\n          multimodalPrompt,\n        );\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          assert(data?.result !== undefined, 'No result in query data');\n          outputResult = (data as any).result;\n        }\n\n        return {\n          output: outputResult,\n          log: { dump: insightDump, isWaitForAssert: opt?.isWaitForAssert },\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const taskExecutor = new Executor(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        onTaskStart: this.onTaskStartCallback,\n      },\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      opt,\n      multimodalPrompt,\n    );\n\n    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n    const result = await taskExecutor.flush();\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      executor: taskExecutor,\n    };\n  }\n\n  async assert(\n    assertion: TUserPrompt,\n    opt?: InsightExtractOption,\n  ): Promise<ExecutionResult<boolean>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n    return await this.createTypeQueryExecution<boolean>(\n      'Assert',\n      textPrompt,\n      opt,\n      multimodalPrompt,\n    );\n  }\n\n  /**\n   * Append a message to the conversation history\n   * For user messages with images:\n   * - Keep max 4 user image messages in history\n   * - Remove oldest user image message when limit reached\n   * For assistant messages:\n   * - Simply append to history\n   * @param conversationHistory Message to append\n   */\n  private appendConversationHistory(\n    conversationHistory: ChatCompletionMessageParam,\n  ) {\n    if (conversationHistory.role === 'user') {\n      // Get all existing user messages with images\n      const userImgItems = this.conversationHistory.filter(\n        (item) => item.role === 'user',\n      );\n\n      // If we already have 4 user image messages\n      if (userImgItems.length >= 4 && conversationHistory.role === 'user') {\n        // Remove first user image message when we already have 4, before adding new one\n        const firstUserImgIndex = this.conversationHistory.findIndex(\n          (item) => item.role === 'user',\n        );\n        if (firstUserImgIndex >= 0) {\n          this.conversationHistory.splice(firstUserImgIndex, 1);\n        }\n      }\n    }\n    // For non-user messages, simply append to history\n    this.conversationHistory.push(conversationHistory);\n  }\n\n  private async appendErrorPlan(taskExecutor: Executor, errorMsg: string) {\n    const errorPlan: PlanningAction<PlanningActionParamError> = {\n      type: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      locate: null,\n    };\n    const { tasks } = await this.convertPlanToExecutable([errorPlan]);\n    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));\n    await taskExecutor.flush();\n\n    return {\n      output: undefined,\n      executor: taskExecutor,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'Assert',\n        textPrompt,\n        {\n          isWaitForAssert: true,\n          returnThought: true,\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n      const result = (await taskExecutor.flush()) as {\n        output: boolean;\n        thought?: string;\n      };\n\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function waitFor',\n        );\n      }\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepPlan: PlanningAction<PlanningActionParamSleep> = {\n          type: 'Sleep',\n          param: {\n            timeMs: timeRemaining,\n          },\n          locate: null,\n        };\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable([\n          sleepPlan,\n        ]);\n        await taskExecutor.append(\n          this.prependExecutorWithScreenshot(sleepTasks[0]),\n        );\n        await taskExecutor.flush();\n      }\n    }\n\n    return this.appendErrorPlan(\n      taskExecutor,\n      `waitFor timeout: ${errorThought}`,\n    );\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","locatePlanForLocate","param","locate","locatePlan","TaskExecutor","timing","base64","item","Date","uiContext","element","_element_attributes","elementId","undefined","xpaths","NodeType","info","elementByPositionWithElementInfo","result","error","taskApply","appendAfterExecution","taskWithScreenshot","context","args","recorder","task","shot","Promise","sleep","shot2","plans","tasks","taskForLocatePlan","plan","detailedLocateParam","onResult","taskFind","taskContext","_this_taskCache","_locateCacheRecord_cacheContent","assert","JSON","insightDump","usage","dumpCollector","dump","_dump_taskInfo","shotTime","recordItem","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","aiLocateHitFlag","currentXpaths","elementXpaths","Error","hitBy","_plan_locate","_plan_locate1","taskLocate","_plan_param","taskActionError","taskActionFinished","taskActionSleep","taskParam","planType","actionSpace","action","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","locateTask","_context_element","actionFn","wrappedTasks","index","executorContext","userInstruction","yamlString","taskExecutor","Executor","taskTitleStr","log","actionContext","startTime","Array","console","planResult","actions","more_actions_needed_by_instruction","rawResponse","finalActions","timeNow","timeRemaining","modelPreferences","_actions_","imagePayload","resizeImageForUiTars","vlmPlanning","action_summary","title","output","userPrompt","planningTask","replanCount","logList","yamlFlow","replanningCycleLimit","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","errorMsg","executables","isCompleted","currentActionCount","maxActionNumber","type","demand","opt","multimodalPrompt","queryTask","ifTypeRestricted","demandInput","returnType","data","thought","outputResult","assertion","textPrompt","parsePrompt","conversationHistory","userImgItems","firstUserImgIndex","errorPlan","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepPlan","sleepTasks","interfaceInstance","insight","opts"],"mappings":";;;;;;;;;;;;;;;;;;;AAiEA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AAE7B,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAEO,MAAMC;IAYX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAgBA,MAAc,iBAAiBC,MAAuC,EAAE;QACtE,MAAMC,SAAS,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB;QACpD,MAAMC,OAA8B;YAClC,MAAM;YACN,IAAIC,KAAK,GAAG;YACZ,YAAYF;YACZD;QACF;QACA,OAAOE;IACT;IAEA,MAAc,gBACZE,SAAiC,EACjCC,OAA4B,EACG;YAoB3BC;QAnBJ,IAAI,CAAE,IAAI,CAAC,SAAS,CAAS,gBAAgB,EAAE,YAC7Cd,MAAM;QAIR,IAAIe,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE;QAC3B,IAAIA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB,AAAD,MAAMG,QAAW;YAC3C,MAAMC,SAAS,MAAO,IAAI,CAAC,SAAS,CAAS,gBAAgB,CAC3D;gBACE,MAAMJ,QAAQ,MAAM,CAAC,EAAE;gBACvB,KAAKA,QAAQ,MAAM,CAAC,EAAE;YACxB,GACAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB;YAG3B,OAAOI;QACT;QAGA,IAAIH,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,sBAAAA,QAAS,UAAU,AAAD,IAAlBA,KAAAA,IAAAA,oBAAqB,QAAQ,AAAD,MAAMI,SAAS,QAAQ,EAAE;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;YACtC,MAAMC,OAAOC,iCACXR,UAAU,IAAI,EACd;gBACE,GAAGC,QAAQ,MAAM,CAAC,EAAE;gBACpB,GAAGA,QAAQ,MAAM,CAAC,EAAE;YACtB,GACA;gBACE,uBAAuB;gBACvB,wBAAwB;YAC1B;YAEF,IAAIM,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE,EACVJ,YAAYI,KAAK,EAAE;iBAEnBnB,MACE,gEACAa;QAGN;QAEA,IAAI,CAACE,WACH;QAEF,IAAI;YACF,MAAMM,SAAS,MAAO,IAAI,CAAC,SAAS,CAAS,aAAa,CAACN;YAC3D,OAAOM;QACT,EAAE,OAAOC,OAAO;YACdtB,MAAM,yBAAyBsB;QACjC;IACF;IAEQ,8BACNC,SAA6B,EAC7BC,uBAAuB,KAAK,EACR;QACpB,MAAMC,qBAAyC;YAC7C,GAAGF,SAAS;YACZ,UAAU,OAAOnB,OAAOsB,SAAS,GAAGC;gBAClC,MAAMC,WAAoC,EAAE;gBAC5C,MAAM,EAAEC,IAAI,EAAE,GAAGH;gBAEjBG,KAAK,QAAQ,GAAGD;gBAChB,MAAME,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAED,KAAK,IAAI,EAAE;gBAC9DD,SAAS,IAAI,CAACE;gBACd,MAAMT,SAAS,MAAME,UAAU,QAAQ,CAACnB,OAAOsB,YAAYC;gBAC3D,IAAIJ,AAAmB,aAAnBA,UAAU,IAAI,EAChB,MAAMQ,QAAQ,GAAG,CAAC;oBACf;wBACC,MAAMC,yBAAM;wBACZ,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;4BAC/BhC,MAAM;4BACN,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY;wBACnC;oBACF;oBACAgC,yBAAM;iBACP;gBAEH,IAAIR,sBAAsB;oBACxB,MAAMS,QAAQ,MAAM,IAAI,CAAC,gBAAgB,CAAC;oBAC1CL,SAAS,IAAI,CAACK;gBAChB;gBACA,OAAOZ;YACT;QACF;QACA,OAAOI;IACT;IAEA,MAAa,wBAAwBS,KAAuB,EAAE;QAC5D,MAAMC,QAA8B,EAAE;QAEtC,MAAMC,oBAAoB,CACxBC,MACAC,qBACAC;YAEA,IAAI,AAA+B,YAA/B,OAAOD,qBACTA,sBAAsB;gBACpB,QAAQA;YACV;YAEF,MAAME,WAA4C;gBAChD,MAAM;gBACN,SAAS;gBACT,OAAOF;gBACP,SAASD,KAAK,OAAO;gBACrB,UAAU,OAAOjC,OAAOqC;wBA6CpBC,iBACaC;oBA7Cf,MAAM,EAAEd,IAAI,EAAE,GAAGY;oBACjBG,OACExC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAEyC,KAAK,SAAS,CACpEzC,QACC;oBAEL,IAAI0C;oBACJ,IAAIC;oBACJ,MAAMC,gBAAgC,CAACC;4BAE7BC;wBADRJ,cAAcG;wBACdF,QAAQG,QAAAA,OAAAA,KAAAA,IAAAA,QAAAA,CAAAA,iBAAAA,KAAM,QAAQ,AAAD,IAAbA,KAAAA,IAAAA,eAAgB,KAAK;wBAE7BrB,KAAK,GAAG,GAAG;4BACT,MAAMiB;wBACR;wBAEAjB,KAAK,KAAK,GAAGkB;oBACf;oBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGC;oBACjC,MAAMG,WAAWxC,KAAK,GAAG;oBAGzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxDiB,KAAK,SAAS,GAAGjB;oBAEjB,MAAMwC,aAAoC;wBACxC,MAAM;wBACN,IAAID;wBACJ,YAAYvC,UAAU,gBAAgB;wBACtC,QAAQ;oBACV;oBACAiB,KAAK,QAAQ,GAAG;wBAACuB;qBAAW;oBAG5B,MAAMC,mBACJjD,MAAM,KAAK,IAAK,IAAI,CAAC,SAAS,CAAS,qBAAqB,GACxD,MAAO,IAAI,CAAC,SAAS,CAAS,qBAAqB,CAACA,MAAM,KAAK,IAC/DY;oBACN,MAAMsC,0BAA0B,CAAC,CAACD;oBAGlC,MAAME,cAAcnD,MAAM,MAAM;oBAChC,MAAMoD,oBAAAA,QACJd,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACa;oBACnC,MAAMtC,SAAS0B,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,MAAM;oBACtD,MAAMc,mBAAmBH,0BACrB,OACA,MAAMI,sBACJ,IAAI,EACJzC,QACAsC,aACAnD,MAAM,SAAS;oBAErB,MAAMuD,eAAe,CAAC,CAACF;oBAGvB,MAAMG,kBACJ,AAACN,2BAA4BK,eAEzB3C,SADA6C,qBAAqBzD,OAAOQ,UAAU,IAAI;oBAEhD,MAAMkD,cAAc,CAAC,CAACF;oBAGtB,MAAMG,sBACJ,AAACT,2BAA4BK,gBAAiBG,cAO1C9C,SALE,OAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAACZ,OAAO;wBAE/B,SAASQ;oBACX,EAAC,EACD,OAAO;oBAEf,MAAMoD,kBAAkB,CAAC,CAACD;oBAE1B,MAAMlD,UACJwC,oBACAI,oBACAG,mBACAG;oBAGF,IAAIE;oBACJ,IACEpD,WACA,IAAI,CAAC,SAAS,IACd,CAAC8C,gBACDvD,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OACrB;wBACA,MAAM8D,gBAAgB,MAAM,IAAI,CAAC,eAAe,CAC9CtD,WACAC;wBAEF,IAAIqD,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,EAAE;4BACzBD,gBAAgBC;4BAChB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gCACE,MAAM;gCACN,QAAQX;gCACR,QAAQW;4BACV,GACAV;wBAEJ,OACExD,MACE,0CACAuD,aACAW;oBAGN;oBACA,IAAI,CAACrD,SACH,MAAM,IAAIsD,MAAM,CAAC,mBAAmB,EAAE/D,MAAM,MAAM,EAAE;oBAGtD,IAAIgE;oBAEJ,IAAId,yBACFc,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,OAAOhE,MAAM,KAAK;wBACpB;oBACF;yBACK,IAAIuD,cACTS,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,iBAAiBnD;4BACjB,cAAcgD;wBAChB;oBACF;yBACK,IAAIH,aACTM,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,IAAIR,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;4BACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;wBAC7B;oBACF;yBACK,IAAII,iBACTI,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,QAAQhE,MAAM,MAAM;wBACtB;oBACF;oBAGFmC,QAAAA,YAAAA,SAAW1B;oBAEX,OAAO;wBACL,QAAQ;4BACNA;wBACF;wBACAD;wBACAwD;oBACF;gBACF;YACF;YACA,OAAO5B;QACT;QAEA,KAAK,MAAMH,QAAQH,MACjB,IAAIG,AAAc,aAAdA,KAAK,IAAI,EAAe;gBAIxBgC,cACAC;YAJF,IACE,CAACjC,KAAK,MAAM,IACZA,AAAgB,SAAhBA,KAAK,MAAM,IACXgC,AAAAA,SAAAA,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,EAAE,AAAD,MAAM,QACpBC,AAAAA,SAAAA,CAAAA,gBAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,EAAE,AAAD,MAAM,QACpB;gBACAtE,MAAM,kDAAkDqC;gBACxD;YACF;YACA,MAAMkC,aAAanC,kBAAkBC,MAAMA,KAAK,MAAM;YAEtDF,MAAM,IAAI,CAACoC;QACb,OAAO,IAAIlC,AAAc,YAAdA,KAAK,IAAI,EAAc;gBAMHmC;YAL7B,MAAMC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOpC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO,aAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD;gBAC3C,QAAQnC,KAAK,MAAM;gBACnB,UAAU;wBAEWmC;oBADnB,MAAM,IAAIL,MACR9B,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAC,SAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD,KAAK;gBAE5C;YACF;YACFrC,MAAM,IAAI,CAACsC;QACb,OAAO,IAAIpC,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACnC,MAAMqC,qBAAqD;gBACzD,MAAM;gBACN,SAAS;gBACT,OAAO;gBACP,SAASrC,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOjC,SAAW;YAC9B;YACA+B,MAAM,IAAI,CAACuC;QACb,OAAO,IAAIrC,AAAc,YAAdA,KAAK,IAAI,EAAc;YAChC,MAAMsC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOtC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOuC;oBACf,MAAM5C,yBAAM4C,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;gBACnC;YACF;YACFzC,MAAM,IAAI,CAACwC;QACb,OAAO;YAEL,MAAME,WAAWxC,KAAK,IAAI;YAC1B,MAAMyC,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;YACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACC,SAAWA,OAAO,IAAI,KAAKF;YAC5D,MAAMzE,QAAQiC,KAAK,KAAK;YAExB,IAAI,CAAC0C,QACH,MAAM,IAAIZ,MAAM,CAAC,aAAa,EAAEU,SAAS,WAAW,CAAC;YAIvD,MAAMG,eAAeD,SACjBE,4BAA4BF,OAAO,WAAW,IAC9C,EAAE;YAEN,MAAMG,uBAAuBH,SACzBE,4BAA4BF,OAAO,WAAW,EAAE,QAChD,EAAE;YAENC,aAAa,OAAO,CAAC,CAACG;gBACpB,IAAI/E,KAAK,CAAC+E,MAAM,EAAE;oBAChB,MAAM7E,aAAaH,oBAAoBC,KAAK,CAAC+E,MAAM;oBACnDnF,MACE,uCACA,CAAC,YAAY,EAAE6E,UAAU,EACzB,CAAC,MAAM,EAAEhC,KAAK,SAAS,CAACzC,KAAK,CAAC+E,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEtC,KAAK,SAAS,CAACvC,aAAa;oBAE5C,MAAM8E,aAAahD,kBACjB9B,YACAF,KAAK,CAAC+E,MAAM,EACZ,CAAC9D;wBACCjB,KAAK,CAAC+E,MAAM,GAAG9D;oBACjB;oBAEFc,MAAM,IAAI,CAACiD;gBACb,OAAO;oBACLxC,OACE,CAACsC,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEN,UAAU;oBAE3E7E,MAAM,CAAC,OAAO,EAAEmF,MAAM,6BAA6B,EAAEN,UAAU;gBACjE;YACF;YAEA,MAAMhD,OAKF;gBACF,MAAM;gBACN,SAASgD;gBACT,SAASxC,KAAK,OAAO;gBACrB,OAAOA,KAAK,KAAK;gBACjB,UAAU,OAAOjC,OAAOsB;wBAKO2D;oBAJ7BrF,MACE,oBACA6E,UACAzE,OACA,CAAC,wBAAwB,EAAE,QAAAiF,CAAAA,mBAAAA,QAAQ,OAAO,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,MAAM,EAAE;oBAItD,MAAMzE,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxDc,QAAQ,IAAI,CAAC,SAAS,GAAGd;oBAEzBsE,qBAAqB,OAAO,CAAC,CAACC;wBAC5BvC,OACExC,KAAK,CAAC+E,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEN,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;oBAE9G;oBAEA,MAAMS,WAAWP,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAChD,MAAMO,SAASlF,OAAOsB;oBAEtB,OAAO;wBACL,QAAQ;4BACN,SAAS;4BACT,QAAQmD;4BACR,OAAOzE;wBACT;oBACF;gBACF;YACF;YACA+B,MAAM,IAAI,CAACN;QACb;QAGF,MAAM0D,eAAepD,MAAM,GAAG,CAC5B,CAACN,MAA0B2D;YACzB,IAAI3D,AAAc,aAAdA,KAAK,IAAI,EACX,OAAO,IAAI,CAAC,6BAA6B,CACvCA,MACA2D,UAAUrD,MAAM,MAAM,GAAG;YAG7B,OAAON;QACT;QAGF,OAAO;YACL,OAAO0D;QACT;IACF;IAEA,MAAc,qBAAqBE,eAAgC,EAAE;QACnE,MAAMtC,WAAWxC,KAAK,GAAG;QACzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QACxD,MAAMwC,aAAoC;YACxC,MAAM;YACN,IAAID;YACJ,YAAYvC,UAAU,gBAAgB;YACtC,QAAQ;QACV;QAEA6E,gBAAgB,IAAI,CAAC,QAAQ,GAAG;YAACrC;SAAW;QAC3CqC,gBAAgB,IAAI,CAA2B,SAAS,GAAG7E;QAE5D,OAAO;YACLA;QACF;IACF;IAEA,MAAM,uBAAuB8E,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,eAAe,IAAIC,SAASC,aAAa,UAAUJ,kBAAkB;YACzE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,MAAM7D,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL6D;YACF;YACA,UAAU,OAAOtF,OAAOqF;gBACtB,MAAM,IAAI,CAAC,oBAAoB,CAACA;gBAChC,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLE;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QAEA,MAAMC,aAAa,MAAM,CAAC/D;QAC1B,MAAM+D,aAAa,KAAK;QAExB,OAAO;YACL,UAAUA;QACZ;IACF;IAEQ,uBACNF,eAAuB,EACvBK,GAAY,EACZC,aAAsB,EACtB;QACA,MAAMnE,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL6D;gBACAK;YACF;YACA,UAAU,OAAO3F,OAAOqF;gBACtB,MAAMQ,YAAYtF,KAAK,GAAG;gBAC1B,MAAM,EAAEC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC6E;gBAEtD7C,OACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMkC,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD9E,MACE,sCACA8E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDnC,OAAOsD,MAAM,OAAO,CAACpB,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBqB,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAM/D,wBAAKjC,MAAM,eAAe,EAAE;oBACnD,SAASQ;oBACT,KAAKR,MAAM,GAAG;oBACd4F;oBACA,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3ClB;gBACF;gBAEA,MAAM,EACJuB,OAAO,EACPN,GAAG,EACHO,kCAAkC,EAClChF,KAAK,EACLyB,KAAK,EACLwD,WAAW,EACXvE,KAAK,EACN,GAAGoE;gBAEJX,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCc;gBACF;gBACAd,gBAAgB,IAAI,CAAC,KAAK,GAAG1C;gBAE7B,MAAMyD,eAAeH,WAAW,EAAE;gBAgClC,IAAIrE,OAAO;oBACT,MAAMyE,UAAU9F,KAAK,GAAG;oBACxB,MAAM+F,gBAAgB1E,QAASyE,CAAAA,UAAUR,SAAQ;oBACjD,IAAIS,gBAAgB,GAClBF,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,OAAO;4BACL,QAAQE;wBACV;wBACA,QAAQ;oBACV;gBAEJ;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrB5D,OACE,CAAC0D,sCAAsCtE,OACvCV,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASkF;wBACTF;wBACAP;wBACA,UAAUK,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAxF;gBACF;YACF;QACF;QAEA,OAAOiB;IACT;IAEQ,mBACN6D,eAAuB,EACvBiB,gBAAmC,EACnC;QACA,MAAM9E,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL6D;YACF;YACA,UAAU,OAAOtF,OAAOqF;oBA8CTmB;gBA7Cb,MAAM,EAAEhG,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC6E;gBAEtD,MAAMoB,eAAe,MAAMC,qBACzBlG,UAAU,gBAAgB,EAC1BA,UAAU,IAAI,EACd+F;gBAGF,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,WAAW;gCACT,KAAKE;4BACP;wBACF;qBACD;gBACH;gBACA,MAAMT,aAMF,MAAMW,YAAY;oBACpB,iBAAiB3G,MAAM,eAAe;oBACtC,qBAAqB,IAAI,CAAC,mBAAmB;oBAC7C,MAAMQ,UAAU,IAAI;oBACpB+F;gBACF;gBAEA,MAAM,EAAEN,OAAO,EAAEW,cAAc,EAAEjE,KAAK,EAAE,GAAGqD;gBAC3CX,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClC,aAAaW,WAAW,WAAW;gBACrC;gBACAX,gBAAgB,IAAI,CAAC,KAAK,GAAG1C;gBAC7B,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAASiE;gBACX;gBACA,OAAO;oBACL,QAAQ;wBACNX;wBACA,SAAS,QAAAO,CAAAA,YAAAA,OAAO,CAAC,EAAE,AAAD,IAATA,KAAAA,IAAAA,UAAY,OAAO;wBAC5B,YAAYP,OAAO,CAAC,EAAE,CAAC,IAAI;wBAC3B,oCAAoC;wBACpC,KAAK;wBACL,UAAUD,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;gBACF;YACF;QACF;QAEA,OAAOvE;IACT;IAEA,MAAM,SACJoF,KAAa,EACb/E,KAAuB,EACG;QAC1B,MAAM0D,eAAe,IAAIC,SAASoB,OAAO;YACvC,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAE9E,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACD;QACrD,MAAM0D,aAAa,MAAM,CAACzD;QAC1B,MAAMd,SAAS,MAAMuE,aAAa,KAAK;QACvC,MAAM,EAAEsB,MAAM,EAAE,GAAG7F;QACnB,OAAO;YACL6F;YACA,UAAUtB;QACZ;IACF;IAEA,MAAM,OACJuB,UAAkB,EAClBnB,aAAsB,EAQtB;QACA,MAAMJ,eAAe,IAAIC,SAASC,aAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,IAAIC,eACF,IAAI,CAAC,sBAAsB,CAACD,YAAYnG,QAAWgF;QACrD,IAAIqB,cAAc;QAClB,MAAMC,UAAoB,EAAE;QAE5B,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJC,oBAAoB,oBAAoB,CACtCC,oCACGxH;QACP,MAAOkH,aAAc;YACnB,IAAIC,cAAcG,sBAAsB;gBACtC,MAAMG,WACJ;gBAEF,OAAO,IAAI,CAAC,eAAe,CAAC/B,cAAc+B;YAC5C;YAGA,MAAM/B,aAAa,MAAM,CAACwB;YAC1B,MAAM/F,SAAS,MAAMuE,aAAa,KAAK;YACvC,MAAMQ,aAAiC/E,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAIuE,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQQ;gBACR,UAAUR;YACZ;YAGF,MAAM1D,QAAQkE,WAAW,OAAO,IAAI,EAAE;YACtCmB,SAAS,IAAI,IAAKnB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAIwB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAAC1F;gBACjD0D,aAAa,MAAM,CAACgC,YAAY,KAAK;YACvC,EAAE,OAAOtG,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzBsE,cACA,CAAC,4CAA4C,EAAEtE,MAAM,SAAS,EAAEuB,KAAK,SAAS,CAC5EX,QACC;YAEP;YAEA,MAAM0D,aAAa,KAAK;YACxB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ5E;gBACR,UAAU4E;YACZ;YAEF,IAAIQ,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,GAAG,EACjBkB,QAAQ,IAAI,CAAClB,WAAW,GAAG;YAG7B,IAAI,CAACA,WAAW,kCAAkC,EAAE;gBAClDgB,eAAe;gBACf;YACF;YACAA,eAAe,IAAI,CAAC,sBAAsB,CACxCD,YACAG,QAAQ,MAAM,GAAG,IAAI,CAAC,EAAE,EAAEA,QAAQ,IAAI,CAAC,SAAS,GAAGtG,QACnDgF;YAEFqB;QACF;QAEA,OAAO;YACL,QAAQ;gBACNE;YACF;YACA,UAAU3B;QACZ;IACF;IAEA,MAAM,aAAauB,UAAkB,EAOnC;QACA,MAAMvB,eAAe,IAAIC,SAASC,aAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,IAAI,CAAC,mBAAmB,GAAG,EAAE;QAC7B,MAAMU,cAAc;QACpB,IAAIC,qBAAqB;QACzB,MAAMC,kBAAkB;QAExB,MAAMR,WAAmC,EAAE;QAC3C,MAAO,CAACM,eAAeC,qBAAqBC,gBAAiB;YAC3DD;YACA9H,MACE,qCACA8H,oBACA,eACAX;YAEF,MAAMC,eAA2C,IAAI,CAAC,kBAAkB,CACtED,YACA;gBACE,QAAQ;YACV;YAEF,MAAMvB,aAAa,MAAM,CAACwB;YAC1B,MAAM/F,SAAS,MAAMuE,aAAa,KAAK;YACvC,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ5E;gBACR,UAAU4E;YACZ;YAEF,IAAI,CAACvE,QACH,MAAM,IAAI8C,MACR;YAGJ,MAAM,EAAE+C,MAAM,EAAE,GAAG7F;YACnB,MAAMa,QAAQgF,OAAO,OAAO;YAC5BK,SAAS,IAAI,IAAKL,OAAO,QAAQ,IAAI,EAAE;YACvC,IAAIU;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAAC1F;gBACjD0D,aAAa,MAAM,CAACgC,YAAY,KAAK;YACvC,EAAE,OAAOtG,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzBsE,cACA,CAAC,4CAA4C,EAAEtE,MAAM,SAAS,EAAEuB,KAAK,SAAS,CAC5EX,QACC;YAEP;YAEA,MAAM0D,aAAa,KAAK;YAExB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ5E;gBACR,UAAU4E;YACZ;YAGF,IAAI1D,AAAkB,eAAlBA,KAAK,CAAC,EAAE,CAAC,IAAI,EACf;QAEJ;QACA,OAAO;YACL,QAAQ;gBACNqF;YACF;YACA,UAAU3B;QACZ;IACF;IAEQ,oBACNoC,IAA0D,EAC1DC,MAA2B,EAC3BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBAEL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO7H,OAAOqC;gBACtB,MAAM,EAAEZ,IAAI,EAAE,GAAGY;gBACjB,IAAIK;gBACJ,MAAME,gBAAgC,CAACC;oBACrCH,cAAcG;gBAChB;gBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGD;gBAGjC,MAAMG,WAAWxC,KAAK,GAAG;gBACzB,MAAMC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;gBACxDiB,KAAK,SAAS,GAAGjB;gBAEjB,MAAMwC,aAAoC;oBACxC,MAAM;oBACN,IAAID;oBACJ,YAAYvC,UAAU,gBAAgB;oBACtC,QAAQ;gBACV;gBACAiB,KAAK,QAAQ,GAAG;oBAACuB;iBAAW;gBAE5B,MAAMiF,mBAAmBL,AAAS,YAATA;gBACzB,IAAIM,cAAcL;gBAClB,IAAII,kBAAkB;oBACpB,MAAME,aAAaP,AAAS,aAATA,OAAoB,YAAYA;oBACnDM,cAAc;wBACZ,QAAQ,GAAGC,WAAW,EAAE,EAAEN,QAAQ;oBACpC;gBACF;gBAEA,MAAM,EAAEO,IAAI,EAAEzF,KAAK,EAAE0F,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACzDH,aACAJ,KACAC;gBAGF,IAAIO,eAAeF;gBACnB,IAAIH,kBAAkB;oBACpBzF,OAAO4F,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,MAAMxH,QAAW;oBACnC0H,eAAgBF,KAAa,MAAM;gBACrC;gBAEA,OAAO;oBACL,QAAQE;oBACR,KAAK;wBAAE,MAAM5F;wBAAa,iBAAiBoF,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe;oBAAC;oBAChEnF;oBACA0F;gBACF;YACF;QACF;QAEA,OAAOL;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvC,eAAe,IAAIC,SACvBC,aACEkC,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpF,KAAK,SAAS,CAACoF,UAEvD;YACE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAGF,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAC,KACAC;QAGF,MAAMvC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACwC;QAC7D,MAAM/G,SAAS,MAAMuE,aAAa,KAAK;QAEvC,IAAI,CAACvE,QACH,MAAM,IAAI8C,MACR;QAIJ,MAAM,EAAE+C,MAAM,EAAEuB,OAAO,EAAE,GAAGpH;QAE5B,OAAO;YACL6F;YACAuB;YACA,UAAU7C;QACZ;IACF;IAEA,MAAM,OACJ+C,SAAsB,EACtBT,GAA0B,EACS;QACnC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,YAAYF;QACrD,OAAO,MAAM,IAAI,CAAC,wBAAwB,CACxC,UACAC,YACAV,KACAC;IAEJ;IAWQ,0BACNW,mBAA+C,EAC/C;QACA,IAAIA,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;YAEvC,MAAMC,eAAe,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAClD,CAACrI,OAASA,AAAc,WAAdA,KAAK,IAAI;YAIrB,IAAIqI,aAAa,MAAM,IAAI,KAAKD,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;gBAEnE,MAAME,oBAAoB,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAC1D,CAACtI,OAASA,AAAc,WAAdA,KAAK,IAAI;gBAErB,IAAIsI,qBAAqB,GACvB,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAACA,mBAAmB;YAEvD;QACF;QAEA,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACF;IAChC;IAEA,MAAc,gBAAgBlD,YAAsB,EAAE+B,QAAgB,EAAE;QACtE,MAAMsB,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACL,SAAStB;YACX;YACA,QAAQ;QACV;QACA,MAAM,EAAExF,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC;YAAC8G;SAAU;QAChE,MAAMrD,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACzD,KAAK,CAAC,EAAE;QACrE,MAAMyD,aAAa,KAAK;QAExB,OAAO;YACL,QAAQ5E;YACR,UAAU4E;QACZ;IACF;IAEA,MAAM,QACJ+C,SAAsB,EACtBT,GAA+B,EACC;QAChC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,YAAYF;QAErD,MAAMO,cAAc,CAAC,SAAS,EAAEN,YAAY;QAC5C,MAAMhD,eAAe,IAAIC,SAASC,aAAa,WAAWoD,cAAc;YACtE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE,GAAGlB;QAEvCtF,OAAO+F,WAAW;QAClB/F,OAAOuG,WAAW;QAClBvG,OAAOwG,iBAAiB;QAExBxG,OACEwG,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmB1I,KAAK,GAAG;QACjC,IAAIsF,YAAYtF,KAAK,GAAG;QACxB,IAAI2I,eAAe;QACnB,MAAO3I,KAAK,GAAG,KAAK0I,mBAAmBF,UAAW;YAChDlD,YAAYtF,KAAK,GAAG;YACpB,MAAMyH,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,UACAQ,YACA;gBACE,iBAAiB;gBACjB,eAAe;gBACf,iBAAiB;YACnB,GACAT;YAGF,MAAMvC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACwC;YAC7D,MAAM/G,SAAU,MAAMuE,aAAa,KAAK;YAKxC,IAAI,CAACvE,QACH,MAAM,IAAI8C,MACR;YAIJ,IAAI9C,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQL;gBACR,UAAU4E;YACZ;YAGF0D,eACEjI,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACd,CAAC,0CAA0C,EAAEuH,YAAY;YAC3D,MAAMW,MAAM5I,KAAK,GAAG;YACpB,IAAI4I,MAAMtD,YAAYmD,iBAAiB;gBACrC,MAAM1C,gBAAgB0C,kBAAmBG,CAAAA,MAAMtD,SAAQ;gBACvD,MAAMuD,YAAsD;oBAC1D,MAAM;oBACN,OAAO;wBACL,QAAQ9C;oBACV;oBACA,QAAQ;gBACV;gBACA,MAAM,EAAE,OAAO+C,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC;oBAC/DD;iBACD;gBACD,MAAM5D,aAAa,MAAM,CACvB,IAAI,CAAC,6BAA6B,CAAC6D,UAAU,CAAC,EAAE;gBAElD,MAAM7D,aAAa,KAAK;YAC1B;QACF;QAEA,OAAO,IAAI,CAAC,eAAe,CACzBA,cACA,CAAC,iBAAiB,EAAE0D,cAAc;IAEtC;IAlpCA,YACEI,iBAAoC,EACpCC,OAAgB,EAChBC,IAGC,CACD;QAtBF;QAEA;QAEA;QAEA,8CAAoD,EAAE;QAEtD;QAeE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;IAC9C;AAuoCF"}

package/dist/es/agent/utils.mjs

@@ -8,19 +8,20 @@ import { assert, logMsg, uuid } from "@midscene/shared/utils";
 import dayjs from "dayjs";
 import { debug } from "./task-cache.mjs";
 const debugProfile = getDebug('web:tool:profile');
-async function commonContextParser(page) {
-    assert(page, 'page is required');
-    debugProfile('Getting page URL');
-    const url = await page.url();
-    debugProfile('URL end');
+async function commonContextParser(interfaceInstance) {
+    var _interfaceInstance_describe;
+    assert(interfaceInstance, 'interfaceInstance is required');
+    debugProfile("Getting interface description");
+    const description = (null == (_interfaceInstance_describe = interfaceInstance.describe) ? void 0 : _interfaceInstance_describe.call(interfaceInstance)) || '';
+    debugProfile("Interface description end");
     debugProfile('Uploading test info to server');
     uploadTestInfoToServer({
-        testUrl: url
+        testUrl: description
     });
     debugProfile('UploadTestInfoToServer end');
-    let screenshotBase64 = await page.screenshotBase64();
+    let screenshotBase64 = await interfaceInstance.screenshotBase64();
     assert(screenshotBase64, 'screenshotBase64 is required');
-    const size = await page.size();
+    const size = await interfaceInstance.size();
     debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);
     if (size.dpr && size.dpr > 1) {
         debugProfile('Resizing screenshot for high DPR display');
@@ -36,8 +37,7 @@ async function commonContextParser(page) {
             children: []
         },
         size,
-        screenshotBase64: screenshotBase64,
-        url
+        screenshotBase64: screenshotBase64
     };
 }
 function getReportFileName(tag = 'web') {
@@ -94,8 +94,8 @@ function matchElementFromPlan(planLocateParam, tree) {
 async function matchElementFromCache(taskExecutor, xpaths, cachePrompt, cacheable) {
     try {
         var _taskExecutor_taskCache;
-        if ((null == xpaths ? void 0 : xpaths.length) && (null == (_taskExecutor_taskCache = taskExecutor.taskCache) ? void 0 : _taskExecutor_taskCache.isCacheResultUsed) && false !== cacheable && taskExecutor.page.getElementInfoByXpath) for(let i = 0; i < xpaths.length; i++){
-            const element = await taskExecutor.page.getElementInfoByXpath(xpaths[i]);
+        if ((null == xpaths ? void 0 : xpaths.length) && (null == (_taskExecutor_taskCache = taskExecutor.taskCache) ? void 0 : _taskExecutor_taskCache.isCacheResultUsed) && false !== cacheable && taskExecutor.interface.getElementInfoByXpath) for(let i = 0; i < xpaths.length; i++){
+            const element = await taskExecutor.interface.getElementInfoByXpath(xpaths[i]);
             if (null == element ? void 0 : element.id) {
                 debug('cache hit, prompt: %s', cachePrompt);
                 debug('found a new element with same xpath, xpath: %s, id: %s', xpaths[i], null == element ? void 0 : element.id);
@@ -123,13 +123,13 @@ function trimContextByViewport(execution) {
     return {
         ...execution,
         tasks: Array.isArray(execution.tasks) ? execution.tasks.map((task)=>{
-            var _task_pageContext;
+            var _task_uiContext;
             const newTask = {
                 ...task
             };
-            if (null == (_task_pageContext = task.pageContext) ? void 0 : _task_pageContext.tree) newTask.pageContext = {
-                ...task.pageContext,
-                tree: filterVisibleTree(task.pageContext.tree) || {
+            if (null == (_task_uiContext = task.uiContext) ? void 0 : _task_uiContext.tree) newTask.uiContext = {
+                ...task.uiContext,
+                tree: filterVisibleTree(task.uiContext.tree) || {
                     node: null,
                     children: []
                 }
@@ -138,7 +138,7 @@ function trimContextByViewport(execution) {
         }) : execution.tasks
     };
 }
-const getMidsceneVersion = ()=>"0.27.4";
+const getMidsceneVersion = ()=>"0.27.5-beta-20250827031507.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.mjs","sources":["webpack://@midscene/core/./src/agent/utils.ts"],"sourcesContent":["import { elementByPositionWithElementInfo } from '@/ai-model';\nimport type { AbstractDevice } from '@/device';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  ExecutionDump,\n  ExecutionTask,\n  ExecutorContext,\n  PlanningLocateParam,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/index';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n} from '@midscene/shared/extractor';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport { debug as cacheDebug } from './task-cache';\nimport type { PageTaskExecutor } from './tasks';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  page: AbstractDevice,\n): Promise<UIContext> {\n  assert(page, 'page is required');\n\n  debugProfile('Getting page URL');\n  const url = await page.url();\n  debugProfile('URL end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({ testUrl: url });\n  debugProfile('UploadTestInfoToServer end');\n\n  let screenshotBase64 = await page.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await page.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  if (size.dpr && size.dpr > 1) {\n    debugProfile('Resizing screenshot for high DPR display');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    debugProfile('ResizeImgBase64 end');\n  }\n\n  return {\n    tree: {\n      node: null,\n      children: [],\n    },\n    size,\n    screenshotBase64: screenshotBase64!,\n    url,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  taskExecutor: PageTaskExecutor,\n  xpaths: string[] | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n) {\n  try {\n    if (\n      xpaths?.length &&\n      taskExecutor.taskCache?.isCacheResultUsed &&\n      cacheable !== false &&\n      (taskExecutor.page as any).getElementInfoByXpath\n    ) {\n      // hit cache, use new id\n      for (let i = 0; i < xpaths.length; i++) {\n        const element = await (taskExecutor.page as any).getElementInfoByXpath(\n          xpaths[i],\n        );\n\n        if (element?.id) {\n          cacheDebug('cache hit, prompt: %s', cachePrompt);\n          cacheDebug(\n            'found a new element with same xpath, xpath: %s, id: %s',\n            xpaths[i],\n            element?.id,\n          );\n          return element;\n        }\n      }\n    }\n  } catch (error) {\n    cacheDebug('get element info by xpath error: ', error);\n  }\n}\n\nexport function trimContextByViewport(execution: ExecutionDump) {\n  function filterVisibleTree(\n    node: ElementTreeNode<BaseElement>,\n  ): ElementTreeNode<BaseElement> | null {\n    if (!node) return null;\n\n    // recursively process all children\n    const filteredChildren = Array.isArray(node.children)\n      ? (node.children\n          .map(filterVisibleTree)\n          .filter((child) => child !== null) as ElementTreeNode<BaseElement>[])\n      : [];\n\n    // if the current node is visible, keep it and the filtered children\n    if (node.node && node.node.isVisible === true) {\n      return {\n        ...node,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible, but has visible children, create an empty node to include these children\n    if (filteredChildren.length > 0) {\n      return {\n        node: null,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible and has no visible children, return null\n    return null;\n  }\n\n  return {\n    ...execution,\n    tasks: Array.isArray(execution.tasks)\n      ? execution.tasks.map((task: ExecutionTask) => {\n          const newTask = { ...task };\n          if (task.pageContext?.tree) {\n            newTask.pageContext = {\n              ...task.pageContext,\n              tree: filterVisibleTree(task.pageContext.tree) || {\n                node: null,\n                children: [],\n              },\n            };\n          }\n          return newTask;\n        })\n      : execution.tasks,\n  };\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["debugProfile","getDebug","commonContextParser","page","assert","url","uploadTestInfoToServer","screenshotBase64","size","resizeImgBase64","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","matchElementFromPlan","planLocateParam","tree","getNodeFromCacheList","centerPosition","Math","element","elementByPositionWithElementInfo","generateElementByPosition","matchElementFromCache","taskExecutor","xpaths","cachePrompt","cacheable","_taskExecutor_taskCache","i","cacheDebug","trimContextByViewport","execution","filterVisibleTree","node","filteredChildren","Array","child","task","_task_pageContext","newTask","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;;;;;;;AA8BA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,oBACpBC,IAAoB;IAEpBC,OAAOD,MAAM;IAEbH,aAAa;IACb,MAAMK,MAAM,MAAMF,KAAK,GAAG;IAC1BH,aAAa;IAEbA,aAAa;IACbM,uBAAuB;QAAE,SAASD;IAAI;IACtCL,aAAa;IAEb,IAAIO,mBAAmB,MAAMJ,KAAK,gBAAgB;IAClDC,OAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAML,KAAK,IAAI;IAC5BH,aAAa,CAAC,MAAM,EAAEQ,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,IAAIA,KAAK,GAAG,IAAIA,KAAK,GAAG,GAAG,GAAG;QAC5BR,aAAa;QACbO,mBAAmB,MAAME,gBAAgBF,kBAAkB;YACzD,OAAOC,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACAR,aAAa;IACf;IAEA,OAAO;QACL,MAAM;YACJ,MAAM;YACN,UAAU,EAAE;QACd;QACAQ;QACA,kBAAkBD;QAClBF;IACF;AACF;AAEO,SAASK,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,OAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,QAAAA,QAAAA,KAAAA,IAAAA,KAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,qBACdC,eAAoC,EACpCC,IAAkC;IAElC,IAAI,CAACD,iBACH;IAEF,IAAIA,gBAAgB,EAAE,EACpB,OAAOE,qBAAqBF,gBAAgB,EAAE;IAGhD,IAAIA,gBAAgB,IAAI,EAAE;QACxB,MAAMG,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGI,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QACA,IAAIK,UAAUC,iCAAiCL,MAAME;QAErD,IAAI,CAACE,SACHA,UAAUE,0BAA0BJ;QAGtC,OAAOE;IACT;AAGF;AAEO,eAAeG,sBACpBC,YAA8B,EAC9BC,MAA4B,EAC5BC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI;YAGAC;QAFF,IACEH,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,KAAC,SACdG,CAAAA,0BAAAA,aAAa,SAAS,AAAD,IAArBA,KAAAA,IAAAA,wBAAwB,iBAAiB,AAAD,KACxCD,AAAc,UAAdA,aACCH,aAAa,IAAI,CAAS,qBAAqB,EAGhD,IAAK,IAAIK,IAAI,GAAGA,IAAIJ,OAAO,MAAM,EAAEI,IAAK;YACtC,MAAMT,UAAU,MAAOI,aAAa,IAAI,CAAS,qBAAqB,CACpEC,MAAM,CAACI,EAAE;YAGX,IAAIT,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE,EAAE;gBACfU,MAAW,yBAAyBJ;gBACpCI,MACE,0DACAL,MAAM,CAACI,EAAE,EACTT,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE;gBAEb,OAAOA;YACT;QACF;IAEJ,EAAE,OAAOvB,OAAO;QACdiC,MAAW,qCAAqCjC;IAClD;AACF;AAEO,SAASkC,sBAAsBC,SAAwB;IAC5D,SAASC,kBACPC,IAAkC;QAElC,IAAI,CAACA,MAAM,OAAO;QAGlB,MAAMC,mBAAmBC,MAAM,OAAO,CAACF,KAAK,QAAQ,IAC/CA,KAAK,QAAQ,CACX,GAAG,CAACD,mBACJ,MAAM,CAAC,CAACI,QAAUA,AAAU,SAAVA,SACrB,EAAE;QAGN,IAAIH,KAAK,IAAI,IAAIA,AAAwB,SAAxBA,KAAK,IAAI,CAAC,SAAS,EAClC,OAAO;YACL,GAAGA,IAAI;YACP,UAAUC;QACZ;QAIF,IAAIA,iBAAiB,MAAM,GAAG,GAC5B,OAAO;YACL,MAAM;YACN,UAAUA;QACZ;QAIF,OAAO;IACT;IAEA,OAAO;QACL,GAAGH,SAAS;QACZ,OAAOI,MAAM,OAAO,CAACJ,UAAU,KAAK,IAChCA,UAAU,KAAK,CAAC,GAAG,CAAC,CAACM;gBAEfC;YADJ,MAAMC,UAAU;gBAAE,GAAGF,IAAI;YAAC;YAC1B,IAAI,QAAAC,CAAAA,oBAAAA,KAAK,WAAW,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,IAAI,EACxBC,QAAQ,WAAW,GAAG;gBACpB,GAAGF,KAAK,WAAW;gBACnB,MAAML,kBAAkBK,KAAK,WAAW,CAAC,IAAI,KAAK;oBAChD,MAAM;oBACN,UAAU,EAAE;gBACd;YACF;YAEF,OAAOE;QACT,KACAR,UAAU,KAAK;IACrB;AACF;AAIO,MAAMS,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkB/B;IACpB;IAEF,OAAO;QACL,YAAY+B,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACA/B;IACN;AACF"}
+{"version":3,"file":"agent/utils.mjs","sources":["webpack://@midscene/core/./src/agent/utils.ts"],"sourcesContent":["import { elementByPositionWithElementInfo } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  ExecutionDump,\n  ExecutionTask,\n  ExecutorContext,\n  PlanningLocateParam,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/index';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n} from '@midscene/shared/extractor';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport { debug as cacheDebug } from './task-cache';\nimport type { TaskExecutor } from './tasks';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n): Promise<UIContext> {\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debugProfile('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debugProfile('Interface description end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({ testUrl: description });\n  debugProfile('UploadTestInfoToServer end');\n\n  let screenshotBase64 = await interfaceInstance.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await interfaceInstance.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  if (size.dpr && size.dpr > 1) {\n    debugProfile('Resizing screenshot for high DPR display');\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\n      width: size.width,\n      height: size.height,\n    });\n    debugProfile('ResizeImgBase64 end');\n  }\n\n  return {\n    tree: {\n      node: null,\n      children: [],\n    },\n    size,\n    screenshotBase64: screenshotBase64!,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  taskExecutor: TaskExecutor,\n  xpaths: string[] | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n) {\n  try {\n    if (\n      xpaths?.length &&\n      taskExecutor.taskCache?.isCacheResultUsed &&\n      cacheable !== false &&\n      (taskExecutor.interface as any).getElementInfoByXpath\n    ) {\n      // hit cache, use new id\n      for (let i = 0; i < xpaths.length; i++) {\n        const element = await (\n          taskExecutor.interface as any\n        ).getElementInfoByXpath(xpaths[i]);\n\n        if (element?.id) {\n          cacheDebug('cache hit, prompt: %s', cachePrompt);\n          cacheDebug(\n            'found a new element with same xpath, xpath: %s, id: %s',\n            xpaths[i],\n            element?.id,\n          );\n          return element;\n        }\n      }\n    }\n  } catch (error) {\n    cacheDebug('get element info by xpath error: ', error);\n  }\n}\n\nexport function trimContextByViewport(execution: ExecutionDump) {\n  function filterVisibleTree(\n    node: ElementTreeNode<BaseElement>,\n  ): ElementTreeNode<BaseElement> | null {\n    if (!node) return null;\n\n    // recursively process all children\n    const filteredChildren = Array.isArray(node.children)\n      ? (node.children\n          .map(filterVisibleTree)\n          .filter((child) => child !== null) as ElementTreeNode<BaseElement>[])\n      : [];\n\n    // if the current node is visible, keep it and the filtered children\n    if (node.node && node.node.isVisible === true) {\n      return {\n        ...node,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible, but has visible children, create an empty node to include these children\n    if (filteredChildren.length > 0) {\n      return {\n        node: null,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible and has no visible children, return null\n    return null;\n  }\n\n  return {\n    ...execution,\n    tasks: Array.isArray(execution.tasks)\n      ? execution.tasks.map((task: ExecutionTask) => {\n          const newTask = { ...task };\n          if (task.uiContext?.tree) {\n            newTask.uiContext = {\n              ...task.uiContext,\n              tree: filterVisibleTree(task.uiContext.tree) || {\n                node: null,\n                children: [],\n              },\n            };\n          }\n          return newTask;\n        })\n      : execution.tasks,\n  };\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["debugProfile","getDebug","commonContextParser","interfaceInstance","assert","description","uploadTestInfoToServer","screenshotBase64","size","resizeImgBase64","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","matchElementFromPlan","planLocateParam","tree","getNodeFromCacheList","centerPosition","Math","element","elementByPositionWithElementInfo","generateElementByPosition","matchElementFromCache","taskExecutor","xpaths","cachePrompt","cacheable","_taskExecutor_taskCache","i","cacheDebug","trimContextByViewport","execution","filterVisibleTree","node","filteredChildren","Array","child","task","_task_uiContext","newTask","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;;;;;;;AA8BA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,oBACpBC,iBAAoC;QAKhBA;IAHpBC,OAAOD,mBAAmB;IAE1BH,aAAa;IACb,MAAMK,cAAcF,AAAAA,SAAAA,CAAAA,8BAAAA,kBAAkB,QAAQ,AAAD,IAAzBA,KAAAA,IAAAA,4BAAAA,IAAAA,CAAAA,kBAAiB,KAAiB;IACtDH,aAAa;IAEbA,aAAa;IACbM,uBAAuB;QAAE,SAASD;IAAY;IAC9CL,aAAa;IAEb,IAAIO,mBAAmB,MAAMJ,kBAAkB,gBAAgB;IAC/DC,OAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAML,kBAAkB,IAAI;IACzCH,aAAa,CAAC,MAAM,EAAEQ,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,IAAIA,KAAK,GAAG,IAAIA,KAAK,GAAG,GAAG,GAAG;QAC5BR,aAAa;QACbO,mBAAmB,MAAME,gBAAgBF,kBAAkB;YACzD,OAAOC,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACAR,aAAa;IACf;IAEA,OAAO;QACL,MAAM;YACJ,MAAM;YACN,UAAU,EAAE;QACd;QACAQ;QACA,kBAAkBD;IACpB;AACF;AAEO,SAASG,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,OAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,QAAAA,QAAAA,KAAAA,IAAAA,KAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,qBACdC,eAAoC,EACpCC,IAAkC;IAElC,IAAI,CAACD,iBACH;IAEF,IAAIA,gBAAgB,EAAE,EACpB,OAAOE,qBAAqBF,gBAAgB,EAAE;IAGhD,IAAIA,gBAAgB,IAAI,EAAE;QACxB,MAAMG,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGI,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QACA,IAAIK,UAAUC,iCAAiCL,MAAME;QAErD,IAAI,CAACE,SACHA,UAAUE,0BAA0BJ;QAGtC,OAAOE;IACT;AAGF;AAEO,eAAeG,sBACpBC,YAA0B,EAC1BC,MAA4B,EAC5BC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI;YAGAC;QAFF,IACEH,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,KAAC,SACdG,CAAAA,0BAAAA,aAAa,SAAS,AAAD,IAArBA,KAAAA,IAAAA,wBAAwB,iBAAiB,AAAD,KACxCD,AAAc,UAAdA,aACCH,aAAa,SAAS,CAAS,qBAAqB,EAGrD,IAAK,IAAIK,IAAI,GAAGA,IAAIJ,OAAO,MAAM,EAAEI,IAAK;YACtC,MAAMT,UAAU,MACdI,aAAa,SAAS,CACtB,qBAAqB,CAACC,MAAM,CAACI,EAAE;YAEjC,IAAIT,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE,EAAE;gBACfU,MAAW,yBAAyBJ;gBACpCI,MACE,0DACAL,MAAM,CAACI,EAAE,EACTT,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE;gBAEb,OAAOA;YACT;QACF;IAEJ,EAAE,OAAOvB,OAAO;QACdiC,MAAW,qCAAqCjC;IAClD;AACF;AAEO,SAASkC,sBAAsBC,SAAwB;IAC5D,SAASC,kBACPC,IAAkC;QAElC,IAAI,CAACA,MAAM,OAAO;QAGlB,MAAMC,mBAAmBC,MAAM,OAAO,CAACF,KAAK,QAAQ,IAC/CA,KAAK,QAAQ,CACX,GAAG,CAACD,mBACJ,MAAM,CAAC,CAACI,QAAUA,AAAU,SAAVA,SACrB,EAAE;QAGN,IAAIH,KAAK,IAAI,IAAIA,AAAwB,SAAxBA,KAAK,IAAI,CAAC,SAAS,EAClC,OAAO;YACL,GAAGA,IAAI;YACP,UAAUC;QACZ;QAIF,IAAIA,iBAAiB,MAAM,GAAG,GAC5B,OAAO;YACL,MAAM;YACN,UAAUA;QACZ;QAIF,OAAO;IACT;IAEA,OAAO;QACL,GAAGH,SAAS;QACZ,OAAOI,MAAM,OAAO,CAACJ,UAAU,KAAK,IAChCA,UAAU,KAAK,CAAC,GAAG,CAAC,CAACM;gBAEfC;YADJ,MAAMC,UAAU;gBAAE,GAAGF,IAAI;YAAC;YAC1B,IAAI,QAAAC,CAAAA,kBAAAA,KAAK,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,IAAI,EACtBC,QAAQ,SAAS,GAAG;gBAClB,GAAGF,KAAK,SAAS;gBACjB,MAAML,kBAAkBK,KAAK,SAAS,CAAC,IAAI,KAAK;oBAC9C,MAAM;oBACN,UAAU,EAAE;gBACd;YACF;YAEF,OAAOE;QACT,KACAR,UAAU,KAAK;IACrB;AACF;AAIO,MAAMS,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkB/B;IACpB;IAEF,OAAO;QACL,YAAY+B,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACA/B;IACN;AACF"}

package/dist/es/ai-model/common.mjs

@@ -124,7 +124,7 @@ function warnGPT4oSizeLimit(size, modelPreferences) {
     var _getModelName;
     if (warned) return;
     if (null == (_getModelName = getModelName(modelPreferences)) ? void 0 : _getModelName.toLowerCase().includes('gpt-4o')) {
-        const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;
+        const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;
         if (Math.max(size.width, size.height) > 2000 || Math.min(size.width, size.height) > 768) {
             console.warn(warningMsg);
             warned = true;

package/dist/es/ai-model/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport { callToGetJSONObject } from './service-caller/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelPreferences,\n  getModelName,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nexport async function callAiFn<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const jsonObject = await callToGetJSONObject<T>(\n    msgs,\n    AIActionTypeValue,\n    modelPreferences,\n  );\n\n  return {\n    content: jsonObject.content,\n    usage: jsonObject.usage,\n  };\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height, modelPreferences);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n): [number, number, number, number] {\n  if (\n    vlLocateMode(modelPreferences) === 'doubao-vision' ||\n    vlLocateMode(modelPreferences) === 'vlm-ui-tars'\n  ) {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlLocateMode(modelPreferences) === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n  offsetX = 0,\n  offsetY = 0,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    modelPreferences,\n  );\n  const rect = {\n    left: left + offsetX,\n    top: top + offsetY,\n    width: right - left,\n    height: bottom - top,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(\n  size: Size,\n  modelPreferences: IModelPreferences,\n) {\n  if (warned) return;\n  if (getModelName(modelPreferences)?.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  modelPreferences: IModelPreferences,\n) {\n  const minEdgeSize =\n    vlLocateMode(modelPreferences) === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n  rect.left = Math.max(0, rect.left - paddingSizeHorizontal);\n  rect.width = Math.min(\n    rect.width + paddingSizeHorizontal * 2,\n    screenSize.width - rect.left,\n  );\n  rect.top = Math.max(0, rect.top - paddingSizeVertical);\n  rect.height = Math.min(\n    rect.height + paddingSizeVertical * 2,\n    screenSize.height - rect.top,\n  );\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationResult;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodUnion (the new MidsceneLocation structure)\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n    return locateFieldFlagName in shape;\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n"],"names":["AIActionType","callAiFn","msgs","AIActionTypeValue","modelPreferences","jsonObject","callToGetJSONObject","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","vlLocateMode","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rect","warned","warnGPT4oSizeLimit","size","_getModelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationResult","getMidsceneLocationSchema","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","Object","key","_field__def","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam"],"mappings":";;;;;;;;AAmCO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQL,eAAeC,SACpBC,IAAY,EACZC,iBAA+B,EAC/BC,gBAAmC;IAEnC,MAAMC,aAAa,MAAMC,oBACvBJ,MACAC,mBACAC;IAGF,OAAO;QACL,SAASC,WAAW,OAAO;QAC3B,OAAOA,WAAW,KAAK;IACzB;AACF;AAEA,MAAME,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAGnC,IAAKO,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGG,UAAUH,OAAO,IAAI,EAAEC,OAAOC,QAAQT;IAGtD,OAAOO;AACT;AAEO,SAASI,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;QACN,YAAnB,OAAOS,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;KAC1B;IACD,OAAOa;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCJ,KAAa,EACbC,MAAc;IAEdU,OACEX,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOG,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;YAC5CQ,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;SAC7C;QAEH,MAAM,IAAIM,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAIH,IACEc,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;QAEhEc,KAAK,GAAG,CACNT,OACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACNR,QACAQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIS,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAGH,MAAMI,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAEnC,IACE2B,AAAmC,oBAAnCA,aAAa3B,qBACb2B,AAAmC,kBAAnCA,aAAa3B,mBAEb,OAAOkB,gBAAgBN,MAAMJ,OAAOC;IAGtC,IAAIkB,AAAmC,aAAnCA,aAAa3B,mBACf,OAAO4B,gBAAgBhB,MAAMJ,OAAOC;IAGtC,OAAOE,cAAcC;AACvB;AAEO,SAASgB,gBACdhB,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC5C,MAAMsB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC5C,MAAMsB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC7C,MAAMwB,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC,EACnCkC,UAAU,CAAC,EACXC,UAAU,CAAC;IAEX/B,kBAAkB,mBAAmBQ,MAAMJ,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UACjCE,MACAJ,OACAC,QACAT;IAEF,MAAMoC,OAAO;QACX,MAAMP,OAAOK;QACb,KAAKJ,MAAMK;QACX,OAAOJ,QAAQF;QACf,QAAQG,SAASF;IACnB;IACA1B,kBAAkB,4BAA4BgC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBACdC,IAAU,EACVvC,gBAAmC;QAG/BwC;IADJ,IAAIH,QAAQ;IACZ,IAAI,QAAAG,CAAAA,gBAAAA,aAAaxC,iBAAgB,IAA7BwC,KAAAA,IAAAA,cAAgC,WAAW,GAAG,QAAQ,CAAC,WAAW;QACpE,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,wFAAwF,CAAC;QAEhN,IACEtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChBnD,gBAAmC;IAEnC,MAAMoD,cACJzB,AAAmC,oBAAnCA,aAAa3B,oBAAwC,MAAM;IAC7D,MAAMqD,iBAAiB;IAEvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IACNjB,KAAK,IAAI,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,IAAI,GAAGkB;IACpClB,KAAK,KAAK,GAAGnB,KAAK,GAAG,CACnBmB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA,uBACbH,WAAW,KAAK,GAAGf,KAAK,IAAI;IAE9BA,KAAK,GAAG,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,GAAG,GAAGmB;IAClCnB,KAAK,MAAM,GAAGnB,KAAK,GAAG,CACpBmB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA,qBACdJ,WAAW,MAAM,GAAGf,KAAK,GAAG;IAE9B,OAAOA;AACT;AAEO,eAAeoB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCnB,IAAU;IAEV,MAAMoB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtBtB;IACF;IACA,OAAOyB;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACX/B,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAE8B,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAME,UAAUD,OAAO,cAAc,IAAID;QACzC,MAAMG,YAAYF,OAAO,WAAW,GAChCG,gBAAgBL,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMI,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAL,KAAK,IAAI,CAACO;IACZ;IAEA,IAAIR,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMQ,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAE5B,MAAMC,yBAAyBN,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAGP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QACpC,OAAOP,uBAAuBQ;IAChC;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvCrE,OACEoE,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMd,QAAQW,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTM;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAM3B,kBAAkB,CAC7B3E,YACAuG;IAEA,MAAMC,gBAAgBV,4BAA4BS;IAClD,MAAMxF,SAAS;QAAE,GAAGf,UAAU;IAAC;IAE/B,KAAK,MAAMyG,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT3F,MAAM,CAAC0F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO3F;AACT;AAEO,MAAM4F,kBAAkB,CAC7B3G,YACAuG;IAEA,MAAMC,gBAAgBV,4BAA4BS;IAClD,MAAMxF,SAAS;QAAE,GAAGf,UAAU;IAAC;IAE/B,KAAK,MAAMyG,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB3F,MAAM,CAAC0F,UAAU,GAAG;YAClB,CAACtB,oBAAoB,EAAE;YACvB,QAAQuB;QACV;IAEJ;IAEA,OAAO3F;AACT"}
+{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport { callToGetJSONObject } from './service-caller/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelPreferences,\n  getModelName,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nexport async function callAiFn<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const jsonObject = await callToGetJSONObject<T>(\n    msgs,\n    AIActionTypeValue,\n    modelPreferences,\n  );\n\n  return {\n    content: jsonObject.content,\n    usage: jsonObject.usage,\n  };\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height, modelPreferences);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n): [number, number, number, number] {\n  if (\n    vlLocateMode(modelPreferences) === 'doubao-vision' ||\n    vlLocateMode(modelPreferences) === 'vlm-ui-tars'\n  ) {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlLocateMode(modelPreferences) === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n  offsetX = 0,\n  offsetY = 0,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    modelPreferences,\n  );\n  const rect = {\n    left: left + offsetX,\n    top: top + offsetY,\n    width: right - left,\n    height: bottom - top,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(\n  size: Size,\n  modelPreferences: IModelPreferences,\n) {\n  if (warned) return;\n  if (getModelName(modelPreferences)?.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  modelPreferences: IModelPreferences,\n) {\n  const minEdgeSize =\n    vlLocateMode(modelPreferences) === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n  rect.left = Math.max(0, rect.left - paddingSizeHorizontal);\n  rect.width = Math.min(\n    rect.width + paddingSizeHorizontal * 2,\n    screenSize.width - rect.left,\n  );\n  rect.top = Math.max(0, rect.top - paddingSizeVertical);\n  rect.height = Math.min(\n    rect.height + paddingSizeVertical * 2,\n    screenSize.height - rect.top,\n  );\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationResult;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodUnion (the new MidsceneLocation structure)\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n    return locateFieldFlagName in shape;\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n"],"names":["AIActionType","callAiFn","msgs","AIActionTypeValue","modelPreferences","jsonObject","callToGetJSONObject","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","vlLocateMode","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rect","warned","warnGPT4oSizeLimit","size","_getModelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationResult","getMidsceneLocationSchema","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","Object","key","_field__def","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam"],"mappings":";;;;;;;;AAmCO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQL,eAAeC,SACpBC,IAAY,EACZC,iBAA+B,EAC/BC,gBAAmC;IAEnC,MAAMC,aAAa,MAAMC,oBACvBJ,MACAC,mBACAC;IAGF,OAAO;QACL,SAASC,WAAW,OAAO;QAC3B,OAAOA,WAAW,KAAK;IACzB;AACF;AAEA,MAAME,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAGnC,IAAKO,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGG,UAAUH,OAAO,IAAI,EAAEC,OAAOC,QAAQT;IAGtD,OAAOO;AACT;AAEO,SAASI,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;QACN,YAAnB,OAAOS,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;KAC1B;IACD,OAAOa;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCJ,KAAa,EACbC,MAAc;IAEdU,OACEX,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOG,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;YAC5CQ,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;SAC7C;QAEH,MAAM,IAAIM,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAIH,IACEc,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;QAEhEc,KAAK,GAAG,CACNT,OACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACNR,QACAQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIS,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAGH,MAAMI,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAEnC,IACE2B,AAAmC,oBAAnCA,aAAa3B,qBACb2B,AAAmC,kBAAnCA,aAAa3B,mBAEb,OAAOkB,gBAAgBN,MAAMJ,OAAOC;IAGtC,IAAIkB,AAAmC,aAAnCA,aAAa3B,mBACf,OAAO4B,gBAAgBhB,MAAMJ,OAAOC;IAGtC,OAAOE,cAAcC;AACvB;AAEO,SAASgB,gBACdhB,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC5C,MAAMsB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC5C,MAAMsB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC7C,MAAMwB,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC,EACnCkC,UAAU,CAAC,EACXC,UAAU,CAAC;IAEX/B,kBAAkB,mBAAmBQ,MAAMJ,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UACjCE,MACAJ,OACAC,QACAT;IAEF,MAAMoC,OAAO;QACX,MAAMP,OAAOK;QACb,KAAKJ,MAAMK;QACX,OAAOJ,QAAQF;QACf,QAAQG,SAASF;IACnB;IACA1B,kBAAkB,4BAA4BgC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBACdC,IAAU,EACVvC,gBAAmC;QAG/BwC;IADJ,IAAIH,QAAQ;IACZ,IAAI,QAAAG,CAAAA,gBAAAA,aAAaxC,iBAAgB,IAA7BwC,KAAAA,IAAAA,cAAgC,WAAW,GAAG,QAAQ,CAAC,WAAW;QACpE,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,6FAA6F,CAAC;QAErN,IACEtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChBnD,gBAAmC;IAEnC,MAAMoD,cACJzB,AAAmC,oBAAnCA,aAAa3B,oBAAwC,MAAM;IAC7D,MAAMqD,iBAAiB;IAEvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IACNjB,KAAK,IAAI,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,IAAI,GAAGkB;IACpClB,KAAK,KAAK,GAAGnB,KAAK,GAAG,CACnBmB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA,uBACbH,WAAW,KAAK,GAAGf,KAAK,IAAI;IAE9BA,KAAK,GAAG,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,GAAG,GAAGmB;IAClCnB,KAAK,MAAM,GAAGnB,KAAK,GAAG,CACpBmB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA,qBACdJ,WAAW,MAAM,GAAGf,KAAK,GAAG;IAE9B,OAAOA;AACT;AAEO,eAAeoB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCnB,IAAU;IAEV,MAAMoB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtBtB;IACF;IACA,OAAOyB;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACX/B,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAE8B,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAME,UAAUD,OAAO,cAAc,IAAID;QACzC,MAAMG,YAAYF,OAAO,WAAW,GAChCG,gBAAgBL,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMI,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAL,KAAK,IAAI,CAACO;IACZ;IAEA,IAAIR,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMQ,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAE5B,MAAMC,yBAAyBN,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAGP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QACpC,OAAOP,uBAAuBQ;IAChC;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvCrE,OACEoE,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMd,QAAQW,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTM;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAM3B,kBAAkB,CAC7B3E,YACAuG;IAEA,MAAMC,gBAAgBV,4BAA4BS;IAClD,MAAMxF,SAAS;QAAE,GAAGf,UAAU;IAAC;IAE/B,KAAK,MAAMyG,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT3F,MAAM,CAAC0F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO3F;AACT;AAEO,MAAM4F,kBAAkB,CAC7B3G,YACAuG;IAEA,MAAMC,gBAAgBV,4BAA4BS;IAClD,MAAMxF,SAAS;QAAE,GAAGf,UAAU;IAAC;IAE/B,KAAK,MAAMyG,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB3F,MAAM,CAAC0F,UAAU,GAAG;YAClB,CAACtB,oBAAoB,EAAE;YACvB,QAAQuB;QACV;IAEJ;IAEA,OAAO3F;AACT"}