@midscene/core 0.27.0 → 0.27.1-beta-20250822094725.0

package/dist/es/agent/tasks.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import {\n  type ChatCompletionMessageParam,\n  elementByPositionWithElementInfo,\n  findAllMidsceneLocatorField,\n  resizeImageForUiTars,\n  vlmPlanning,\n} from '@/ai-model';\nimport type { AbstractPage } from '@/device';\nimport {\n  type AIUsageInfo,\n  type BaseElement,\n  type DetailedLocateParam,\n  type DumpSubscriber,\n  type ExecutionRecorderItem,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskHitBy,\n  type ExecutionTaskInsightLocateApply,\n  type ExecutionTaskInsightQueryApply,\n  type ExecutionTaskPlanning,\n  type ExecutionTaskPlanningApply,\n  type ExecutionTaskProgressOptions,\n  Executor,\n  type ExecutorContext,\n  type Insight,\n  type InsightDump,\n  type InsightExtractOption,\n  type InsightExtractParam,\n  type LocateResultElement,\n  type MidsceneYamlFlowItem,\n  type PageType,\n  type PlanningAIResponse,\n  type PlanningAction,\n  type PlanningActionParamError,\n  type PlanningActionParamSleep,\n  type PlanningActionParamWaitFor,\n  type PlanningLocateParam,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  type UIContext,\n  plan,\n} from '@/index';\nimport { sleep } from '@/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelPreferences,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  getAIConfigInNumber,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { taskTitleStr } from './ui-utils';\nimport {\n  matchElementFromCache,\n  matchElementFromPlan,\n  parsePrompt,\n} from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  executor: Executor;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\nexport class PageTaskExecutor {\n  page: AbstractPage;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  conversationHistory: ChatCompletionMessageParam[] = [];\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    page: AbstractPage,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n    },\n  ) {\n    this.page = page;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n  }\n\n  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {\n    const base64 = await this.page.screenshotBase64();\n    const item: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot: base64,\n      timing,\n    };\n    return item;\n  }\n\n  private async getElementXpath(\n    pageContext: UIContext<BaseElement>,\n    element: LocateResultElement,\n  ): Promise<string[] | undefined> {\n    if (!(this.page as any).getXpathsByPoint) {\n      debug('getXpathsByPoint is not supported for this page');\n      return undefined;\n    }\n\n    let elementId = element?.id;\n    if (element?.isOrderSensitive !== undefined) {\n      const xpaths = await (this.page as any).getXpathsByPoint(\n        {\n          left: element.center[0],\n          top: element.center[1],\n        },\n        element?.isOrderSensitive,\n      );\n\n      return xpaths;\n    }\n\n    // find the nearest xpath for the element\n    if (element?.attributes?.nodeType === NodeType.POSITION) {\n      await this.insight.contextRetrieverFn('locate');\n      const info = elementByPositionWithElementInfo(\n        pageContext.tree,\n        {\n          x: element.center[0],\n          y: element.center[1],\n        },\n        {\n          requireStrictDistance: false,\n          filterPositionElements: true,\n        },\n      );\n      if (info?.id) {\n        elementId = info.id;\n      } else {\n        debug(\n          'no element id found for position node, will not update cache',\n          element,\n        );\n      }\n    }\n\n    if (!elementId) {\n      return undefined;\n    }\n    try {\n      const result = await (this.page as any).getXpathsById(elementId);\n      return result;\n    } catch (error) {\n      debug('getXpathsById error: ', error);\n    }\n  }\n\n  private prependExecutorWithScreenshot(\n    taskApply: ExecutionTaskApply,\n    appendAfterExecution = false,\n  ): ExecutionTaskApply {\n    const taskWithScreenshot: ExecutionTaskApply = {\n      ...taskApply,\n      executor: async (param, context, ...args) => {\n        const recorder: ExecutionRecorderItem[] = [];\n        const { task } = context;\n        // set the recorder before executor in case of error\n        task.recorder = recorder;\n        const shot = await this.recordScreenshot(`before ${task.type}`);\n        recorder.push(shot);\n        const result = await taskApply.executor(param, context, ...args);\n        if (taskApply.type === 'Action') {\n          await Promise.all([\n            (async () => {\n              await sleep(100);\n              if (this.page.beforeAction) {\n                debug('will call \"beforeAction\" for page');\n                await this.page.beforeAction();\n              }\n            })(),\n            sleep(200),\n          ]);\n        }\n        if (appendAfterExecution) {\n          const shot2 = await this.recordScreenshot('after Action');\n          recorder.push(shot2);\n        }\n        return result;\n      },\n    };\n    return taskWithScreenshot;\n  }\n\n  public async convertPlanToExecutable(plans: PlanningAction[]) {\n    const tasks: ExecutionTaskApply[] = [];\n\n    const taskForLocatePlan = (\n      plan: PlanningAction<PlanningLocateParam>,\n      detailedLocateParam: DetailedLocateParam | string,\n      onResult?: (result: LocateResultElement) => void,\n    ): ExecutionTaskInsightLocateApply => {\n      if (typeof detailedLocateParam === 'string') {\n        detailedLocateParam = {\n          prompt: detailedLocateParam,\n        };\n      }\n      const taskFind: ExecutionTaskInsightLocateApply = {\n        type: 'Insight',\n        subType: 'Locate',\n        param: detailedLocateParam,\n        thought: plan.thought,\n        executor: async (param, taskContext) => {\n          const { task } = taskContext;\n          assert(\n            param?.prompt || param?.id || param?.bbox,\n            `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n              param,\n            )}`,\n          );\n          let insightDump: InsightDump | undefined;\n          let usage: AIUsageInfo | undefined;\n          const dumpCollector: DumpSubscriber = (dump) => {\n            insightDump = dump;\n            usage = dump?.taskInfo?.usage;\n\n            task.log = {\n              dump: insightDump,\n            };\n\n            task.usage = usage;\n          };\n          this.insight.onceDumpUpdatedFn = dumpCollector;\n          const shotTime = Date.now();\n\n          // Get context through contextRetrieverFn which handles frozen context\n          const pageContext = await this.insight.contextRetrieverFn('locate');\n          task.pageContext = pageContext;\n\n          const recordItem: ExecutionRecorderItem = {\n            type: 'screenshot',\n            ts: shotTime,\n            screenshot: pageContext.screenshotBase64,\n            timing: 'before Insight',\n          };\n          task.recorder = [recordItem];\n\n          // try matching xpath\n          const elementFromXpath =\n            param.xpath && (this.page as any).getElementInfoByXpath\n              ? await (this.page as any).getElementInfoByXpath(param.xpath)\n              : undefined;\n          const userExpectedPathHitFlag = !!elementFromXpath;\n\n          // try matching cache\n          const cachePrompt = param.prompt;\n          const locateCacheRecord =\n            this.taskCache?.matchLocateCache(cachePrompt);\n          const xpaths = locateCacheRecord?.cacheContent?.xpaths;\n          const elementFromCache = userExpectedPathHitFlag\n            ? null\n            : await matchElementFromCache(\n                this,\n                xpaths,\n                cachePrompt,\n                param.cacheable,\n              );\n          const cacheHitFlag = !!elementFromCache;\n\n          // try matching plan\n          const elementFromPlan =\n            !userExpectedPathHitFlag && !cacheHitFlag\n              ? matchElementFromPlan(param, pageContext.tree)\n              : undefined;\n          const planHitFlag = !!elementFromPlan;\n\n          // try ai locate\n          const elementFromAiLocate =\n            !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag\n              ? (\n                  await this.insight.locate(param, {\n                    // fallback to ai locate\n                    context: pageContext,\n                  })\n                ).element\n              : undefined;\n          const aiLocateHitFlag = !!elementFromAiLocate;\n\n          const element =\n            elementFromXpath || // highest priority\n            elementFromCache || // second priority\n            elementFromPlan || // third priority\n            elementFromAiLocate;\n\n          // update cache\n          let currentXpaths: string[] | undefined;\n          if (\n            element &&\n            this.taskCache &&\n            !cacheHitFlag &&\n            param?.cacheable !== false\n          ) {\n            const elementXpaths = await this.getElementXpath(\n              pageContext,\n              element,\n            );\n            if (elementXpaths?.length) {\n              currentXpaths = elementXpaths;\n              this.taskCache.updateOrAppendCacheRecord(\n                {\n                  type: 'locate',\n                  prompt: cachePrompt,\n                  xpaths: elementXpaths,\n                },\n                locateCacheRecord,\n              );\n            } else {\n              debug(\n                'no xpaths found, will not update cache',\n                cachePrompt,\n                elementXpaths,\n              );\n            }\n          }\n          if (!element) {\n            throw new Error(`Element not found: ${param.prompt}`);\n          }\n\n          let hitBy: ExecutionTaskHitBy | undefined;\n\n          if (userExpectedPathHitFlag) {\n            hitBy = {\n              from: 'User expected path',\n              context: {\n                xpath: param.xpath,\n              },\n            };\n          } else if (cacheHitFlag) {\n            hitBy = {\n              from: 'Cache',\n              context: {\n                xpathsFromCache: xpaths,\n                xpathsToSave: currentXpaths,\n              },\n            };\n          } else if (planHitFlag) {\n            hitBy = {\n              from: 'Planning',\n              context: {\n                id: elementFromPlan?.id,\n                bbox: elementFromPlan?.bbox,\n              },\n            };\n          } else if (aiLocateHitFlag) {\n            hitBy = {\n              from: 'AI model',\n              context: {\n                prompt: param.prompt,\n              },\n            };\n          }\n\n          onResult?.(element);\n\n          return {\n            output: {\n              element,\n            },\n            pageContext,\n            hitBy,\n          };\n        },\n      };\n      return taskFind;\n    };\n\n    for (const plan of plans) {\n      if (plan.type === 'Locate') {\n        if (\n          !plan.locate ||\n          plan.locate === null ||\n          plan.locate?.id === null ||\n          plan.locate?.id === 'null'\n        ) {\n          debug('Locate action with id is null, will be ignored', plan);\n          continue;\n        }\n        const taskLocate = taskForLocatePlan(plan, plan.locate);\n\n        tasks.push(taskLocate);\n      } else if (plan.type === 'Error') {\n        const taskActionError: ExecutionTaskActionApply<PlanningActionParamError> =\n          {\n            type: 'Action',\n            subType: 'Error',\n            param: plan.param,\n            thought: plan.thought || plan.param?.thought,\n            locate: plan.locate,\n            executor: async () => {\n              throw new Error(\n                plan?.thought || plan.param?.thought || 'error without thought',\n              );\n            },\n          };\n        tasks.push(taskActionError);\n      } else if (plan.type === 'Finished') {\n        const taskActionFinished: ExecutionTaskActionApply<null> = {\n          type: 'Action',\n          subType: 'Finished',\n          param: null,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async (param) => {},\n        };\n        tasks.push(taskActionFinished);\n      } else if (plan.type === 'Sleep') {\n        const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =\n          {\n            type: 'Action',\n            subType: 'Sleep',\n            param: plan.param,\n            thought: plan.thought,\n            locate: plan.locate,\n            executor: async (taskParam) => {\n              await sleep(taskParam?.timeMs || 3000);\n            },\n          };\n        tasks.push(taskActionSleep);\n      } else if (plan.type === 'Drag') {\n        const taskActionDrag: ExecutionTaskActionApply<{\n          start_box: { x: number; y: number };\n          end_box: { x: number; y: number };\n        }> = {\n          type: 'Action',\n          subType: 'Drag',\n          param: plan.param,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async (taskParam) => {\n            assert(\n              taskParam?.start_box && taskParam?.end_box,\n              'No start_box or end_box to drag',\n            );\n            await this.page.mouse.drag(taskParam.start_box, taskParam.end_box);\n          },\n        };\n        tasks.push(taskActionDrag);\n      } else {\n        // action in action space\n        const planType = plan.type;\n        const actionSpace = await this.page.actionSpace();\n        const action = actionSpace.find((action) => action.name === planType);\n        const param = plan.param;\n\n        if (!action) {\n          throw new Error(`Action type '${planType}' not found`);\n        }\n\n        // find all params that needs location\n        const locateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema)\n          : [];\n\n        const requiredLocateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema, true)\n          : [];\n\n        locateFields.forEach((field) => {\n          if (param[field]) {\n            const locatePlan = locatePlanForLocate(param[field]);\n            debug(\n              'will prepend locate param for field',\n              `action.type=${planType}`,\n              `param=${JSON.stringify(param[field])}`,\n              `locatePlan=${JSON.stringify(locatePlan)}`,\n            );\n            const locateTask = taskForLocatePlan(\n              locatePlan,\n              param[field],\n              (result) => {\n                param[field] = result;\n              },\n            );\n            tasks.push(locateTask);\n          } else {\n            assert(\n              !requiredLocateFields.includes(field),\n              `Required locate field '${field}' is not provided for action ${planType}`,\n            );\n            debug(`field '${field}' is not provided for action ${planType}`);\n          }\n        });\n\n        const task: ExecutionTaskApply<\n          'Action',\n          any,\n          { success: boolean; action: string; param: any },\n          void\n        > = {\n          type: 'Action',\n          subType: planType,\n          thought: plan.thought,\n          param: plan.param,\n          executor: async (param, context) => {\n            debug(\n              'executing action',\n              planType,\n              param,\n              `context.element.center: ${context.element?.center}`,\n            );\n\n            // Get page context for actionSpace operations to ensure size info is available\n            const pageContext = await this.insight.contextRetrieverFn('locate');\n            context.task.pageContext = pageContext;\n\n            requiredLocateFields.forEach((field) => {\n              assert(\n                param[field],\n                `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n              );\n            });\n\n            const actionFn = action.call.bind(this.page);\n            await actionFn(param, context);\n            // Return a proper result for report generation\n            return {\n              output: {\n                success: true,\n                action: planType,\n                param: param,\n              },\n            };\n          },\n        };\n        tasks.push(task);\n      }\n    }\n\n    const wrappedTasks = tasks.map(\n      (task: ExecutionTaskApply, index: number) => {\n        if (task.type === 'Action') {\n          return this.prependExecutorWithScreenshot(\n            task,\n            index === tasks.length - 1,\n          );\n        }\n        return task;\n      },\n    );\n\n    return {\n      tasks: wrappedTasks,\n    };\n  }\n\n  private async setupPlanningContext(executorContext: ExecutorContext) {\n    const shotTime = Date.now();\n    const pageContext = await this.insight.contextRetrieverFn('locate');\n    const recordItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: shotTime,\n      screenshot: pageContext.screenshotBase64,\n      timing: 'before Planning',\n    };\n\n    executorContext.task.recorder = [recordItem];\n    (executorContext.task as ExecutionTaskPlanning).pageContext = pageContext;\n\n    return {\n      pageContext,\n    };\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        await this.setupPlanningContext(executorContext);\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n\n    await taskExecutor.append(task);\n    await taskExecutor.flush();\n\n    return {\n      executor: taskExecutor,\n    };\n  }\n\n  private planningTaskFromPrompt(\n    userInstruction: string,\n    log?: string,\n    actionContext?: string,\n  ) {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n        log,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { pageContext } =\n          await this.setupPlanningContext(executorContext);\n\n        assert(\n          this.page.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.page.actionSpace();\n        debug(\n          'actionSpace for page is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.page.pageType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await plan(param.userInstruction, {\n          context: pageContext,\n          log: param.log,\n          actionContext,\n          pageType: this.page.pageType as PageType,\n          actionSpace,\n        });\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        // TODO: check locate result\n        // let bboxCollected = false;\n        // (actions || []).reduce<PlanningAction[]>(\n        //   (acc, planningAction) => {\n        //     // TODO: magic field \"locate\" is used to indicate the action requires a locate\n        //     if (planningAction.locate) {\n        //       // we only collect bbox once, let qwen re-locate in the following steps\n        //       if (bboxCollected && planningAction.locate.bbox) {\n        //         // biome-ignore lint/performance/noDelete: <explanation>\n        //         delete planningAction.locate.bbox;\n        //       }\n\n        //       if (planningAction.locate.bbox) {\n        //         bboxCollected = true;\n        //       }\n\n        //       acc.push({\n        //         type: 'Locate',\n        //         locate: planningAction.locate,\n        //         param: null,\n        //         // thought is prompt created by ai, always a string\n        //         thought: planningAction.locate.prompt as string,\n        //       });\n        //     }\n        //     acc.push(planningAction);\n        //     return acc;\n        //   },\n        //   [],\n        // );\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push({\n              type: 'Sleep',\n              param: {\n                timeMs: timeRemaining,\n              },\n              locate: null,\n            } as PlanningAction<PlanningActionParamSleep>);\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          pageContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  private planningTaskToGoal(\n    userInstruction: string,\n    modelPreferences: IModelPreferences,\n  ) {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { pageContext } =\n          await this.setupPlanningContext(executorContext);\n\n        const imagePayload = await resizeImageForUiTars(\n          pageContext.screenshotBase64,\n          pageContext.size,\n          modelPreferences,\n        );\n\n        this.appendConversationHistory({\n          role: 'user',\n          content: [\n            {\n              type: 'image_url',\n              image_url: {\n                url: imagePayload,\n              },\n            },\n          ],\n        });\n        const planResult: {\n          actions: PlanningAction<any>[];\n          action_summary: string;\n          usage?: AIUsageInfo;\n          yamlFlow?: MidsceneYamlFlowItem[];\n          rawResponse?: string;\n        } = await vlmPlanning({\n          userInstruction: param.userInstruction,\n          conversationHistory: this.conversationHistory,\n          size: pageContext.size,\n          modelPreferences,\n        });\n\n        const { actions, action_summary, usage } = planResult;\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse: planResult.rawResponse,\n        };\n        executorContext.task.usage = usage;\n        this.appendConversationHistory({\n          role: 'assistant',\n          content: action_summary,\n        });\n        return {\n          output: {\n            actions,\n            thought: actions[0]?.thought,\n            actionType: actions[0].type,\n            more_actions_needed_by_instruction: true,\n            log: '',\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n  ): Promise<ExecutionResult> {\n    const taskExecutor = new Executor(title, {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { tasks } = await this.convertPlanToExecutable(plans);\n    await taskExecutor.append(tasks);\n    const result = await taskExecutor.flush();\n    const { output } = result!;\n    return {\n      output,\n      executor: taskExecutor,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    actionContext?: string,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    let planningTask: ExecutionTaskPlanningApply | null =\n      this.planningTaskFromPrompt(userPrompt, undefined, actionContext);\n    let replanCount = 0;\n    const logList: string[] = [];\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      getAIConfigInNumber(MIDSCENE_REPLANNING_CYCLE_LIMIT) ||\n      defaultReplanningCycleLimit;\n    while (planningTask) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg =\n          'Replanning too many times, please split the task into multiple steps';\n\n        return this.appendErrorPlan(taskExecutor, errorMsg);\n      }\n\n      // plan\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      const planResult: PlanningAIResponse = result?.output;\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: planResult,\n          executor: taskExecutor,\n        };\n      }\n\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n\n      await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (planResult?.log) {\n        logList.push(planResult.log);\n      }\n\n      if (!planResult.more_actions_needed_by_instruction) {\n        planningTask = null;\n        break;\n      }\n      planningTask = this.planningTaskFromPrompt(\n        userPrompt,\n        logList.length > 0 ? `- ${logList.join('\\n- ')}` : undefined,\n        actionContext,\n      );\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  async actionToGoal(userPrompt: string): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    this.conversationHistory = [];\n    const isCompleted = false;\n    let currentActionCount = 0;\n    const maxActionNumber = 40;\n\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    while (!isCompleted && currentActionCount < maxActionNumber) {\n      currentActionCount++;\n      debug(\n        'actionToGoal, currentActionCount:',\n        currentActionCount,\n        'userPrompt:',\n        userPrompt,\n      );\n      const planningTask: ExecutionTaskPlanningApply = this.planningTaskToGoal(\n        userPrompt,\n        {\n          intent: 'planning',\n        },\n      );\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function actionToGoal',\n        );\n      }\n      const { output } = result;\n      const plans = output.actions;\n      yamlFlow.push(...(output.yamlFlow || []));\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans);\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n\n      await taskExecutor.flush();\n\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      if (plans[0].type === 'Finished') {\n        break;\n      }\n    }\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        // TODO: display image thumbnail in report\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let insightDump: InsightDump | undefined;\n        const dumpCollector: DumpSubscriber = (dump) => {\n          insightDump = dump;\n        };\n        this.insight.onceDumpUpdatedFn = dumpCollector;\n\n        // Get page context for query operations\n        const shotTime = Date.now();\n        const pageContext = await this.insight.contextRetrieverFn('extract');\n        task.pageContext = pageContext;\n\n        const recordItem: ExecutionRecorderItem = {\n          type: 'screenshot',\n          ts: shotTime,\n          screenshot: pageContext.screenshotBase64,\n          timing: 'before Extract',\n        };\n        task.recorder = [recordItem];\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        if (ifTypeRestricted) {\n          const returnType = type === 'Assert' ? 'Boolean' : type;\n          demandInput = {\n            result: `${returnType}, ${demand}`,\n          };\n        }\n\n        const { data, usage, thought } = await this.insight.extract<any>(\n          demandInput,\n          opt,\n          multimodalPrompt,\n        );\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          assert(data?.result !== undefined, 'No result in query data');\n          outputResult = (data as any).result;\n        }\n\n        return {\n          output: outputResult,\n          log: { dump: insightDump, isWaitForAssert: opt?.isWaitForAssert },\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const taskExecutor = new Executor(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        onTaskStart: this.onTaskStartCallback,\n      },\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      opt,\n      multimodalPrompt,\n    );\n\n    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n    const result = await taskExecutor.flush();\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      executor: taskExecutor,\n    };\n  }\n\n  async assert(\n    assertion: TUserPrompt,\n    opt?: InsightExtractOption,\n  ): Promise<ExecutionResult<boolean>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n    return await this.createTypeQueryExecution<boolean>(\n      'Assert',\n      textPrompt,\n      opt,\n      multimodalPrompt,\n    );\n  }\n\n  /**\n   * Append a message to the conversation history\n   * For user messages with images:\n   * - Keep max 4 user image messages in history\n   * - Remove oldest user image message when limit reached\n   * For assistant messages:\n   * - Simply append to history\n   * @param conversationHistory Message to append\n   */\n  private appendConversationHistory(\n    conversationHistory: ChatCompletionMessageParam,\n  ) {\n    if (conversationHistory.role === 'user') {\n      // Get all existing user messages with images\n      const userImgItems = this.conversationHistory.filter(\n        (item) => item.role === 'user',\n      );\n\n      // If we already have 4 user image messages\n      if (userImgItems.length >= 4 && conversationHistory.role === 'user') {\n        // Remove first user image message when we already have 4, before adding new one\n        const firstUserImgIndex = this.conversationHistory.findIndex(\n          (item) => item.role === 'user',\n        );\n        if (firstUserImgIndex >= 0) {\n          this.conversationHistory.splice(firstUserImgIndex, 1);\n        }\n      }\n    }\n    // For non-user messages, simply append to history\n    this.conversationHistory.push(conversationHistory);\n  }\n\n  private async appendErrorPlan(taskExecutor: Executor, errorMsg: string) {\n    const errorPlan: PlanningAction<PlanningActionParamError> = {\n      type: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      locate: null,\n    };\n    const { tasks } = await this.convertPlanToExecutable([errorPlan]);\n    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));\n    await taskExecutor.flush();\n\n    return {\n      output: undefined,\n      executor: taskExecutor,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'Assert',\n        textPrompt,\n        {\n          isWaitForAssert: true,\n          returnThought: true,\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n      const result = (await taskExecutor.flush()) as {\n        output: boolean;\n        thought?: string;\n      };\n\n      if (!result) {\n        throw new Error(\n          'result of taskExecutor.flush() is undefined in function waitFor',\n        );\n      }\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepPlan: PlanningAction<PlanningActionParamSleep> = {\n          type: 'Sleep',\n          param: {\n            timeMs: timeRemaining,\n          },\n          locate: null,\n        };\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable([\n          sleepPlan,\n        ]);\n        await taskExecutor.append(\n          this.prependExecutorWithScreenshot(sleepTasks[0]),\n        );\n        await taskExecutor.flush();\n      }\n    }\n\n    return this.appendErrorPlan(\n      taskExecutor,\n      `waitFor timeout: ${errorThought}`,\n    );\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","locatePlanForLocate","param","locate","locatePlan","PageTaskExecutor","timing","base64","item","Date","pageContext","element","_element_attributes","elementId","undefined","xpaths","NodeType","info","elementByPositionWithElementInfo","result","error","taskApply","appendAfterExecution","taskWithScreenshot","context","args","recorder","task","shot","Promise","sleep","shot2","plans","tasks","taskForLocatePlan","plan","detailedLocateParam","onResult","taskFind","taskContext","_this_taskCache","_locateCacheRecord_cacheContent","assert","JSON","insightDump","usage","dumpCollector","dump","_dump_taskInfo","shotTime","recordItem","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","aiLocateHitFlag","currentXpaths","elementXpaths","Error","hitBy","_plan_locate","_plan_locate1","taskLocate","_plan_param","taskActionError","taskActionFinished","taskActionSleep","taskParam","taskActionDrag","planType","actionSpace","action","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","locateTask","_context_element","actionFn","wrappedTasks","index","executorContext","userInstruction","yamlString","taskExecutor","Executor","taskTitleStr","log","actionContext","startTime","Array","console","planResult","actions","more_actions_needed_by_instruction","rawResponse","finalActions","timeNow","timeRemaining","modelPreferences","_actions_","imagePayload","resizeImageForUiTars","vlmPlanning","action_summary","title","output","userPrompt","planningTask","replanCount","logList","yamlFlow","replanningCycleLimit","getAIConfigInNumber","MIDSCENE_REPLANNING_CYCLE_LIMIT","errorMsg","executables","isCompleted","currentActionCount","maxActionNumber","type","demand","opt","multimodalPrompt","queryTask","ifTypeRestricted","demandInput","returnType","data","thought","outputResult","assertion","textPrompt","parsePrompt","conversationHistory","userImgItems","firstUserImgIndex","errorPlan","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepPlan","sleepTasks","page","insight","opts"],"mappings":";;;;;;;;;;;;;;;;;;;AAiEA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AAE7B,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAEO,MAAMC;IAyBX,MAAc,iBAAiBC,MAAuC,EAAE;QACtE,MAAMC,SAAS,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAC/C,MAAMC,OAA8B;YAClC,MAAM;YACN,IAAIC,KAAK,GAAG;YACZ,YAAYF;YACZD;QACF;QACA,OAAOE;IACT;IAEA,MAAc,gBACZE,WAAmC,EACnCC,OAA4B,EACG;YAoB3BC;QAnBJ,IAAI,CAAE,IAAI,CAAC,IAAI,CAAS,gBAAgB,EAAE,YACxCd,MAAM;QAIR,IAAIe,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,EAAE;QAC3B,IAAIA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB,AAAD,MAAMG,QAAW;YAC3C,MAAMC,SAAS,MAAO,IAAI,CAAC,IAAI,CAAS,gBAAgB,CACtD;gBACE,MAAMJ,QAAQ,MAAM,CAAC,EAAE;gBACvB,KAAKA,QAAQ,MAAM,CAAC,EAAE;YACxB,GACAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,gBAAgB;YAG3B,OAAOI;QACT;QAGA,IAAIH,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,sBAAAA,QAAS,UAAU,AAAD,IAAlBA,KAAAA,IAAAA,oBAAqB,QAAQ,AAAD,MAAMI,SAAS,QAAQ,EAAE;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;YACtC,MAAMC,OAAOC,iCACXR,YAAY,IAAI,EAChB;gBACE,GAAGC,QAAQ,MAAM,CAAC,EAAE;gBACpB,GAAGA,QAAQ,MAAM,CAAC,EAAE;YACtB,GACA;gBACE,uBAAuB;gBACvB,wBAAwB;YAC1B;YAEF,IAAIM,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE,EACVJ,YAAYI,KAAK,EAAE;iBAEnBnB,MACE,gEACAa;QAGN;QAEA,IAAI,CAACE,WACH;QAEF,IAAI;YACF,MAAMM,SAAS,MAAO,IAAI,CAAC,IAAI,CAAS,aAAa,CAACN;YACtD,OAAOM;QACT,EAAE,OAAOC,OAAO;YACdtB,MAAM,yBAAyBsB;QACjC;IACF;IAEQ,8BACNC,SAA6B,EAC7BC,uBAAuB,KAAK,EACR;QACpB,MAAMC,qBAAyC;YAC7C,GAAGF,SAAS;YACZ,UAAU,OAAOnB,OAAOsB,SAAS,GAAGC;gBAClC,MAAMC,WAAoC,EAAE;gBAC5C,MAAM,EAAEC,IAAI,EAAE,GAAGH;gBAEjBG,KAAK,QAAQ,GAAGD;gBAChB,MAAME,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAED,KAAK,IAAI,EAAE;gBAC9DD,SAAS,IAAI,CAACE;gBACd,MAAMT,SAAS,MAAME,UAAU,QAAQ,CAACnB,OAAOsB,YAAYC;gBAC3D,IAAIJ,AAAmB,aAAnBA,UAAU,IAAI,EAChB,MAAMQ,QAAQ,GAAG,CAAC;oBACf;wBACC,MAAMC,yBAAM;wBACZ,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;4BAC1BhC,MAAM;4BACN,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY;wBAC9B;oBACF;oBACAgC,yBAAM;iBACP;gBAEH,IAAIR,sBAAsB;oBACxB,MAAMS,QAAQ,MAAM,IAAI,CAAC,gBAAgB,CAAC;oBAC1CL,SAAS,IAAI,CAACK;gBAChB;gBACA,OAAOZ;YACT;QACF;QACA,OAAOI;IACT;IAEA,MAAa,wBAAwBS,KAAuB,EAAE;QAC5D,MAAMC,QAA8B,EAAE;QAEtC,MAAMC,oBAAoB,CACxBC,MACAC,qBACAC;YAEA,IAAI,AAA+B,YAA/B,OAAOD,qBACTA,sBAAsB;gBACpB,QAAQA;YACV;YAEF,MAAME,WAA4C;gBAChD,MAAM;gBACN,SAAS;gBACT,OAAOF;gBACP,SAASD,KAAK,OAAO;gBACrB,UAAU,OAAOjC,OAAOqC;wBA6CpBC,iBACaC;oBA7Cf,MAAM,EAAEd,IAAI,EAAE,GAAGY;oBACjBG,OACExC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAEyC,KAAK,SAAS,CACpEzC,QACC;oBAEL,IAAI0C;oBACJ,IAAIC;oBACJ,MAAMC,gBAAgC,CAACC;4BAE7BC;wBADRJ,cAAcG;wBACdF,QAAQG,QAAAA,OAAAA,KAAAA,IAAAA,QAAAA,CAAAA,iBAAAA,KAAM,QAAQ,AAAD,IAAbA,KAAAA,IAAAA,eAAgB,KAAK;wBAE7BrB,KAAK,GAAG,GAAG;4BACT,MAAMiB;wBACR;wBAEAjB,KAAK,KAAK,GAAGkB;oBACf;oBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGC;oBACjC,MAAMG,WAAWxC,KAAK,GAAG;oBAGzB,MAAMC,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBAC1DiB,KAAK,WAAW,GAAGjB;oBAEnB,MAAMwC,aAAoC;wBACxC,MAAM;wBACN,IAAID;wBACJ,YAAYvC,YAAY,gBAAgB;wBACxC,QAAQ;oBACV;oBACAiB,KAAK,QAAQ,GAAG;wBAACuB;qBAAW;oBAG5B,MAAMC,mBACJjD,MAAM,KAAK,IAAK,IAAI,CAAC,IAAI,CAAS,qBAAqB,GACnD,MAAO,IAAI,CAAC,IAAI,CAAS,qBAAqB,CAACA,MAAM,KAAK,IAC1DY;oBACN,MAAMsC,0BAA0B,CAAC,CAACD;oBAGlC,MAAME,cAAcnD,MAAM,MAAM;oBAChC,MAAMoD,oBAAAA,QACJd,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACa;oBACnC,MAAMtC,SAAS0B,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,MAAM;oBACtD,MAAMc,mBAAmBH,0BACrB,OACA,MAAMI,sBACJ,IAAI,EACJzC,QACAsC,aACAnD,MAAM,SAAS;oBAErB,MAAMuD,eAAe,CAAC,CAACF;oBAGvB,MAAMG,kBACJ,AAACN,2BAA4BK,eAEzB3C,SADA6C,qBAAqBzD,OAAOQ,YAAY,IAAI;oBAElD,MAAMkD,cAAc,CAAC,CAACF;oBAGtB,MAAMG,sBACJ,AAACT,2BAA4BK,gBAAiBG,cAO1C9C,SALE,OAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAACZ,OAAO;wBAE/B,SAASQ;oBACX,EAAC,EACD,OAAO;oBAEf,MAAMoD,kBAAkB,CAAC,CAACD;oBAE1B,MAAMlD,UACJwC,oBACAI,oBACAG,mBACAG;oBAGF,IAAIE;oBACJ,IACEpD,WACA,IAAI,CAAC,SAAS,IACd,CAAC8C,gBACDvD,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OACrB;wBACA,MAAM8D,gBAAgB,MAAM,IAAI,CAAC,eAAe,CAC9CtD,aACAC;wBAEF,IAAIqD,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,MAAM,EAAE;4BACzBD,gBAAgBC;4BAChB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gCACE,MAAM;gCACN,QAAQX;gCACR,QAAQW;4BACV,GACAV;wBAEJ,OACExD,MACE,0CACAuD,aACAW;oBAGN;oBACA,IAAI,CAACrD,SACH,MAAM,IAAIsD,MAAM,CAAC,mBAAmB,EAAE/D,MAAM,MAAM,EAAE;oBAGtD,IAAIgE;oBAEJ,IAAId,yBACFc,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,OAAOhE,MAAM,KAAK;wBACpB;oBACF;yBACK,IAAIuD,cACTS,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,iBAAiBnD;4BACjB,cAAcgD;wBAChB;oBACF;yBACK,IAAIH,aACTM,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,IAAIR,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;4BACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;wBAC7B;oBACF;yBACK,IAAII,iBACTI,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,QAAQhE,MAAM,MAAM;wBACtB;oBACF;oBAGFmC,QAAAA,YAAAA,SAAW1B;oBAEX,OAAO;wBACL,QAAQ;4BACNA;wBACF;wBACAD;wBACAwD;oBACF;gBACF;YACF;YACA,OAAO5B;QACT;QAEA,KAAK,MAAMH,QAAQH,MACjB,IAAIG,AAAc,aAAdA,KAAK,IAAI,EAAe;gBAIxBgC,cACAC;YAJF,IACE,CAACjC,KAAK,MAAM,IACZA,AAAgB,SAAhBA,KAAK,MAAM,IACXgC,AAAAA,SAAAA,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,EAAE,AAAD,MAAM,QACpBC,AAAAA,SAAAA,CAAAA,gBAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,EAAE,AAAD,MAAM,QACpB;gBACAtE,MAAM,kDAAkDqC;gBACxD;YACF;YACA,MAAMkC,aAAanC,kBAAkBC,MAAMA,KAAK,MAAM;YAEtDF,MAAM,IAAI,CAACoC;QACb,OAAO,IAAIlC,AAAc,YAAdA,KAAK,IAAI,EAAc;gBAMHmC;YAL7B,MAAMC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOpC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO,aAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD;gBAC3C,QAAQnC,KAAK,MAAM;gBACnB,UAAU;wBAEWmC;oBADnB,MAAM,IAAIL,MACR9B,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAC,SAAImC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD,KAAK;gBAE5C;YACF;YACFrC,MAAM,IAAI,CAACsC;QACb,OAAO,IAAIpC,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACnC,MAAMqC,qBAAqD;gBACzD,MAAM;gBACN,SAAS;gBACT,OAAO;gBACP,SAASrC,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOjC,SAAW;YAC9B;YACA+B,MAAM,IAAI,CAACuC;QACb,OAAO,IAAIrC,AAAc,YAAdA,KAAK,IAAI,EAAc;YAChC,MAAMsC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAOtC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOuC;oBACf,MAAM5C,yBAAM4C,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;gBACnC;YACF;YACFzC,MAAM,IAAI,CAACwC;QACb,OAAO,IAAItC,AAAc,WAAdA,KAAK,IAAI,EAAa;YAC/B,MAAMwC,iBAGD;gBACH,MAAM;gBACN,SAAS;gBACT,OAAOxC,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOuC;oBACfhC,OACEgC,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,SAAS,AAAD,KAAKA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,OAAO,AAAD,GACzC;oBAEF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAACA,UAAU,SAAS,EAAEA,UAAU,OAAO;gBACnE;YACF;YACAzC,MAAM,IAAI,CAAC0C;QACb,OAAO;YAEL,MAAMC,WAAWzC,KAAK,IAAI;YAC1B,MAAM0C,cAAc,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW;YAC/C,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACC,SAAWA,OAAO,IAAI,KAAKF;YAC5D,MAAM1E,QAAQiC,KAAK,KAAK;YAExB,IAAI,CAAC2C,QACH,MAAM,IAAIb,MAAM,CAAC,aAAa,EAAEW,SAAS,WAAW,CAAC;YAIvD,MAAMG,eAAeD,SACjBE,4BAA4BF,OAAO,WAAW,IAC9C,EAAE;YAEN,MAAMG,uBAAuBH,SACzBE,4BAA4BF,OAAO,WAAW,EAAE,QAChD,EAAE;YAENC,aAAa,OAAO,CAAC,CAACG;gBACpB,IAAIhF,KAAK,CAACgF,MAAM,EAAE;oBAChB,MAAM9E,aAAaH,oBAAoBC,KAAK,CAACgF,MAAM;oBACnDpF,MACE,uCACA,CAAC,YAAY,EAAE8E,UAAU,EACzB,CAAC,MAAM,EAAEjC,KAAK,SAAS,CAACzC,KAAK,CAACgF,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEvC,KAAK,SAAS,CAACvC,aAAa;oBAE5C,MAAM+E,aAAajD,kBACjB9B,YACAF,KAAK,CAACgF,MAAM,EACZ,CAAC/D;wBACCjB,KAAK,CAACgF,MAAM,GAAG/D;oBACjB;oBAEFc,MAAM,IAAI,CAACkD;gBACb,OAAO;oBACLzC,OACE,CAACuC,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEN,UAAU;oBAE3E9E,MAAM,CAAC,OAAO,EAAEoF,MAAM,6BAA6B,EAAEN,UAAU;gBACjE;YACF;YAEA,MAAMjD,OAKF;gBACF,MAAM;gBACN,SAASiD;gBACT,SAASzC,KAAK,OAAO;gBACrB,OAAOA,KAAK,KAAK;gBACjB,UAAU,OAAOjC,OAAOsB;wBAKO4D;oBAJ7BtF,MACE,oBACA8E,UACA1E,OACA,CAAC,wBAAwB,EAAE,QAAAkF,CAAAA,mBAAAA,QAAQ,OAAO,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,MAAM,EAAE;oBAItD,MAAM1E,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBAC1Dc,QAAQ,IAAI,CAAC,WAAW,GAAGd;oBAE3BuE,qBAAqB,OAAO,CAAC,CAACC;wBAC5BxC,OACExC,KAAK,CAACgF,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEN,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;oBAE9G;oBAEA,MAAMS,WAAWP,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;oBAC3C,MAAMO,SAASnF,OAAOsB;oBAEtB,OAAO;wBACL,QAAQ;4BACN,SAAS;4BACT,QAAQoD;4BACR,OAAO1E;wBACT;oBACF;gBACF;YACF;YACA+B,MAAM,IAAI,CAACN;QACb;QAGF,MAAM2D,eAAerD,MAAM,GAAG,CAC5B,CAACN,MAA0B4D;YACzB,IAAI5D,AAAc,aAAdA,KAAK,IAAI,EACX,OAAO,IAAI,CAAC,6BAA6B,CACvCA,MACA4D,UAAUtD,MAAM,MAAM,GAAG;YAG7B,OAAON;QACT;QAGF,OAAO;YACL,OAAO2D;QACT;IACF;IAEA,MAAc,qBAAqBE,eAAgC,EAAE;QACnE,MAAMvC,WAAWxC,KAAK,GAAG;QACzB,MAAMC,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAC1D,MAAMwC,aAAoC;YACxC,MAAM;YACN,IAAID;YACJ,YAAYvC,YAAY,gBAAgB;YACxC,QAAQ;QACV;QAEA8E,gBAAgB,IAAI,CAAC,QAAQ,GAAG;YAACtC;SAAW;QAC3CsC,gBAAgB,IAAI,CAA2B,WAAW,GAAG9E;QAE9D,OAAO;YACLA;QACF;IACF;IAEA,MAAM,uBAAuB+E,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,eAAe,IAAIC,SAASC,aAAa,UAAUJ,kBAAkB;YACzE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,MAAM9D,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL8D;YACF;YACA,UAAU,OAAOvF,OAAOsF;gBACtB,MAAM,IAAI,CAAC,oBAAoB,CAACA;gBAChC,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLE;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QAEA,MAAMC,aAAa,MAAM,CAAChE;QAC1B,MAAMgE,aAAa,KAAK;QAExB,OAAO;YACL,UAAUA;QACZ;IACF;IAEQ,uBACNF,eAAuB,EACvBK,GAAY,EACZC,aAAsB,EACtB;QACA,MAAMpE,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL8D;gBACAK;YACF;YACA,UAAU,OAAO5F,OAAOsF;gBACtB,MAAMQ,YAAYvF,KAAK,GAAG;gBAC1B,MAAM,EAAEC,WAAW,EAAE,GACnB,MAAM,IAAI,CAAC,oBAAoB,CAAC8E;gBAElC9C,OACE,IAAI,CAAC,IAAI,CAAC,WAAW,EACrB;gBAEF,MAAMmC,cAAc,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW;gBAC/C/E,MACE,4BACA+E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDpC,OAAOuD,MAAM,OAAO,CAACpB,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBqB,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gDAAgD,CAAC;gBAI3F,MAAMC,aAAa,MAAMhE,wBAAKjC,MAAM,eAAe,EAAE;oBACnD,SAASQ;oBACT,KAAKR,MAAM,GAAG;oBACd6F;oBACA,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAC5BlB;gBACF;gBAEA,MAAM,EACJuB,OAAO,EACPN,GAAG,EACHO,kCAAkC,EAClCjF,KAAK,EACLyB,KAAK,EACLyD,WAAW,EACXxE,KAAK,EACN,GAAGqE;gBAEJX,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCc;gBACF;gBACAd,gBAAgB,IAAI,CAAC,KAAK,GAAG3C;gBAE7B,MAAM0D,eAAeH,WAAW,EAAE;gBAgClC,IAAItE,OAAO;oBACT,MAAM0E,UAAU/F,KAAK,GAAG;oBACxB,MAAMgG,gBAAgB3E,QAAS0E,CAAAA,UAAUR,SAAQ;oBACjD,IAAIS,gBAAgB,GAClBF,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,OAAO;4BACL,QAAQE;wBACV;wBACA,QAAQ;oBACV;gBAEJ;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrB7D,OACE,CAAC2D,sCAAsCvE,OACvCV,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASmF;wBACTF;wBACAP;wBACA,UAAUK,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAzF;gBACF;YACF;QACF;QAEA,OAAOiB;IACT;IAEQ,mBACN8D,eAAuB,EACvBiB,gBAAmC,EACnC;QACA,MAAM/E,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL8D;YACF;YACA,UAAU,OAAOvF,OAAOsF;oBA+CTmB;gBA9Cb,MAAM,EAAEjG,WAAW,EAAE,GACnB,MAAM,IAAI,CAAC,oBAAoB,CAAC8E;gBAElC,MAAMoB,eAAe,MAAMC,qBACzBnG,YAAY,gBAAgB,EAC5BA,YAAY,IAAI,EAChBgG;gBAGF,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,WAAW;gCACT,KAAKE;4BACP;wBACF;qBACD;gBACH;gBACA,MAAMT,aAMF,MAAMW,YAAY;oBACpB,iBAAiB5G,MAAM,eAAe;oBACtC,qBAAqB,IAAI,CAAC,mBAAmB;oBAC7C,MAAMQ,YAAY,IAAI;oBACtBgG;gBACF;gBAEA,MAAM,EAAEN,OAAO,EAAEW,cAAc,EAAElE,KAAK,EAAE,GAAGsD;gBAC3CX,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClC,aAAaW,WAAW,WAAW;gBACrC;gBACAX,gBAAgB,IAAI,CAAC,KAAK,GAAG3C;gBAC7B,IAAI,CAAC,yBAAyB,CAAC;oBAC7B,MAAM;oBACN,SAASkE;gBACX;gBACA,OAAO;oBACL,QAAQ;wBACNX;wBACA,SAAS,QAAAO,CAAAA,YAAAA,OAAO,CAAC,EAAE,AAAD,IAATA,KAAAA,IAAAA,UAAY,OAAO;wBAC5B,YAAYP,OAAO,CAAC,EAAE,CAAC,IAAI;wBAC3B,oCAAoC;wBACpC,KAAK;wBACL,UAAUD,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;gBACF;YACF;QACF;QAEA,OAAOxE;IACT;IAEA,MAAM,SACJqF,KAAa,EACbhF,KAAuB,EACG;QAC1B,MAAM2D,eAAe,IAAIC,SAASoB,OAAO;YACvC,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAE/E,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACD;QACrD,MAAM2D,aAAa,MAAM,CAAC1D;QAC1B,MAAMd,SAAS,MAAMwE,aAAa,KAAK;QACvC,MAAM,EAAEsB,MAAM,EAAE,GAAG9F;QACnB,OAAO;YACL8F;YACA,UAAUtB;QACZ;IACF;IAEA,MAAM,OACJuB,UAAkB,EAClBnB,aAAsB,EAQtB;QACA,MAAMJ,eAAe,IAAIC,SAASC,aAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,IAAIC,eACF,IAAI,CAAC,sBAAsB,CAACD,YAAYpG,QAAWiF;QACrD,IAAIqB,cAAc;QAClB,MAAMC,UAAoB,EAAE;QAE5B,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJC,oBAAoBC,oCACpBzH;QACF,MAAOmH,aAAc;YACnB,IAAIC,cAAcG,sBAAsB;gBACtC,MAAMG,WACJ;gBAEF,OAAO,IAAI,CAAC,eAAe,CAAC/B,cAAc+B;YAC5C;YAGA,MAAM/B,aAAa,MAAM,CAACwB;YAC1B,MAAMhG,SAAS,MAAMwE,aAAa,KAAK;YACvC,MAAMQ,aAAiChF,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAIwE,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQQ;gBACR,UAAUR;YACZ;YAGF,MAAM3D,QAAQmE,WAAW,OAAO,IAAI,EAAE;YACtCmB,SAAS,IAAI,IAAKnB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAIwB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAAC3F;gBACjD2D,aAAa,MAAM,CAACgC,YAAY,KAAK;YACvC,EAAE,OAAOvG,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzBuE,cACA,CAAC,4CAA4C,EAAEvE,MAAM,SAAS,EAAEuB,KAAK,SAAS,CAC5EX,QACC;YAEP;YAEA,MAAM2D,aAAa,KAAK;YACxB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ7E;gBACR,UAAU6E;YACZ;YAEF,IAAIQ,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,GAAG,EACjBkB,QAAQ,IAAI,CAAClB,WAAW,GAAG;YAG7B,IAAI,CAACA,WAAW,kCAAkC,EAAE;gBAClDgB,eAAe;gBACf;YACF;YACAA,eAAe,IAAI,CAAC,sBAAsB,CACxCD,YACAG,QAAQ,MAAM,GAAG,IAAI,CAAC,EAAE,EAAEA,QAAQ,IAAI,CAAC,SAAS,GAAGvG,QACnDiF;YAEFqB;QACF;QAEA,OAAO;YACL,QAAQ;gBACNE;YACF;YACA,UAAU3B;QACZ;IACF;IAEA,MAAM,aAAauB,UAAkB,EAOnC;QACA,MAAMvB,eAAe,IAAIC,SAASC,aAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,IAAI,CAAC,mBAAmB,GAAG,EAAE;QAC7B,MAAMU,cAAc;QACpB,IAAIC,qBAAqB;QACzB,MAAMC,kBAAkB;QAExB,MAAMR,WAAmC,EAAE;QAC3C,MAAO,CAACM,eAAeC,qBAAqBC,gBAAiB;YAC3DD;YACA/H,MACE,qCACA+H,oBACA,eACAX;YAEF,MAAMC,eAA2C,IAAI,CAAC,kBAAkB,CACtED,YACA;gBACE,QAAQ;YACV;YAEF,MAAMvB,aAAa,MAAM,CAACwB;YAC1B,MAAMhG,SAAS,MAAMwE,aAAa,KAAK;YACvC,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ7E;gBACR,UAAU6E;YACZ;YAEF,IAAI,CAACxE,QACH,MAAM,IAAI8C,MACR;YAGJ,MAAM,EAAEgD,MAAM,EAAE,GAAG9F;YACnB,MAAMa,QAAQiF,OAAO,OAAO;YAC5BK,SAAS,IAAI,IAAKL,OAAO,QAAQ,IAAI,EAAE;YACvC,IAAIU;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAAC3F;gBACjD2D,aAAa,MAAM,CAACgC,YAAY,KAAK;YACvC,EAAE,OAAOvG,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzBuE,cACA,CAAC,4CAA4C,EAAEvE,MAAM,SAAS,EAAEuB,KAAK,SAAS,CAC5EX,QACC;YAEP;YAEA,MAAM2D,aAAa,KAAK;YAExB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQ7E;gBACR,UAAU6E;YACZ;YAGF,IAAI3D,AAAkB,eAAlBA,KAAK,CAAC,EAAE,CAAC,IAAI,EACf;QAEJ;QACA,OAAO;YACL,QAAQ;gBACNsF;YACF;YACA,UAAU3B;QACZ;IACF;IAEQ,oBACNoC,IAA0D,EAC1DC,MAA2B,EAC3BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBAEL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO9H,OAAOqC;gBACtB,MAAM,EAAEZ,IAAI,EAAE,GAAGY;gBACjB,IAAIK;gBACJ,MAAME,gBAAgC,CAACC;oBACrCH,cAAcG;gBAChB;gBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGD;gBAGjC,MAAMG,WAAWxC,KAAK,GAAG;gBACzB,MAAMC,cAAc,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;gBAC1DiB,KAAK,WAAW,GAAGjB;gBAEnB,MAAMwC,aAAoC;oBACxC,MAAM;oBACN,IAAID;oBACJ,YAAYvC,YAAY,gBAAgB;oBACxC,QAAQ;gBACV;gBACAiB,KAAK,QAAQ,GAAG;oBAACuB;iBAAW;gBAE5B,MAAMkF,mBAAmBL,AAAS,YAATA;gBACzB,IAAIM,cAAcL;gBAClB,IAAII,kBAAkB;oBACpB,MAAME,aAAaP,AAAS,aAATA,OAAoB,YAAYA;oBACnDM,cAAc;wBACZ,QAAQ,GAAGC,WAAW,EAAE,EAAEN,QAAQ;oBACpC;gBACF;gBAEA,MAAM,EAAEO,IAAI,EAAE1F,KAAK,EAAE2F,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACzDH,aACAJ,KACAC;gBAGF,IAAIO,eAAeF;gBACnB,IAAIH,kBAAkB;oBACpB1F,OAAO6F,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,MAAMzH,QAAW;oBACnC2H,eAAgBF,KAAa,MAAM;gBACrC;gBAEA,OAAO;oBACL,QAAQE;oBACR,KAAK;wBAAE,MAAM7F;wBAAa,iBAAiBqF,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe;oBAAC;oBAChEpF;oBACA2F;gBACF;YACF;QACF;QAEA,OAAOL;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvC,eAAe,IAAIC,SACvBC,aACEkC,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASrF,KAAK,SAAS,CAACqF,UAEvD;YACE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAGF,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAC,KACAC;QAGF,MAAMvC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACwC;QAC7D,MAAMhH,SAAS,MAAMwE,aAAa,KAAK;QAEvC,IAAI,CAACxE,QACH,MAAM,IAAI8C,MACR;QAIJ,MAAM,EAAEgD,MAAM,EAAEuB,OAAO,EAAE,GAAGrH;QAE5B,OAAO;YACL8F;YACAuB;YACA,UAAU7C;QACZ;IACF;IAEA,MAAM,OACJ+C,SAAsB,EACtBT,GAA0B,EACS;QACnC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,YAAYF;QACrD,OAAO,MAAM,IAAI,CAAC,wBAAwB,CACxC,UACAC,YACAV,KACAC;IAEJ;IAWQ,0BACNW,mBAA+C,EAC/C;QACA,IAAIA,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;YAEvC,MAAMC,eAAe,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAClD,CAACtI,OAASA,AAAc,WAAdA,KAAK,IAAI;YAIrB,IAAIsI,aAAa,MAAM,IAAI,KAAKD,AAA6B,WAA7BA,oBAAoB,IAAI,EAAa;gBAEnE,MAAME,oBAAoB,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAC1D,CAACvI,OAASA,AAAc,WAAdA,KAAK,IAAI;gBAErB,IAAIuI,qBAAqB,GACvB,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAACA,mBAAmB;YAEvD;QACF;QAEA,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACF;IAChC;IAEA,MAAc,gBAAgBlD,YAAsB,EAAE+B,QAAgB,EAAE;QACtE,MAAMsB,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACL,SAAStB;YACX;YACA,QAAQ;QACV;QACA,MAAM,EAAEzF,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC;YAAC+G;SAAU;QAChE,MAAMrD,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC1D,KAAK,CAAC,EAAE;QACrE,MAAM0D,aAAa,KAAK;QAExB,OAAO;YACL,QAAQ7E;YACR,UAAU6E;QACZ;IACF;IAEA,MAAM,QACJ+C,SAAsB,EACtBT,GAA+B,EACC;QAChC,MAAM,EAAEU,UAAU,EAAET,gBAAgB,EAAE,GAAGU,YAAYF;QAErD,MAAMO,cAAc,CAAC,SAAS,EAAEN,YAAY;QAC5C,MAAMhD,eAAe,IAAIC,SAASC,aAAa,WAAWoD,cAAc;YACtE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE,GAAGlB;QAEvCvF,OAAOgG,WAAW;QAClBhG,OAAOwG,WAAW;QAClBxG,OAAOyG,iBAAiB;QAExBzG,OACEyG,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmB3I,KAAK,GAAG;QACjC,IAAIuF,YAAYvF,KAAK,GAAG;QACxB,IAAI4I,eAAe;QACnB,MAAO5I,KAAK,GAAG,KAAK2I,mBAAmBF,UAAW;YAChDlD,YAAYvF,KAAK,GAAG;YACpB,MAAM0H,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,UACAQ,YACA;gBACE,iBAAiB;gBACjB,eAAe;gBACf,iBAAiB;YACnB,GACAT;YAGF,MAAMvC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACwC;YAC7D,MAAMhH,SAAU,MAAMwE,aAAa,KAAK;YAKxC,IAAI,CAACxE,QACH,MAAM,IAAI8C,MACR;YAIJ,IAAI9C,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQL;gBACR,UAAU6E;YACZ;YAGF0D,eACElI,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACd,CAAC,0CAA0C,EAAEwH,YAAY;YAC3D,MAAMW,MAAM7I,KAAK,GAAG;YACpB,IAAI6I,MAAMtD,YAAYmD,iBAAiB;gBACrC,MAAM1C,gBAAgB0C,kBAAmBG,CAAAA,MAAMtD,SAAQ;gBACvD,MAAMuD,YAAsD;oBAC1D,MAAM;oBACN,OAAO;wBACL,QAAQ9C;oBACV;oBACA,QAAQ;gBACV;gBACA,MAAM,EAAE,OAAO+C,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC;oBAC/DD;iBACD;gBACD,MAAM5D,aAAa,MAAM,CACvB,IAAI,CAAC,6BAA6B,CAAC6D,UAAU,CAAC,EAAE;gBAElD,MAAM7D,aAAa,KAAK;YAC1B;QACF;QAEA,OAAO,IAAI,CAAC,eAAe,CACzBA,cACA,CAAC,iBAAiB,EAAE0D,cAAc;IAEtC;IAtqCA,YACEI,IAAkB,EAClBC,OAAgB,EAChBC,IAGC,CACD;QAjBF;QAEA;QAEA;QAEA,8CAAoD,EAAE;QAEtD;QAUE,IAAI,CAAC,IAAI,GAAGF;QACZ,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;IAC9C;AA2pCF"}

package/dist/es/agent/ui-utils.mjs

@@ -0,0 +1,89 @@
+function typeStr(task) {
+    return task.subType && 'Plan' !== task.subType ? `${task.type} / ${task.subType || ''}` : task.type;
+}
+function getKeyCommands(value) {
+    const keys = Array.isArray(value) ? value : [
+        value
+    ];
+    return keys.reduce((acc, k)=>{
+        const includeMeta = keys.includes('Meta') || keys.includes('Control');
+        if (includeMeta && ('a' === k || 'A' === k)) return acc.concat([
+            {
+                key: k,
+                command: 'SelectAll'
+            }
+        ]);
+        if (includeMeta && ('c' === k || 'C' === k)) return acc.concat([
+            {
+                key: k,
+                command: 'Copy'
+            }
+        ]);
+        if (includeMeta && ('v' === k || 'V' === k)) return acc.concat([
+            {
+                key: k,
+                command: 'Paste'
+            }
+        ]);
+        return acc.concat([
+            {
+                key: k
+            }
+        ]);
+    }, []);
+}
+function locateParamStr(locate) {
+    if (!locate) return '';
+    if ('string' == typeof locate) return locate;
+    if ('string' == typeof locate.prompt) return locate.prompt;
+    if ('object' == typeof locate.prompt && locate.prompt.prompt) return locate.prompt.prompt;
+    return '';
+}
+function scrollParamStr(scrollParam) {
+    if (!scrollParam) return '';
+    return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;
+}
+function pullParamStr(pullParam) {
+    if (!pullParam) return '';
+    const parts = [];
+    parts.push(`direction: ${pullParam.direction || 'down'}`);
+    if (pullParam.distance) parts.push(`distance: ${pullParam.distance}`);
+    if (pullParam.duration) parts.push(`duration: ${pullParam.duration}ms`);
+    return parts.join(', ');
+}
+function taskTitleStr(type, prompt) {
+    if (prompt) return `${type} - ${prompt}`;
+    return type;
+}
+function paramStr(task) {
+    let value;
+    if ('Planning' === task.type) {
+        var _task_param;
+        value = null == task ? void 0 : null == (_task_param = task.param) ? void 0 : _task_param.userInstruction;
+    }
+    if ('Insight' === task.type) {
+        var _task_param1, _task_param2, _task_param3, _task_param4;
+        value = (null == task ? void 0 : null == (_task_param1 = task.param) ? void 0 : _task_param1.prompt) || (null == task ? void 0 : null == (_task_param2 = task.param) ? void 0 : _task_param2.id) || (null == task ? void 0 : null == (_task_param3 = task.param) ? void 0 : _task_param3.dataDemand) || (null == task ? void 0 : null == (_task_param4 = task.param) ? void 0 : _task_param4.assertion);
+    }
+    if ('Action' === task.type) {
+        var _task_param5, _task_param6, _task_param7, _task_param8;
+        const locate = null == task ? void 0 : task.locate;
+        const locateStr = locate ? locateParamStr(locate) : '';
+        value = task.thought || '';
+        if ('number' == typeof (null == task ? void 0 : null == (_task_param5 = task.param) ? void 0 : _task_param5.timeMs)) {
+            var _task_param9;
+            value = `${null == task ? void 0 : null == (_task_param9 = task.param) ? void 0 : _task_param9.timeMs}ms`;
+        } else if ('string' == typeof (null == task ? void 0 : null == (_task_param6 = task.param) ? void 0 : _task_param6.scrollType)) value = scrollParamStr(null == task ? void 0 : task.param);
+        else if ('string' == typeof (null == task ? void 0 : null == (_task_param7 = task.param) ? void 0 : _task_param7.direction) && (null == task ? void 0 : task.subType) === 'AndroidPull') value = pullParamStr(null == task ? void 0 : task.param);
+        else if (void 0 !== (null == task ? void 0 : null == (_task_param8 = task.param) ? void 0 : _task_param8.value)) {
+            var _task_param10;
+            value = null == task ? void 0 : null == (_task_param10 = task.param) ? void 0 : _task_param10.value;
+        }
+        if (locateStr) value = value ? `${locateStr} - ${value}` : locateStr;
+    }
+    if (void 0 === value) return '';
+    return 'string' == typeof value ? value : JSON.stringify(value, void 0, 2);
+}
+export { getKeyCommands, locateParamStr, paramStr, pullParamStr, scrollParamStr, taskTitleStr, typeStr };
+
+//# sourceMappingURL=ui-utils.mjs.map

package/dist/es/agent/ui-utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  AndroidPullParam,\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ScrollParam,\n} from '@/index';\n\nexport function typeStr(task: ExecutionTask) {\n  return task.subType && task.subType !== 'Plan'\n    ? `${task.type} / ${task.subType || ''}`\n    : task.type;\n}\n\nexport function getKeyCommands(\n  value: string | string[],\n): Array<{ key: string; command?: string }> {\n  // Ensure value is an array of keys\n  const keys = Array.isArray(value) ? value : [value];\n\n  // Process each key to attach a corresponding command if needed, based on the presence of 'Meta' or 'Control' in the keys array.\n  // ref: https://github.com/puppeteer/puppeteer/pull/9357/files#diff-32cf475237b000f980eb214a0a823e45a902bddb7d2426d677cae96397aa0ae4R94\n  return keys.reduce((acc: Array<{ key: string; command?: string }>, k) => {\n    const includeMeta = keys.includes('Meta') || keys.includes('Control');\n    if (includeMeta && (k === 'a' || k === 'A')) {\n      return acc.concat([{ key: k, command: 'SelectAll' }]);\n    }\n    if (includeMeta && (k === 'c' || k === 'C')) {\n      return acc.concat([{ key: k, command: 'Copy' }]);\n    }\n    if (includeMeta && (k === 'v' || k === 'V')) {\n      return acc.concat([{ key: k, command: 'Paste' }]);\n    }\n    return acc.concat([{ key: k }]);\n  }, []);\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam) {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate.prompt === 'string') {\n    return locate.prompt;\n  }\n\n  if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n    return locate.prompt.prompt;\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: AndroidPullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      (task as ExecutionTaskInsightLocate)?.param?.prompt ||\n      (task as ExecutionTaskInsightLocate)?.param?.id ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n  return typeof value === 'string'\n    ? value\n    : JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","getKeyCommands","value","keys","Array","acc","k","includeMeta","locateParamStr","locate","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","prompt","paramStr","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","_task_param7","_task_param8","locateStr","_task_param9","_task_param10","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IACzC,OAAOA,KAAK,OAAO,IAAIA,AAAiB,WAAjBA,KAAK,OAAO,GAC/B,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,IAAI,IAAI,GACtCA,KAAK,IAAI;AACf;AAEO,SAASC,eACdC,KAAwB;IAGxB,MAAMC,OAAOC,MAAM,OAAO,CAACF,SAASA,QAAQ;QAACA;KAAM;IAInD,OAAOC,KAAK,MAAM,CAAC,CAACE,KAA+CC;QACjE,MAAMC,cAAcJ,KAAK,QAAQ,CAAC,WAAWA,KAAK,QAAQ,CAAC;QAC3D,IAAII,eAAgBD,CAAAA,AAAM,QAANA,KAAaA,AAAM,QAANA,CAAQ,GACvC,OAAOD,IAAI,MAAM,CAAC;YAAC;gBAAE,KAAKC;gBAAG,SAAS;YAAY;SAAE;QAEtD,IAAIC,eAAgBD,CAAAA,AAAM,QAANA,KAAaA,AAAM,QAANA,CAAQ,GACvC,OAAOD,IAAI,MAAM,CAAC;YAAC;gBAAE,KAAKC;gBAAG,SAAS;YAAO;SAAE;QAEjD,IAAIC,eAAgBD,CAAAA,AAAM,QAANA,KAAaA,AAAM,QAANA,CAAQ,GACvC,OAAOD,IAAI,MAAM,CAAC;YAAC;gBAAE,KAAKC;gBAAG,SAAS;YAAQ;SAAE;QAElD,OAAOD,IAAI,MAAM,CAAC;YAAC;gBAAE,KAAKC;YAAE;SAAE;IAChC,GAAG,EAAE;AACP;AAEO,SAASE,eAAeC,MAA4B;IACzD,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;IAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAC3D,OAAOA,OAAO,MAAM,CAAC,MAAM;IAG7B,OAAO;AACT;AAEO,SAASC,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAA4B;IACvD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZC,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGD,KAAK,GAAG,EAAEC,QAAQ;IAE9B,OAAOD;AACT;AAEO,SAASE,SAASlB,IAAmB;IAC1C,IAAIE;IACJ,IAAIF,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBmB;QAARjB,QAASF,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,cAACnB,KAAgC,KAAK,AAAD,IAArCmB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAInB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAEzBoB,cACAC,cACAC,cACAC;QAJFrB,QACEkB,AAACpB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAAqC,KAAK,AAAD,IAA1CoB,KAAAA,IAAAA,aAA6C,MAAM,AAAD,KACjDpB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAAqC,KAAK,AAAD,IAA1CqB,KAAAA,IAAAA,aAA6C,EAAE,AAAD,KAC7CrB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAAoC,KAAK,AAAD,IAAzCsB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDtB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAAwC,KAAK,AAAD,IAA7CuB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAIvB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfwB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMlB,SAAUT,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAM4B,YAAYnB,SAASD,eAAeC,UAAU;QAEpDP,QAAQF,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADwB,CAAAA,eAACxB,KAA8B,KAAK,AAAD,IAAnCwB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAX3B,QAAQ,GAAIF,QAAAA,OAAAA,KAAAA,IAAAA,QAAD6B,CAAAA,eAAC7B,KAA8B,KAAK,AAAD,IAAnC6B,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQ7B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADyB,CAAAA,eAACzB,KAA8B,KAAK,AAAD,IAAnCyB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDvB,QAAQQ,eAAgBV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAAC1B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CE,QAAQU,aAAcZ,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAAD2B,CAAAA,eAAC3B,KAA8B,KAAK,AAAD,IAAnC2B,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAAR5B,QAASF,QAAAA,OAAAA,KAAAA,IAAAA,QAAD8B,CAAAA,gBAAC9B,KAA8B,KAAK,AAAD,IAAnC8B,KAAAA,IAAAA,cAAsC,KAAK;QACrD;QAEA,IAAIF,WAEA1B,QADEA,QACM,GAAG0B,UAAU,GAAG,EAAE1B,OAAO,GAEzB0B;IAGd;IAEA,IAAI,AAAiB,WAAV1B,OAAuB,OAAO;IACzC,OAAO,AAAiB,YAAjB,OAAOA,QACVA,QACA6B,KAAK,SAAS,CAAC7B,OAAO8B,QAAW;AACvC"}

package/dist/es/agent/utils.mjs

@@ -0,0 +1,337 @@
+import { elementByPositionWithElementInfo } from "../ai-model/index.mjs";
+import { getMidsceneLocationSchema, z } from "../index.mjs";
+import { sleep, uploadTestInfoToServer } from "../utils.mjs";
+import { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from "@midscene/shared/env";
+import { generateElementByPosition, getNodeFromCacheList } from "@midscene/shared/extractor";
+import { resizeImgBase64 } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { assert, logMsg, uuid } from "@midscene/shared/utils";
+import dayjs from "dayjs";
+import { debug } from "./task-cache.mjs";
+import { getKeyCommands } from "./ui-utils.mjs";
+const debugProfile = getDebug('web:tool:profile');
+const debugUtils = getDebug('web:tool:utils');
+async function commonContextParser(page) {
+    assert(page, 'page is required');
+    debugProfile('Getting page URL');
+    const url = await page.url();
+    debugProfile('URL end');
+    debugProfile('Uploading test info to server');
+    uploadTestInfoToServer({
+        testUrl: url
+    });
+    debugProfile('UploadTestInfoToServer end');
+    let screenshotBase64 = await page.screenshotBase64();
+    assert(screenshotBase64, 'screenshotBase64 is required');
+    const size = await page.size();
+    debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);
+    if (size.dpr && size.dpr > 1) {
+        debugProfile('Resizing screenshot for high DPR display');
+        screenshotBase64 = await resizeImgBase64(screenshotBase64, {
+            width: size.width,
+            height: size.height
+        });
+        debugProfile('ResizeImgBase64 end');
+    }
+    return {
+        tree: {
+            node: null,
+            children: []
+        },
+        size,
+        screenshotBase64: screenshotBase64,
+        url
+    };
+}
+function getReportFileName(tag = 'web') {
+    const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);
+    const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');
+    const uniqueId = uuid().substring(0, 8);
+    return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
+}
+function printReportMsg(filepath) {
+    logMsg(`Midscene - report file updated: ${filepath}`);
+}
+function getCurrentExecutionFile(trace) {
+    const error = new Error();
+    const stackTrace = trace || error.stack;
+    const pkgDir = process.cwd() || '';
+    if (stackTrace) {
+        const stackLines = stackTrace.split('\n');
+        for (const line of stackLines)if (line.includes('.spec.') || line.includes('.test.') || line.includes('.ts') || line.includes('.js')) {
+            const match = line.match(/(?:at\s+)?(.*?\.(?:spec|test)\.[jt]s)/);
+            if (null == match ? void 0 : match[1]) {
+                const targetFileName = match[1].replace(pkgDir, '').trim().replace('at ', '');
+                return targetFileName;
+            }
+        }
+    }
+    return false;
+}
+const testFileIndex = new Map();
+function generateCacheId(fileName) {
+    let taskFile = fileName || getCurrentExecutionFile();
+    if (!taskFile) {
+        taskFile = uuid();
+        console.warn('Midscene - using random UUID for cache id. Cache may be invalid.');
+    }
+    if (testFileIndex.has(taskFile)) {
+        const currentIndex = testFileIndex.get(taskFile);
+        if (void 0 !== currentIndex) testFileIndex.set(taskFile, currentIndex + 1);
+    } else testFileIndex.set(taskFile, 1);
+    return `${taskFile}-${testFileIndex.get(taskFile)}`;
+}
+function matchElementFromPlan(planLocateParam, tree) {
+    if (!planLocateParam) return;
+    if (planLocateParam.id) return getNodeFromCacheList(planLocateParam.id);
+    if (planLocateParam.bbox) {
+        const centerPosition = {
+            x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
+            y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
+        };
+        let element = elementByPositionWithElementInfo(tree, centerPosition);
+        if (!element) element = generateElementByPosition(centerPosition);
+        return element;
+    }
+}
+async function matchElementFromCache(taskExecutor, xpaths, cachePrompt, cacheable) {
+    try {
+        var _taskExecutor_taskCache;
+        if ((null == xpaths ? void 0 : xpaths.length) && (null == (_taskExecutor_taskCache = taskExecutor.taskCache) ? void 0 : _taskExecutor_taskCache.isCacheResultUsed) && false !== cacheable && taskExecutor.page.getElementInfoByXpath) for(let i = 0; i < xpaths.length; i++){
+            const element = await taskExecutor.page.getElementInfoByXpath(xpaths[i]);
+            if (null == element ? void 0 : element.id) {
+                debug('cache hit, prompt: %s', cachePrompt);
+                debug('found a new element with same xpath, xpath: %s, id: %s', xpaths[i], null == element ? void 0 : element.id);
+                return element;
+            }
+        }
+    } catch (error) {
+        debug('get element info by xpath error: ', error);
+    }
+}
+function trimContextByViewport(execution) {
+    function filterVisibleTree(node) {
+        if (!node) return null;
+        const filteredChildren = Array.isArray(node.children) ? node.children.map(filterVisibleTree).filter((child)=>null !== child) : [];
+        if (node.node && true === node.node.isVisible) return {
+            ...node,
+            children: filteredChildren
+        };
+        if (filteredChildren.length > 0) return {
+            node: null,
+            children: filteredChildren
+        };
+        return null;
+    }
+    return {
+        ...execution,
+        tasks: Array.isArray(execution.tasks) ? execution.tasks.map((task)=>{
+            var _task_pageContext;
+            const newTask = {
+                ...task
+            };
+            if (null == (_task_pageContext = task.pageContext) ? void 0 : _task_pageContext.tree) newTask.pageContext = {
+                ...task.pageContext,
+                tree: filterVisibleTree(task.pageContext.tree) || {
+                    node: null,
+                    children: []
+                }
+            };
+            return newTask;
+        }) : execution.tasks
+    };
+}
+function buildDetailedLocateParam(locatePrompt, opt) {
+    debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);
+    let prompt = locatePrompt || (null == opt ? void 0 : opt.prompt) || (null == opt ? void 0 : opt.locate);
+    let deepThink = false;
+    let cacheable = true;
+    let xpath;
+    if ('object' == typeof opt && null !== opt) {
+        deepThink = opt.deepThink ?? false;
+        cacheable = opt.cacheable ?? true;
+        xpath = opt.xpath;
+        if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) console.warn('conflict prompt for item', locatePrompt, opt, 'maybe you put the prompt in the wrong place');
+        prompt = prompt || opt.prompt;
+    }
+    if (!prompt) return void debugUtils('no prompt, will return undefined in buildDetailedLocateParam', opt);
+    return {
+        prompt,
+        deepThink,
+        cacheable,
+        xpath
+    };
+}
+function buildDetailedLocateParamAndRestParams(locatePrompt, opt, excludeKeys = []) {
+    const locateParam = buildDetailedLocateParam(locatePrompt, opt);
+    const restParams = {};
+    if ('object' == typeof opt && null !== opt) {
+        const allKeys = Object.keys(opt);
+        const locateParamKeys = Object.keys(locateParam || {});
+        for (const key of allKeys)if (!locateParamKeys.includes(key) && !excludeKeys.includes(key) && 'locate' !== key) restParams[key] = opt[key];
+    }
+    return {
+        locateParam,
+        restParams
+    };
+}
+const getMidsceneVersion = ()=>"0.27.1-beta-20250822094725.0";
+const parsePrompt = (prompt)=>{
+    if ('string' == typeof prompt) return {
+        textPrompt: prompt,
+        multimodalPrompt: void 0
+    };
+    return {
+        textPrompt: prompt.prompt,
+        multimodalPrompt: prompt.images ? {
+            images: prompt.images,
+            convertHttpImage2Base64: !!prompt.convertHttpImage2Base64
+        } : void 0
+    };
+};
+const commonWebActionsForWebPage = (page)=>[
+        {
+            name: 'Tap',
+            description: 'Tap the element',
+            interfaceAlias: 'aiTap',
+            paramSchema: z.object({
+                locate: getMidsceneLocationSchema().describe('The element to be tapped')
+            }),
+            call: async (param)=>{
+                const element = param.locate;
+                assert(element, 'Element not found, cannot tap');
+                await page.mouse.click(element.center[0], element.center[1], {
+                    button: 'left'
+                });
+            }
+        },
+        {
+            name: 'RightClick',
+            description: 'Right click the element',
+            interfaceAlias: 'aiRightClick',
+            paramSchema: z.object({
+                locate: getMidsceneLocationSchema().describe('The element to be right clicked')
+            }),
+            call: async (param)=>{
+                const element = param.locate;
+                assert(element, 'Element not found, cannot right click');
+                await page.mouse.click(element.center[0], element.center[1], {
+                    button: 'right'
+                });
+            }
+        },
+        {
+            name: 'Hover',
+            description: 'Move the mouse to the element',
+            interfaceAlias: 'aiHover',
+            paramSchema: z.object({
+                locate: getMidsceneLocationSchema().describe('The element to be hovered')
+            }),
+            call: async (param)=>{
+                const element = param.locate;
+                assert(element, 'Element not found, cannot hover');
+                await page.mouse.move(element.center[0], element.center[1]);
+            }
+        },
+        {
+            name: 'Input',
+            description: 'Replace the input field with a new value. `value` is the final that should be filled in the input box. No matter what modifications are required, just provide the final value to replace the existing input value. Giving a blank string means clear the input field.',
+            interfaceAlias: 'aiInput',
+            paramSchema: z.object({
+                value: z.string().describe('The final value that should be filled in the input box'),
+                locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
+            }),
+            call: async (param)=>{
+                const element = param.locate;
+                if (element) {
+                    await page.clearInput(element);
+                    if (!param || !param.value) return;
+                }
+                await page.keyboard.type(param.value);
+            }
+        },
+        {
+            name: 'KeyboardPress',
+            description: 'Press a function key, like "Enter", "Tab", "Escape". Do not use this to type text.',
+            interfaceAlias: 'aiKeyboardPress',
+            paramSchema: z.object({
+                locate: getMidsceneLocationSchema().describe('The element to be clicked before pressing the key').optional(),
+                keyName: z.string().describe('The key to be pressed')
+            }),
+            call: async (param)=>{
+                const element = param.locate;
+                if (element) await page.mouse.click(element.center[0], element.center[1], {
+                    button: 'left'
+                });
+                const keys = getKeyCommands(param.keyName);
+                await page.keyboard.press(keys);
+            }
+        },
+        {
+            name: 'Scroll',
+            description: 'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',
+            interfaceAlias: 'aiScroll',
+            paramSchema: z.object({
+                direction: z["enum"]([
+                    'down',
+                    'up',
+                    'right',
+                    'left'
+                ]).default('down').describe('The direction to scroll'),
+                scrollType: z["enum"]([
+                    'once',
+                    'untilBottom',
+                    'untilTop',
+                    'untilRight',
+                    'untilLeft'
+                ]).default('once').describe('The scroll type'),
+                distance: z.number().nullable().optional().describe('The distance in pixels to scroll'),
+                locate: getMidsceneLocationSchema().optional().describe('The element to be scrolled')
+            }),
+            call: async (param)=>{
+                const element = param.locate;
+                const startingPoint = element ? {
+                    left: element.center[0],
+                    top: element.center[1]
+                } : void 0;
+                const scrollToEventName = null == param ? void 0 : param.scrollType;
+                if ('untilTop' === scrollToEventName) await page.scrollUntilTop(startingPoint);
+                else if ('untilBottom' === scrollToEventName) await page.scrollUntilBottom(startingPoint);
+                else if ('untilRight' === scrollToEventName) await page.scrollUntilRight(startingPoint);
+                else if ('untilLeft' === scrollToEventName) await page.scrollUntilLeft(startingPoint);
+                else if ('once' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+                else {
+                    if ((null == param ? void 0 : param.direction) !== 'down' && param && param.direction) if ('up' === param.direction) await page.scrollUp(param.distance || void 0, startingPoint);
+                    else if ('left' === param.direction) await page.scrollLeft(param.distance || void 0, startingPoint);
+                    else if ('right' === param.direction) await page.scrollRight(param.distance || void 0, startingPoint);
+                    else throw new Error(`Unknown scroll direction: ${param.direction}`);
+                    else await page.scrollDown((null == param ? void 0 : param.distance) || void 0, startingPoint);
+                    await sleep(500);
+                }
+            }
+        },
+        {
+            name: 'DragAndDrop',
+            description: 'Drag and drop the element',
+            paramSchema: z.object({
+                from: getMidsceneLocationSchema().describe('The position to be dragged'),
+                to: getMidsceneLocationSchema().describe('The position to be dropped')
+            }),
+            call: async (param)=>{
+                const from = param.from;
+                const to = param.to;
+                assert(from, 'missing "from" param for drag and drop');
+                assert(to, 'missing "to" param for drag and drop');
+                await page.mouse.drag({
+                    x: from.center[0],
+                    y: from.center[1]
+                }, {
+                    x: to.center[0],
+                    y: to.center[1]
+                });
+            }
+        }
+    ];
+export { buildDetailedLocateParam, buildDetailedLocateParamAndRestParams, commonContextParser, commonWebActionsForWebPage, generateCacheId, getCurrentExecutionFile, getMidsceneVersion, getReportFileName, matchElementFromCache, matchElementFromPlan, parsePrompt, printReportMsg, trimContextByViewport };
+
+//# sourceMappingURL=utils.mjs.map