@midscene/core 1.0.1-beta-20251208075922.0 → 1.0.1-beta-20251208112226.0

package/dist/lib/ai-model/conversation-history.js

@@ -55,10 +55,27 @@ class ConversationHistory {
     reset() {
         this.messages.length = 0;
     }
-    snapshot() {
-        return [
+    snapshot(maxImages) {
+        if (void 0 === maxImages) return [
             ...this.messages
         ];
+        const clonedMessages = structuredClone(this.messages);
+        let imageCount = 0;
+        for(let i = clonedMessages.length - 1; i >= 0; i--){
+            const message = clonedMessages[i];
+            const content = message.content;
+            if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
+                const item = content[j];
+                if ('image_url' === item.type) {
+                    imageCount++;
+                    if (imageCount > maxImages) content[j] = {
+                        type: 'text',
+                        text: '(image ignored due to size optimization)'
+                    };
+                }
+            }
+        }
+        return clonedMessages;
     }
     get length() {
         return this.messages.length;
@@ -77,9 +94,9 @@ class ConversationHistory {
     }
 }
 exports.ConversationHistory = __webpack_exports__.ConversationHistory;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "ConversationHistory"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/conversation-history.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/conversation-history.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/conversation-history.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  snapshot(): ChatCompletionMessageParam[] {\n    return [...this.messages];\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ConversationHistory","message","messages","options"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;eCyCGG,OAAO,QAAQ;;AAzCX,MAAMC;IAYX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAEA,WAAyC;QACvC,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACF,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IA1CA,YAAYI,OAAoC,CAAE;QAJlD,uBAAiB,YAAyC,EAAE;QAE5D,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAsCF"}
+{"version":3,"file":"ai-model/conversation-history.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/conversation-history.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  /**\n   * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n   * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n   * @returns The snapshot of the conversation history.\n   */\n  snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n    if (maxImages === undefined) {\n      return [...this.messages];\n    }\n\n    const clonedMessages = structuredClone(this.messages);\n    let imageCount = 0;\n\n    // Traverse from the end to the beginning\n    for (let i = clonedMessages.length - 1; i >= 0; i--) {\n      const message = clonedMessages[i];\n      const content = message.content;\n\n      // Only process if content is an array\n      if (Array.isArray(content)) {\n        for (let j = 0; j < content.length; j++) {\n          const item = content[j];\n\n          // Check if this is an image\n          if (item.type === 'image_url') {\n            imageCount++;\n\n            // If we've exceeded the limit, replace with text\n            if (imageCount > maxImages) {\n              content[j] = {\n                type: 'text',\n                text: '(image ignored due to size optimization)',\n              };\n            }\n          }\n        }\n      }\n    }\n\n    return clonedMessages;\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","options"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;eC+EGG,OAAO,QAAQ;;AA/EX,MAAMC;IAYX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAhFA,YAAYc,OAAoC,CAAE;QAJlD,uBAAiB,YAAyC,EAAE;QAE5D,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AA4EF"}

package/dist/lib/ai-model/index.js

@@ -93,7 +93,7 @@ exports.plan = __webpack_exports__.plan;
 exports.resizeImageForUiTars = __webpack_exports__.resizeImageForUiTars;
 exports.systemPromptToLocateElement = __webpack_exports__.systemPromptToLocateElement;
 exports.uiTarsPlanning = __webpack_exports__.uiTarsPlanning;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "AIActionType",
     "AiExtractElementInfo",
     "AiJudgeOrderSensitive",
@@ -123,7 +123,7 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "resizeImageForUiTars",
     "systemPromptToLocateElement",
     "uiTarsPlanning"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/inspect.js

@@ -307,12 +307,12 @@ exports.AiExtractElementInfo = __webpack_exports__.AiExtractElementInfo;
 exports.AiJudgeOrderSensitive = __webpack_exports__.AiJudgeOrderSensitive;
 exports.AiLocateElement = __webpack_exports__.AiLocateElement;
 exports.AiLocateSection = __webpack_exports__.AiLocateSection;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "AiExtractElementInfo",
     "AiJudgeOrderSensitive",
     "AiLocateElement",
     "AiLocateSection"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/llm-planning.js

@@ -53,13 +53,14 @@ async function plan(userInstruction, opts) {
         imageHeight = paddedResult.height;
         imagePayload = paddedResult.imageBase64;
     }
+    const actionContext = opts.actionContext ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n` : '';
     const instruction = [
         {
             role: 'user',
             content: [
                 {
                     type: 'text',
-                    text: `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n<user_instruction>${userInstruction}</user_instruction>`
+                    text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`
                 }
             ]
         }
@@ -100,7 +101,7 @@ async function plan(userInstruction, opts) {
         ]
     };
     conversationHistory.append(latestFeedbackMessage);
-    const historyLog = conversationHistory.snapshot();
+    const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
     const msgs = [
         {
             role: 'system',
@@ -145,9 +146,9 @@ async function plan(userInstruction, opts) {
     return returnValue;
 }
 exports.plan = __webpack_exports__.plan;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "plan"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  AIActionType,\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory: ConversationHistory;\n    includeBbox: boolean;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { vlMode } = modelConfig;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode,\n    includeBbox: opts.includeBbox,\n  });\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = size.width;\n  let imageHeight = size.height;\n  const rightLimit = imageWidth;\n  const bottomLimit = imageHeight;\n\n  // Process image based on VL mode requirements\n  if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n<user_instruction>${userInstruction}</user_instruction>`,\n        },\n      ],\n    },\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The last screenshot is attached. Please going on according to the instruction.`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'this is the latest screenshot',\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n  const historyLog = conversationHistory.snapshot();\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    content: planFromAI,\n    contentString: rawResponse,\n    usage,\n  } = await callAIWithObjectResponse<RawResponsePlanningAIResponse>(\n    msgs,\n    AIActionType.PLAN,\n    modelConfig,\n  );\n\n  const actions = planFromAI.action ? [planFromAI.action] : [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult && vlMode !== undefined) {\n        // Always use VL mode to fill bbox parameters\n        action.param[field] = fillBboxParam(\n          locateResult,\n          imageWidth,\n          imageHeight,\n          rightLimit,\n          bottomLimit,\n          vlMode,\n        );\n      }\n    });\n  });\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","plan","userInstruction","opts","context","modelConfig","conversationHistory","screenshotBase64","size","vlMode","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","rightLimit","bottomLimit","paddedResult","paddingToMatchBlockByBase64","instruction","latestFeedbackMessage","historyLog","msgs","planFromAI","rawResponse","usage","callAIWithObjectResponse","AIActionType","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","undefined","fillBboxParam","console"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACgBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAQC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGJ;IAEnC,MAAM,EAAEK,MAAM,EAAE,GAAGJ;IAEnB,MAAMK,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7BM;QACA,aAAaN,KAAK,WAAW;IAC/B;IAEA,IAAIS,eAAeL;IACnB,IAAIM,aAAaL,KAAK,KAAK;IAC3B,IAAIM,cAAcN,KAAK,MAAM;IAC7B,MAAMO,aAAaF;IACnB,MAAMG,cAAcF;IAGpB,IAAIL,AAAW,iBAAXA,QAAyB;QAC3B,MAAMQ,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAME,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,CAAC,yBAAyB,EAAEhB,KAAK,aAAa,CAAC,8CAA8C,EAAED,gBAAgB,mBAAmB,CAAC;gBAC3I;aACD;QACH;KACD;IAED,IAAIkB;IAEJ,IAAId,oBAAoB,sBAAsB,EAAE;QAC9Cc,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGd,oBAAoB,sBAAsB,CAAC,gFAAgF,CAAC;gBACvI;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKM;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEc,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACc;IAC3B,MAAMC,aAAaf,oBAAoB,QAAQ;IAE/C,MAAMgB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASZ;QAAa;WACrCS;WACAE;KACJ;IAED,MAAM,EACJ,SAASE,UAAU,EACnB,eAAeC,WAAW,EAC1BC,KAAK,EACN,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACRJ,MACAK,mCAAAA,YAAAA,CAAAA,IAAiB,EACjBtB;IAGF,MAAMuB,UAAUL,WAAW,MAAM,GAAG;QAACA,WAAW,MAAM;KAAC,GAAG,EAAE;IAC5D,MAAMM,cAAkC;QACtC,GAAGN,UAAU;QACbK;QACAJ;QACAC;QACA,UAAUK,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EACRF,SACAzB,KAAK,WAAW,EAChBoB,WAAW,KAAK;IAEpB;IAEAQ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOR,YAAY;IAEnBK,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsB/B,KAAK,WAAW,CAAC,IAAI,CAC/C,CAAC6B,SAAWA,OAAO,IAAI,KAAKC;QAG9BlC,MAAM,+BAA+BmC;QACrC,MAAMC,eAAeD,sBACjBE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENnC,MAAM,gBAAgBoC;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,gBAAgB7B,AAAW8B,WAAX9B,QAElBuB,OAAO,KAAK,CAACK,MAAM,GAAGG,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EACpBF,cACAzB,YACAC,aACAC,YACAC,aACAP;QAGN;IACF;IAEA,IACEmB,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACAvC;IAIJI,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMkB;YACR;SACD;IACH;IAEA,OAAOK;AACT"}
+{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  AIActionType,\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory: ConversationHistory;\n    includeBbox: boolean;\n    imagesIncludeCount?: number;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { vlMode } = modelConfig;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode,\n    includeBbox: opts.includeBbox,\n  });\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = size.width;\n  let imageHeight = size.height;\n  const rightLimit = imageWidth;\n  const bottomLimit = imageHeight;\n\n  // Process image based on VL mode requirements\n  if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n        },\n      ],\n    },\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The last screenshot is attached. Please going on according to the instruction.`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'this is the latest screenshot',\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    content: planFromAI,\n    contentString: rawResponse,\n    usage,\n  } = await callAIWithObjectResponse<RawResponsePlanningAIResponse>(\n    msgs,\n    AIActionType.PLAN,\n    modelConfig,\n  );\n\n  const actions = planFromAI.action ? [planFromAI.action] : [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult && vlMode !== undefined) {\n        // Always use VL mode to fill bbox parameters\n        action.param[field] = fillBboxParam(\n          locateResult,\n          imageWidth,\n          imageHeight,\n          rightLimit,\n          bottomLimit,\n          vlMode,\n        );\n      }\n    });\n  });\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","plan","userInstruction","opts","context","modelConfig","conversationHistory","screenshotBase64","size","vlMode","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","rightLimit","bottomLimit","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","historyLog","msgs","planFromAI","rawResponse","usage","callAIWithObjectResponse","AIActionType","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","undefined","fillBboxParam","console"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACgBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IASC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGJ;IAEnC,MAAM,EAAEK,MAAM,EAAE,GAAGJ;IAEnB,MAAMK,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7BM;QACA,aAAaN,KAAK,WAAW;IAC/B;IAEA,IAAIS,eAAeL;IACnB,IAAIM,aAAaL,KAAK,KAAK;IAC3B,IAAIM,cAAcN,KAAK,MAAM;IAC7B,MAAMO,aAAaF;IACnB,MAAMG,cAAcF;IAGpB,IAAIL,AAAW,iBAAXA,QAAyB;QAC3B,MAAMQ,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBhB,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEjB,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAImB;IAEJ,IAAIf,oBAAoB,sBAAsB,EAAE;QAC9Ce,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGf,oBAAoB,sBAAsB,CAAC,gFAAgF,CAAC;gBACvI;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKM;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEe,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACe;IAC3B,MAAMC,aAAahB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMoB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASb;QAAa;WACrCU;WACAE;KACJ;IAED,MAAM,EACJ,SAASE,UAAU,EACnB,eAAeC,WAAW,EAC1BC,KAAK,EACN,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACRJ,MACAK,mCAAAA,YAAAA,CAAAA,IAAiB,EACjBvB;IAGF,MAAMwB,UAAUL,WAAW,MAAM,GAAG;QAACA,WAAW,MAAM;KAAC,GAAG,EAAE;IAC5D,MAAMM,cAAkC;QACtC,GAAGN,UAAU;QACbK;QACAJ;QACAC;QACA,UAAUK,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EACRF,SACA1B,KAAK,WAAW,EAChBqB,WAAW,KAAK;IAEpB;IAEAQ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOR,YAAY;IAEnBK,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBhC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAAC8B,SAAWA,OAAO,IAAI,KAAKC;QAG9BnC,MAAM,+BAA+BoC;QACrC,MAAMC,eAAeD,sBACjBE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENpC,MAAM,gBAAgBqC;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,gBAAgB9B,AAAW+B,WAAX/B,QAElBwB,OAAO,KAAK,CAACK,MAAM,GAAGG,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EACpBF,cACA1B,YACAC,aACAC,YACAC,aACAP;QAGN;IACF;IAEA,IACEoB,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACAxC;IAIJI,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMmB;YACR;SACD;IACH;IAEA,OAAOK;AACT"}

package/dist/lib/ai-model/prompt/assertion.js

@@ -55,9 +55,9 @@ const assertSchema = {
     }
 };
 exports.assertSchema = __webpack_exports__.assertSchema;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "assertSchema"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/common.js

@@ -31,9 +31,9 @@ function bboxDescription(vlMode) {
     return '2d bounding box as [xmin, ymin, xmax, ymax]';
 }
 exports.bboxDescription = __webpack_exports__.bboxDescription;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "bboxDescription"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/describe.js

@@ -68,9 +68,9 @@ Return JSON:
   "error"?: "error message if any"
 }`;
 exports.elementDescriberInstruction = __webpack_exports__.elementDescriberInstruction;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "elementDescriberInstruction"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/extraction.js

@@ -157,11 +157,11 @@ const extractDataSchema = {
 exports.extractDataQueryPrompt = __webpack_exports__.extractDataQueryPrompt;
 exports.extractDataSchema = __webpack_exports__.extractDataSchema;
 exports.systemPromptToExtract = __webpack_exports__.systemPromptToExtract;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "extractDataQueryPrompt",
     "extractDataSchema",
     "systemPromptToExtract"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/llm-locator.js

@@ -70,10 +70,10 @@ When no element is found:
 const findElementPrompt = (targetElementDescription)=>`Find: ${targetElementDescription}`;
 exports.findElementPrompt = __webpack_exports__.findElementPrompt;
 exports.systemPromptToLocateElement = __webpack_exports__.systemPromptToLocateElement;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "findElementPrompt",
     "systemPromptToLocateElement"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/llm-planning.js

@@ -24,9 +24,8 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    systemPromptToTaskPlanning: ()=>systemPromptToTaskPlanning,
     descriptionForAction: ()=>descriptionForAction,
-    planSchema: ()=>planSchema
+    systemPromptToTaskPlanning: ()=>systemPromptToTaskPlanning
 });
 const external_common_js_namespaceObject = require("../../common.js");
 const external_common_js_namespaceObject_1 = require("./common.js");
@@ -164,7 +163,7 @@ Please tell what the next one action is (or null if no action should be done) to
 - Make sure the previous actions are completed successfully before performing the next step
 - If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the "error" field to the error message.
 - If there is nothing to do but waiting, set the "sleep" field to the positive waiting time in milliseconds and null for the "action" field.
-- When the next step is to assert something, this is a very important step, you should think about it carefully and give a solid result. Write your result in the "log" field like this: "Assert: <condition>. I think <...>, so the result is <true / false>". You don't need to give the next one action when you are asserting something. If the assertion result is false, think this an fatal error and set the reason into the "error" field. If the assertion result is true, you can continue to the next step.
+- Assertions are also important steps. When getting the assertion instruction, a solid conclusion is required. You should explicitly state your conclusion by calling the "Print_Assert_Result" action.
 
 ## Supporting actions
 ${actionList}
@@ -179,113 +178,36 @@ Return in JSON format:
   ${commonOutputFields}
   "action": 
     {
-      // one of the supporting actions
+      "type": string, // the type of the action
+      "param"?: { // The parameter of the action, if any
+         // k-v style parameter fields
+      }, 
     } | null,
   ,
   "sleep"?: number, // The sleep time after the action, in milliseconds.
 }
+
+For example, if the instruction is to login and the form has already been filled, this is a valid return value:
+
+{
+  "log": "Click the login button",
+  "more_actions_needed_by_instruction": false,
+  "action": {
+    "type": "Tap",
+    "param": {
+      "locate": { 
+        "prompt": "The login button"${vlMode ? ', "bbox": [100, 200, 300, 400]' : ''}
+      }
+    }
+  }
 `;
 }
-const planSchema = {
-    type: 'json_schema',
-    json_schema: {
-        name: 'action_items',
-        strict: false,
-        schema: {
-            type: 'object',
-            strict: false,
-            properties: {
-                actions: {
-                    type: 'array',
-                    items: {
-                        type: 'object',
-                        strict: false,
-                        properties: {
-                            thought: {
-                                type: 'string',
-                                description: 'Reasons for generating this task, and why this task is feasible on this page'
-                            },
-                            type: {
-                                type: 'string',
-                                description: 'Type of action'
-                            },
-                            param: {
-                                anyOf: [
-                                    {
-                                        type: 'null'
-                                    },
-                                    {
-                                        type: 'object',
-                                        additionalProperties: true
-                                    }
-                                ],
-                                description: 'Parameter of the action'
-                            },
-                            locate: {
-                                type: [
-                                    'object',
-                                    'null'
-                                ],
-                                properties: {
-                                    id: {
-                                        type: 'string'
-                                    },
-                                    prompt: {
-                                        type: 'string'
-                                    }
-                                },
-                                required: [
-                                    'id',
-                                    'prompt'
-                                ],
-                                additionalProperties: false,
-                                description: 'Location information for the target element'
-                            }
-                        },
-                        required: [
-                            'thought',
-                            'type',
-                            'param',
-                            'locate'
-                        ],
-                        additionalProperties: false
-                    },
-                    description: 'List of actions to be performed'
-                },
-                more_actions_needed_by_instruction: {
-                    type: 'boolean',
-                    description: 'If all the actions described in the instruction have been covered by this action and logs, set this field to false.'
-                },
-                log: {
-                    type: 'string',
-                    description: 'Log what these planned actions do. Do not include further actions that have not been planned.'
-                },
-                error: {
-                    type: [
-                        'string',
-                        'null'
-                    ],
-                    description: 'Error messages about unexpected situations'
-                }
-            },
-            required: [
-                'actions',
-                'more_actions_needed_by_instruction',
-                'log',
-                'error'
-            ],
-            additionalProperties: false
-        }
-    }
-};
 exports.descriptionForAction = __webpack_exports__.descriptionForAction;
-exports.planSchema = __webpack_exports__.planSchema;
 exports.systemPromptToTaskPlanning = __webpack_exports__.systemPromptToTaskPlanning;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "descriptionForAction",
-    "planSchema",
     "systemPromptToTaskPlanning"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/types';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport { ifMidsceneLocatorField } from '../../common';\nimport { bboxDescription } from './common';\n\n// Note: put the log field first to trigger the CoT\n\nconst commonOutputFields = `\"error\"?: string, // Error messages about unexpected situations, if any. Only think it is an error when the situation is not foreseeable according to the instruction. Use the same language as the user's instruction.\n  \"more_actions_needed_by_instruction\": boolean, // Consider if there is still more action(s) to do after the action in \"Log\" is done, according to the instruction. If so, set this field to true. Otherwise, set it to false.`;\n\nconst vlLocateParam = (vlMode: TVlModeTypes | undefined) => {\n  if (vlMode) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(vlMode)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as any;\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      // Helper function to get type name from zod schema\n      const getTypeName = (field: any): string => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        const actualField = unwrapField(field);\n        const fieldTypeName = actualField._def?.typeName;\n\n        if (fieldTypeName === 'ZodString') return 'string';\n        if (fieldTypeName === 'ZodNumber') return 'number';\n        if (fieldTypeName === 'ZodBoolean') return 'boolean';\n        if (fieldTypeName === 'ZodArray') return 'array';\n        if (fieldTypeName === 'ZodObject') {\n          // Check if this is a passthrough object (like MidsceneLocation)\n          if (ifMidsceneLocatorField(actualField)) {\n            return locatorSchemaTypeDescription;\n          }\n          return 'object';\n        }\n        if (fieldTypeName === 'ZodEnum') {\n          const values =\n            (actualField._def?.values as unknown[] | undefined)\n              ?.map((option: unknown) => String(`'${option}'`))\n              .join(', ') ?? 'enum';\n\n          return `enum(${values})`;\n        }\n        // Handle ZodUnion by taking the first option (for display purposes)\n        if (fieldTypeName === 'ZodUnion') {\n          const options = actualField._def?.options as any[] | undefined;\n          if (options && options.length > 0) {\n            // For unions, list all types\n            const types = options.map((opt: any) => getTypeName(opt));\n            return types.join(' | ');\n          }\n          return 'union';\n        }\n\n        console.warn(\n          'failed to parse Zod type. This may lead to wrong params from the LLM.\\n',\n          actualField._def,\n        );\n        return actualField.toString();\n      };\n\n      // Helper function to get description from zod schema\n      const getDescription = (field: z.ZodTypeAny): string | null => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        // Check for direct description on the original field (wrapper may have description)\n        if ('description' in field) {\n          return field.description || null;\n        }\n\n        const actualField = unwrapField(field);\n\n        // Check for description on the unwrapped field\n        if ('description' in actualField) {\n          return actualField.description || null;\n        }\n\n        // Check for MidsceneLocation fields and add description\n        if (actualField._def?.typeName === 'ZodObject') {\n          if ('midscene_location_field_flag' in actualField._def.shape()) {\n            return 'Location information for the target element';\n          }\n        }\n\n        return null;\n      };\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as any).isOptional === 'function' &&\n            (field as any).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name\n          const typeName = getTypeName(field);\n\n          // Get description\n          const description = getDescription(field as z.ZodTypeAny);\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          if (description) {\n            paramLine += ` // ${description}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      // For simple primitive types, the param should be passed directly as the value\n      const schemaTypeName = schema._def?.typeName;\n      let typeName = 'unknown';\n\n      if (schemaTypeName === 'ZodString') typeName = 'string';\n      else if (schemaTypeName === 'ZodNumber') typeName = 'number';\n      else if (schemaTypeName === 'ZodBoolean') typeName = 'boolean';\n\n      // Get description if available\n      const description = 'description' in schema ? schema.description : null;\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  vlMode,\n  includeBbox,\n}: {\n  actionSpace: DeviceAction<any>[];\n  vlMode: TVlModeTypes | undefined;\n  includeBbox: boolean;\n}) {\n  // Validate parameters: if includeBbox is true, vlMode must be defined\n  if (includeBbox && !vlMode) {\n    throw new Error(\n      'vlMode cannot be undefined when includeBbox is true. A valid vlMode is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? vlMode : undefined),\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const logFieldInstruction = `\n## About the \\`log\\` field (preamble message)\n\nThe \\`log\\` field is a brief preamble message to the user explaining what you’re about to do. It should follow these principles and examples:\n\n- **Use the same language as the user's instruction**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- \"Click the login button\"\n- \"Scroll to find the 'Yes' button in popup\"\n- \"Previous actions failed to find the 'Yes' button, i will try again\"\n- \"Go back to find the login button\"\n`;\n\n  return `\nTarget: User will give you an instruction, some screenshots and previous logs indicating what have been done. Your task is to plan the next one action according to current situation to accomplish the instruction.\n\nPlease tell what the next one action is (or null if no action should be done) to do the tasks the instruction requires. \n\n## Rules\n\n- Don't give extra actions or plans beyond the instruction. For example, don't try to submit the form if the instruction is only to fill something.\n- Give just the next ONE action you should do\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully before performing the next step\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the \"error\" field to the error message.\n- If there is nothing to do but waiting, set the \"sleep\" field to the positive waiting time in milliseconds and null for the \"action\" field.\n- When the next step is to assert something, this is a very important step, you should think about it carefully and give a solid result. Write your result in the \"log\" field like this: \"Assert: <condition>. I think <...>, so the result is <true / false>\". You don't need to give the next one action when you are asserting something. If the assertion result is false, think this an fatal error and set the reason into the \"error\" field. If the assertion result is true, you can continue to the next step.\n\n## Supporting actions\n${actionList}\n\n${logFieldInstruction}\n\n## Return format\n\nReturn in JSON format:\n{\n  \"log\": string, // a brief preamble to the user explaining what you’re about to do\n  ${commonOutputFields}\n  \"action\": \n    {\n      // one of the supporting actions\n    } | null,\n  ,\n  \"sleep\"?: number, // The sleep time after the action, in milliseconds.\n}\n`;\n}\n\nexport const planSchema: ResponseFormatJSONSchema = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'action_items',\n    strict: false,\n    schema: {\n      type: 'object',\n      strict: false,\n      properties: {\n        actions: {\n          type: 'array',\n          items: {\n            type: 'object',\n            strict: false,\n            properties: {\n              thought: {\n                type: 'string',\n                description:\n                  'Reasons for generating this task, and why this task is feasible on this page',\n              },\n              type: {\n                type: 'string',\n                description: 'Type of action',\n              },\n              param: {\n                anyOf: [\n                  { type: 'null' },\n                  {\n                    type: 'object',\n                    additionalProperties: true,\n                  },\n                ],\n                description: 'Parameter of the action',\n              },\n              locate: {\n                type: ['object', 'null'],\n                properties: {\n                  id: { type: 'string' },\n                  prompt: { type: 'string' },\n                },\n                required: ['id', 'prompt'],\n                additionalProperties: false,\n                description: 'Location information for the target element',\n              },\n            },\n            required: ['thought', 'type', 'param', 'locate'],\n            additionalProperties: false,\n          },\n          description: 'List of actions to be performed',\n        },\n        more_actions_needed_by_instruction: {\n          type: 'boolean',\n          description:\n            'If all the actions described in the instruction have been covered by this action and logs, set this field to false.',\n        },\n        log: {\n          type: 'string',\n          description:\n            'Log what these planned actions do. Do not include further actions that have not been planned.',\n        },\n        error: {\n          type: ['string', 'null'],\n          description: 'Error messages about unexpected situations',\n        },\n      },\n      required: [\n        'actions',\n        'more_actions_needed_by_instruction',\n        'log',\n        'error',\n      ],\n      additionalProperties: false,\n    },\n  },\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","commonOutputFields","vlLocateParam","vlMode","bboxDescription","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","getTypeName","field","unwrapField","f","typeName","actualField","fieldTypeName","ifMidsceneLocatorField","values","option","String","options","types","opt","console","getDescription","isOptional","keyWithOptional","description","paramLine","line","schemaTypeName","paramDescription","systemPromptToTaskPlanning","actionSpace","includeBbox","Error","actionDescriptionList","undefined","actionList","logFieldInstruction","planSchema"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACGA,MAAMI,qBAAqB,CAAC;+NACmM,CAAC;AAEhO,MAAMC,gBAAgB,CAACC;IACrB,IAAIA,QACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,SAAS;IAElG,OAAO;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC;IAEA,MAAMC,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QACjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAG1B,MAAMG,cAAc,CAACC;gBAEnB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAEA,MAAME,cAAcH,YAAYD;gBAChC,MAAMK,gBAAgBD,YAAY,IAAI,EAAE;gBAExC,IAAIC,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,iBAAlBA,eAAgC,OAAO;gBAC3C,IAAIA,AAAkB,eAAlBA,eAA8B,OAAO;gBACzC,IAAIA,AAAkB,gBAAlBA,eAA+B;oBAEjC,IAAIC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBF,cACzB,OAAOZ;oBAET,OAAO;gBACT;gBACA,IAAIa,AAAkB,cAAlBA,eAA6B;oBAC/B,MAAME,SACHH,YAAY,IAAI,EAAE,QACf,IAAI,CAACI,SAAoBC,OAAO,CAAC,CAAC,EAAED,OAAO,CAAC,CAAC,GAC9C,KAAK,SAAS;oBAEnB,OAAO,CAAC,KAAK,EAAED,OAAO,CAAC,CAAC;gBAC1B;gBAEA,IAAIF,AAAkB,eAAlBA,eAA8B;oBAChC,MAAMK,UAAUN,YAAY,IAAI,EAAE;oBAClC,IAAIM,WAAWA,QAAQ,MAAM,GAAG,GAAG;wBAEjC,MAAMC,QAAQD,QAAQ,GAAG,CAAC,CAACE,MAAab,YAAYa;wBACpD,OAAOD,MAAM,IAAI,CAAC;oBACpB;oBACA,OAAO;gBACT;gBAEAE,QAAQ,IAAI,CACV,2EACAT,YAAY,IAAI;gBAElB,OAAOA,YAAY,QAAQ;YAC7B;YAGA,MAAMU,iBAAiB,CAACd;gBAEtB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAGA,IAAI,iBAAiBF,OACnB,OAAOA,MAAM,WAAW,IAAI;gBAG9B,MAAMI,cAAcH,YAAYD;gBAGhC,IAAI,iBAAiBI,aACnB,OAAOA,YAAY,WAAW,IAAI;gBAIpC,IAAIA,YAAY,IAAI,EAAE,aAAa,aACjC;oBAAA,IAAI,kCAAkCA,YAAY,IAAI,CAAC,KAAK,IAC1D,OAAO;gBACT;gBAGF,OAAO;YACT;YAEA,KAAK,MAAM,CAACvB,KAAKmB,MAAM,IAAIlB,OAAO,OAAO,CAACgB,OACxC,IAAIE,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMe,aACJ,AAAqC,cAArC,OAAQf,MAAc,UAAU,IAC/BA,MAAc,UAAU;gBAC3B,MAAMgB,kBAAkBD,aAAa,GAAGlC,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMsB,WAAWJ,YAAYC;gBAG7B,MAAMiB,cAAcH,eAAed;gBAGnC,IAAIkB,YAAY,GAAGF,gBAAgB,EAAE,EAAEb,UAAU;gBACjD,IAAIc,aACFC,aAAa,CAAC,IAAI,EAAED,aAAa;gBAGnCtB,WAAW,IAAI,CAACuB;YAClB;YAIF,IAAIvB,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACwB;oBAClBzB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEyB,MAAM;gBAC3B;YACF;QACF,OAAO;YAGL,MAAMC,iBAAiBxB,OAAO,IAAI,EAAE;YACpC,IAAIO,WAAW;YAEf,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC1C,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC/C,IAAIiB,AAAmB,iBAAnBA,gBAAiCjB,WAAW;YAGrD,MAAMc,cAAc,iBAAiBrB,SAASA,OAAO,WAAW,GAAG;YAGnE,IAAIyB,mBAAmB,CAAC,SAAS,EAAElB,UAAU;YAC7C,IAAIc,aACFI,oBAAoB,CAAC,IAAI,EAAEJ,aAAa;YAE1CI,oBAAoB;YAEpB3B,OAAO,IAAI,CAAC2B;QACd;IACF;IAEA,OAAO,CAAC,EAAE,EAAE9B,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAe6B,2BAA2B,EAC/CC,WAAW,EACXnC,MAAM,EACNoC,WAAW,EAKZ;IAEC,IAAIA,eAAe,CAACpC,QAClB,MAAM,IAAIqC,MACR;IAIJ,MAAMC,wBAAwBH,YAAY,GAAG,CAAC,CAAChC,SACtCD,qBACLC,QACAJ,cAAcqC,cAAcpC,SAASuC;IAGzC,MAAMC,aAAaF,sBAAsB,IAAI,CAAC;IAE9C,MAAMG,sBAAsB,CAAC;;;;;;;;;;;;;;;AAe/B,CAAC;IAEC,OAAO,CAAC;;;;;;;;;;;;;;;;AAgBV,EAAED,WAAW;;AAEb,EAAEC,oBAAoB;;;;;;;EAOpB,EAAE3C,mBAAmB;;;;;;;;AAQvB,CAAC;AACD;AAEO,MAAM4C,aAAuC;IAClD,MAAM;IACN,aAAa;QACX,MAAM;QACN,QAAQ;QACR,QAAQ;YACN,MAAM;YACN,QAAQ;YACR,YAAY;gBACV,SAAS;oBACP,MAAM;oBACN,OAAO;wBACL,MAAM;wBACN,QAAQ;wBACR,YAAY;4BACV,SAAS;gCACP,MAAM;gCACN,aACE;4BACJ;4BACA,MAAM;gCACJ,MAAM;gCACN,aAAa;4BACf;4BACA,OAAO;gCACL,OAAO;oCACL;wCAAE,MAAM;oCAAO;oCACf;wCACE,MAAM;wCACN,sBAAsB;oCACxB;iCACD;gCACD,aAAa;4BACf;4BACA,QAAQ;gCACN,MAAM;oCAAC;oCAAU;iCAAO;gCACxB,YAAY;oCACV,IAAI;wCAAE,MAAM;oCAAS;oCACrB,QAAQ;wCAAE,MAAM;oCAAS;gCAC3B;gCACA,UAAU;oCAAC;oCAAM;iCAAS;gCAC1B,sBAAsB;gCACtB,aAAa;4BACf;wBACF;wBACA,UAAU;4BAAC;4BAAW;4BAAQ;4BAAS;yBAAS;wBAChD,sBAAsB;oBACxB;oBACA,aAAa;gBACf;gBACA,oCAAoC;oBAClC,MAAM;oBACN,aACE;gBACJ;gBACA,KAAK;oBACH,MAAM;oBACN,aACE;gBACJ;gBACA,OAAO;oBACL,MAAM;wBAAC;wBAAU;qBAAO;oBACxB,aAAa;gBACf;YACF;YACA,UAAU;gBACR;gBACA;gBACA;gBACA;aACD;YACD,sBAAsB;QACxB;IACF;AACF"}
+{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/types';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport type { z } from 'zod';\nimport { ifMidsceneLocatorField } from '../../common';\nimport { bboxDescription } from './common';\n\n// Note: put the log field first to trigger the CoT\n\nconst commonOutputFields = `\"error\"?: string, // Error messages about unexpected situations, if any. Only think it is an error when the situation is not foreseeable according to the instruction. Use the same language as the user's instruction.\n  \"more_actions_needed_by_instruction\": boolean, // Consider if there is still more action(s) to do after the action in \"Log\" is done, according to the instruction. If so, set this field to true. Otherwise, set it to false.`;\n\nconst vlLocateParam = (vlMode: TVlModeTypes | undefined) => {\n  if (vlMode) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(vlMode)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as any;\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      // Helper function to get type name from zod schema\n      const getTypeName = (field: any): string => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        const actualField = unwrapField(field);\n        const fieldTypeName = actualField._def?.typeName;\n\n        if (fieldTypeName === 'ZodString') return 'string';\n        if (fieldTypeName === 'ZodNumber') return 'number';\n        if (fieldTypeName === 'ZodBoolean') return 'boolean';\n        if (fieldTypeName === 'ZodArray') return 'array';\n        if (fieldTypeName === 'ZodObject') {\n          // Check if this is a passthrough object (like MidsceneLocation)\n          if (ifMidsceneLocatorField(actualField)) {\n            return locatorSchemaTypeDescription;\n          }\n          return 'object';\n        }\n        if (fieldTypeName === 'ZodEnum') {\n          const values =\n            (actualField._def?.values as unknown[] | undefined)\n              ?.map((option: unknown) => String(`'${option}'`))\n              .join(', ') ?? 'enum';\n\n          return `enum(${values})`;\n        }\n        // Handle ZodUnion by taking the first option (for display purposes)\n        if (fieldTypeName === 'ZodUnion') {\n          const options = actualField._def?.options as any[] | undefined;\n          if (options && options.length > 0) {\n            // For unions, list all types\n            const types = options.map((opt: any) => getTypeName(opt));\n            return types.join(' | ');\n          }\n          return 'union';\n        }\n\n        console.warn(\n          'failed to parse Zod type. This may lead to wrong params from the LLM.\\n',\n          actualField._def,\n        );\n        return actualField.toString();\n      };\n\n      // Helper function to get description from zod schema\n      const getDescription = (field: z.ZodTypeAny): string | null => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        // Check for direct description on the original field (wrapper may have description)\n        if ('description' in field) {\n          return field.description || null;\n        }\n\n        const actualField = unwrapField(field);\n\n        // Check for description on the unwrapped field\n        if ('description' in actualField) {\n          return actualField.description || null;\n        }\n\n        // Check for MidsceneLocation fields and add description\n        if (actualField._def?.typeName === 'ZodObject') {\n          if ('midscene_location_field_flag' in actualField._def.shape()) {\n            return 'Location information for the target element';\n          }\n        }\n\n        return null;\n      };\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as any).isOptional === 'function' &&\n            (field as any).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name\n          const typeName = getTypeName(field);\n\n          // Get description\n          const description = getDescription(field as z.ZodTypeAny);\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          if (description) {\n            paramLine += ` // ${description}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      // For simple primitive types, the param should be passed directly as the value\n      const schemaTypeName = schema._def?.typeName;\n      let typeName = 'unknown';\n\n      if (schemaTypeName === 'ZodString') typeName = 'string';\n      else if (schemaTypeName === 'ZodNumber') typeName = 'number';\n      else if (schemaTypeName === 'ZodBoolean') typeName = 'boolean';\n\n      // Get description if available\n      const description = 'description' in schema ? schema.description : null;\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  vlMode,\n  includeBbox,\n}: {\n  actionSpace: DeviceAction<any>[];\n  vlMode: TVlModeTypes | undefined;\n  includeBbox: boolean;\n}) {\n  // Validate parameters: if includeBbox is true, vlMode must be defined\n  if (includeBbox && !vlMode) {\n    throw new Error(\n      'vlMode cannot be undefined when includeBbox is true. A valid vlMode is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? vlMode : undefined),\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const logFieldInstruction = `\n## About the \\`log\\` field (preamble message)\n\nThe \\`log\\` field is a brief preamble message to the user explaining what you’re about to do. It should follow these principles and examples:\n\n- **Use the same language as the user's instruction**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- \"Click the login button\"\n- \"Scroll to find the 'Yes' button in popup\"\n- \"Previous actions failed to find the 'Yes' button, i will try again\"\n- \"Go back to find the login button\"\n`;\n\n  return `\nTarget: User will give you an instruction, some screenshots and previous logs indicating what have been done. Your task is to plan the next one action according to current situation to accomplish the instruction.\n\nPlease tell what the next one action is (or null if no action should be done) to do the tasks the instruction requires. \n\n## Rules\n\n- Don't give extra actions or plans beyond the instruction. For example, don't try to submit the form if the instruction is only to fill something.\n- Give just the next ONE action you should do\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully before performing the next step\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the \"error\" field to the error message.\n- If there is nothing to do but waiting, set the \"sleep\" field to the positive waiting time in milliseconds and null for the \"action\" field.\n- Assertions are also important steps. When getting the assertion instruction, a solid conclusion is required. You should explicitly state your conclusion by calling the \"Print_Assert_Result\" action.\n\n## Supporting actions\n${actionList}\n\n${logFieldInstruction}\n\n## Return format\n\nReturn in JSON format:\n{\n  \"log\": string, // a brief preamble to the user explaining what you’re about to do\n  ${commonOutputFields}\n  \"action\": \n    {\n      \"type\": string, // the type of the action\n      \"param\"?: { // The parameter of the action, if any\n         // k-v style parameter fields\n      }, \n    } | null,\n  ,\n  \"sleep\"?: number, // The sleep time after the action, in milliseconds.\n}\n\nFor example, if the instruction is to login and the form has already been filled, this is a valid return value:\n\n{\n  \"log\": \"Click the login button\",\n  \"more_actions_needed_by_instruction\": false,\n  \"action\": {\n    \"type\": \"Tap\",\n    \"param\": {\n      \"locate\": { \n        \"prompt\": \"The login button\"${vlMode ? `, \"bbox\": [100, 200, 300, 400]` : ''}\n      }\n    }\n  }\n`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","commonOutputFields","vlLocateParam","vlMode","bboxDescription","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","getTypeName","field","unwrapField","f","typeName","actualField","fieldTypeName","ifMidsceneLocatorField","values","option","String","options","types","opt","console","getDescription","isOptional","keyWithOptional","description","paramLine","line","schemaTypeName","paramDescription","systemPromptToTaskPlanning","actionSpace","includeBbox","Error","actionDescriptionList","undefined","actionList","logFieldInstruction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACEA,MAAMI,qBAAqB,CAAC;+NACmM,CAAC;AAEhO,MAAMC,gBAAgB,CAACC;IACrB,IAAIA,QACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,SAAS;IAElG,OAAO;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC;IAEA,MAAMC,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QACjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAG1B,MAAMG,cAAc,CAACC;gBAEnB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAEA,MAAME,cAAcH,YAAYD;gBAChC,MAAMK,gBAAgBD,YAAY,IAAI,EAAE;gBAExC,IAAIC,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,iBAAlBA,eAAgC,OAAO;gBAC3C,IAAIA,AAAkB,eAAlBA,eAA8B,OAAO;gBACzC,IAAIA,AAAkB,gBAAlBA,eAA+B;oBAEjC,IAAIC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBF,cACzB,OAAOZ;oBAET,OAAO;gBACT;gBACA,IAAIa,AAAkB,cAAlBA,eAA6B;oBAC/B,MAAME,SACHH,YAAY,IAAI,EAAE,QACf,IAAI,CAACI,SAAoBC,OAAO,CAAC,CAAC,EAAED,OAAO,CAAC,CAAC,GAC9C,KAAK,SAAS;oBAEnB,OAAO,CAAC,KAAK,EAAED,OAAO,CAAC,CAAC;gBAC1B;gBAEA,IAAIF,AAAkB,eAAlBA,eAA8B;oBAChC,MAAMK,UAAUN,YAAY,IAAI,EAAE;oBAClC,IAAIM,WAAWA,QAAQ,MAAM,GAAG,GAAG;wBAEjC,MAAMC,QAAQD,QAAQ,GAAG,CAAC,CAACE,MAAab,YAAYa;wBACpD,OAAOD,MAAM,IAAI,CAAC;oBACpB;oBACA,OAAO;gBACT;gBAEAE,QAAQ,IAAI,CACV,2EACAT,YAAY,IAAI;gBAElB,OAAOA,YAAY,QAAQ;YAC7B;YAGA,MAAMU,iBAAiB,CAACd;gBAEtB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAGA,IAAI,iBAAiBF,OACnB,OAAOA,MAAM,WAAW,IAAI;gBAG9B,MAAMI,cAAcH,YAAYD;gBAGhC,IAAI,iBAAiBI,aACnB,OAAOA,YAAY,WAAW,IAAI;gBAIpC,IAAIA,YAAY,IAAI,EAAE,aAAa,aACjC;oBAAA,IAAI,kCAAkCA,YAAY,IAAI,CAAC,KAAK,IAC1D,OAAO;gBACT;gBAGF,OAAO;YACT;YAEA,KAAK,MAAM,CAACvB,KAAKmB,MAAM,IAAIlB,OAAO,OAAO,CAACgB,OACxC,IAAIE,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMe,aACJ,AAAqC,cAArC,OAAQf,MAAc,UAAU,IAC/BA,MAAc,UAAU;gBAC3B,MAAMgB,kBAAkBD,aAAa,GAAGlC,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMsB,WAAWJ,YAAYC;gBAG7B,MAAMiB,cAAcH,eAAed;gBAGnC,IAAIkB,YAAY,GAAGF,gBAAgB,EAAE,EAAEb,UAAU;gBACjD,IAAIc,aACFC,aAAa,CAAC,IAAI,EAAED,aAAa;gBAGnCtB,WAAW,IAAI,CAACuB;YAClB;YAIF,IAAIvB,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACwB;oBAClBzB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEyB,MAAM;gBAC3B;YACF;QACF,OAAO;YAGL,MAAMC,iBAAiBxB,OAAO,IAAI,EAAE;YACpC,IAAIO,WAAW;YAEf,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC1C,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC/C,IAAIiB,AAAmB,iBAAnBA,gBAAiCjB,WAAW;YAGrD,MAAMc,cAAc,iBAAiBrB,SAASA,OAAO,WAAW,GAAG;YAGnE,IAAIyB,mBAAmB,CAAC,SAAS,EAAElB,UAAU;YAC7C,IAAIc,aACFI,oBAAoB,CAAC,IAAI,EAAEJ,aAAa;YAE1CI,oBAAoB;YAEpB3B,OAAO,IAAI,CAAC2B;QACd;IACF;IAEA,OAAO,CAAC,EAAE,EAAE9B,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAe6B,2BAA2B,EAC/CC,WAAW,EACXnC,MAAM,EACNoC,WAAW,EAKZ;IAEC,IAAIA,eAAe,CAACpC,QAClB,MAAM,IAAIqC,MACR;IAIJ,MAAMC,wBAAwBH,YAAY,GAAG,CAAC,CAAChC,SACtCD,qBACLC,QACAJ,cAAcqC,cAAcpC,SAASuC;IAGzC,MAAMC,aAAaF,sBAAsB,IAAI,CAAC;IAE9C,MAAMG,sBAAsB,CAAC;;;;;;;;;;;;;;;AAe/B,CAAC;IAEC,OAAO,CAAC;;;;;;;;;;;;;;;;AAgBV,EAAED,WAAW;;AAEb,EAAEC,oBAAoB;;;;;;;EAOpB,EAAE3C,mBAAmB;;;;;;;;;;;;;;;;;;;;;oCAqBa,EAAEE,SAAS,mCAAmC,GAAG;;;;AAIrF,CAAC;AACD"}

package/dist/lib/ai-model/prompt/llm-section-locator.js

@@ -68,10 +68,10 @@ If the description is "delete button on the second row with title 'Peter'", retu
 const sectionLocatorInstruction = (sectionDescription)=>`Find section containing: ${sectionDescription}`;
 exports.sectionLocatorInstruction = __webpack_exports__.sectionLocatorInstruction;
 exports.systemPromptToLocateSection = __webpack_exports__.systemPromptToLocateSection;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "sectionLocatorInstruction",
     "systemPromptToLocateSection"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/order-sensitive-judge.js

@@ -61,10 +61,10 @@ Return true if the description is order-sensitive, false otherwise.
 const orderSensitiveJudgePrompt = (description)=>`Analyze this element description: "${description}"`;
 exports.orderSensitiveJudgePrompt = __webpack_exports__.orderSensitiveJudgePrompt;
 exports.systemPromptToJudgeOrderSensitive = __webpack_exports__.systemPromptToJudgeOrderSensitive;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "orderSensitiveJudgePrompt",
     "systemPromptToJudgeOrderSensitive"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/playwright-generator.js

@@ -159,7 +159,7 @@ exports.getScreenshotsForLLM = __webpack_exports__.getScreenshotsForLLM;
 exports.prepareEventSummary = __webpack_exports__.prepareEventSummary;
 exports.processEventsForLLM = __webpack_exports__.processEventsForLLM;
 exports.validateEvents = __webpack_exports__.validateEvents;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "createEventCounts",
     "createMessageContent",
     "extractInputDescriptions",
@@ -170,7 +170,7 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "prepareEventSummary",
     "processEventsForLLM",
     "validateEvents"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/ui-tars-locator.js

@@ -58,9 +58,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or
     `;
 }
 exports.systemPromptToLocateElementPosition = __webpack_exports__.systemPromptToLocateElementPosition;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "systemPromptToLocateElementPosition"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/ui-tars-planning.js

@@ -62,10 +62,10 @@ finished(content='xxx') # Use escape characters \\', \\", and \\n in content par
 const getSummary = (prediction)=>prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, '').trim();
 exports.getSummary = __webpack_exports__.getSummary;
 exports.getUiTarsPlanningPrompt = __webpack_exports__.getUiTarsPlanningPrompt;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "getSummary",
     "getUiTarsPlanningPrompt"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/util.js

@@ -73,14 +73,14 @@ exports.describeUserPage = __webpack_exports__.describeUserPage;
 exports.distance = __webpack_exports__.distance;
 exports.distanceThreshold = __webpack_exports__.distanceThreshold;
 exports.samplePageDescription = __webpack_exports__.samplePageDescription;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "describeElement",
     "describeSize",
     "describeUserPage",
     "distance",
     "distanceThreshold",
     "samplePageDescription"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });