@midscene/core 1.2.2-beta-20260116114131.0 → 1.2.2-beta-20260119114334.0

package/dist/lib/ai-model/auto-glm/index.js

@@ -26,6 +26,7 @@ __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     parseAction: ()=>external_parser_js_namespaceObject.parseAction,
     isAutoGLM: ()=>external_util_js_namespaceObject.isAutoGLM,
+    isUITars: ()=>external_util_js_namespaceObject.isUITars,
     autoGLMPlanning: ()=>external_planning_js_namespaceObject.autoGLMPlanning,
     getAutoGLMPlanPrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMPlanPrompt,
     getAutoGLMLocatePrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMLocatePrompt,
@@ -42,6 +43,7 @@ exports.autoGLMPlanning = __webpack_exports__.autoGLMPlanning;
 exports.getAutoGLMLocatePrompt = __webpack_exports__.getAutoGLMLocatePrompt;
 exports.getAutoGLMPlanPrompt = __webpack_exports__.getAutoGLMPlanPrompt;
 exports.isAutoGLM = __webpack_exports__.isAutoGLM;
+exports.isUITars = __webpack_exports__.isUITars;
 exports.parseAction = __webpack_exports__.parseAction;
 exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
 exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
@@ -51,6 +53,7 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "getAutoGLMLocatePrompt",
     "getAutoGLMPlanPrompt",
     "isAutoGLM",
+    "isUITars",
     "parseAction",
     "parseAutoGLMLocateResponse",
     "parseAutoGLMResponse",

package/dist/lib/ai-model/auto-glm/planning.js

@@ -34,7 +34,7 @@ const external_prompt_js_namespaceObject = require("./prompt.js");
 const debug = (0, logger_namespaceObject.getDebug)('auto-glm-planning');
 async function autoGLMPlanning(userInstruction, options) {
     const { conversationHistory, context, modelConfig, actionContext } = options;
-    const systemPrompt = (0, external_prompt_js_namespaceObject.getAutoGLMPlanPrompt)(modelConfig.vlMode) + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>` : '');
+    const systemPrompt = (0, external_prompt_js_namespaceObject.getAutoGLMPlanPrompt)(modelConfig.modelFamily) + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>` : '');
     const imagePayload = context.screenshot;
     conversationHistory.append({
         role: 'user',

package/dist/lib/ai-model/auto-glm/planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAIResponse, UIContext } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ConversationHistory } from '../conversation-history';\nimport { callAIWithStringResponse } from '../service-caller/index';\nimport { transformAutoGLMAction } from './actions';\nimport { parseAction, parseAutoGLMResponse } from './parser';\nimport { getAutoGLMPlanPrompt } from './prompt';\n\nconst debug = getDebug('auto-glm-planning');\n\nexport async function autoGLMPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n\n  const systemPrompt =\n    getAutoGLMPlanPrompt(modelConfig.vlMode) +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>`\n      : '');\n\n  const imagePayload = context.screenshot;\n\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'text', text: userInstruction }],\n  });\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'image_url', image_url: { url: imagePayload.base64 } }],\n  });\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...conversationHistory.snapshot(1),\n  ];\n\n  const { content: rawResponse, usage } = await callAIWithStringResponse(\n    msgs,\n    modelConfig,\n  );\n\n  debug('autoGLMPlanning rawResponse:', rawResponse);\n\n  const parsedResponse = parseAutoGLMResponse(rawResponse);\n  debug('thinking in response:', parsedResponse.think);\n  debug('action in response:', parsedResponse.content);\n\n  let transformedActions = [] as ReturnType<typeof transformAutoGLMAction>;\n\n  const parsedAction = parseAction(parsedResponse);\n  debug('Parsed action object:', parsedAction);\n  transformedActions = transformAutoGLMAction(parsedAction, context.size);\n  debug('Transformed actions:', transformedActions);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`,\n  });\n\n  const shouldContinuePlanning = !parsedResponse.content.startsWith('finish(');\n\n  return {\n    actions: transformedActions,\n    log: rawResponse,\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(rawResponse, undefined, 2),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","autoGLMPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","systemPrompt","getAutoGLMPlanPrompt","imagePayload","msgs","rawResponse","usage","callAIWithStringResponse","parsedResponse","parseAutoGLMResponse","transformedActions","parsedAction","parseAction","transformAutoGLMAction","shouldContinuePlanning","JSON","undefined"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACIA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,gBACpBC,eAAuB,EACvBC,OAKC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IAErE,MAAMK,eACJC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBH,YAAY,MAAM,IACtCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,0BAA0B,CAAC,GACrE,EAAC;IAEP,MAAMG,eAAeL,QAAQ,UAAU;IAEvCD,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAQ,MAAMF;YAAgB;SAAE;IACpD;IACAE,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKM,aAAa,MAAM;gBAAC;YAAE;SAAE;IAC3E;IAEA,MAAMC,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASH;QAAa;WACrCJ,oBAAoB,QAAQ,CAAC;KACjC;IAED,MAAM,EAAE,SAASQ,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC5CH,MACAL;IAGFP,MAAM,gCAAgCa;IAEtC,MAAMG,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBJ;IAC5Cb,MAAM,yBAAyBgB,eAAe,KAAK;IACnDhB,MAAM,uBAAuBgB,eAAe,OAAO;IAEnD,IAAIE,qBAAqB,EAAE;IAE3B,MAAMC,eAAeC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYJ;IACjChB,MAAM,yBAAyBmB;IAC/BD,qBAAqBG,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EAAuBF,cAAcb,QAAQ,IAAI;IACtEN,MAAM,wBAAwBkB;IAE9Bb,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS,CAAC,OAAO,EAAEW,eAAe,KAAK,CAAC,gBAAgB,EAAEA,eAAe,OAAO,CAAC,SAAS,CAAC;IAC7F;IAEA,MAAMM,yBAAyB,CAACN,eAAe,OAAO,CAAC,UAAU,CAAC;IAElE,OAAO;QACL,SAASE;QACT,KAAKL;QACLC;QACAQ;QACA,aAAaC,KAAK,SAAS,CAACV,aAAaW,QAAW;IACtD;AACF"}
+{"version":3,"file":"ai-model/auto-glm/planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAIResponse, UIContext } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ConversationHistory } from '../conversation-history';\nimport { callAIWithStringResponse } from '../service-caller/index';\nimport { transformAutoGLMAction } from './actions';\nimport { parseAction, parseAutoGLMResponse } from './parser';\nimport { getAutoGLMPlanPrompt } from './prompt';\n\nconst debug = getDebug('auto-glm-planning');\n\nexport async function autoGLMPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n\n  const systemPrompt =\n    getAutoGLMPlanPrompt(modelConfig.modelFamily) +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>`\n      : '');\n\n  const imagePayload = context.screenshot;\n\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'text', text: userInstruction }],\n  });\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'image_url', image_url: { url: imagePayload.base64 } }],\n  });\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...conversationHistory.snapshot(1),\n  ];\n\n  const { content: rawResponse, usage } = await callAIWithStringResponse(\n    msgs,\n    modelConfig,\n  );\n\n  debug('autoGLMPlanning rawResponse:', rawResponse);\n\n  const parsedResponse = parseAutoGLMResponse(rawResponse);\n  debug('thinking in response:', parsedResponse.think);\n  debug('action in response:', parsedResponse.content);\n\n  let transformedActions = [] as ReturnType<typeof transformAutoGLMAction>;\n\n  const parsedAction = parseAction(parsedResponse);\n  debug('Parsed action object:', parsedAction);\n  transformedActions = transformAutoGLMAction(parsedAction, context.size);\n  debug('Transformed actions:', transformedActions);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`,\n  });\n\n  const shouldContinuePlanning = !parsedResponse.content.startsWith('finish(');\n\n  return {\n    actions: transformedActions,\n    log: rawResponse,\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(rawResponse, undefined, 2),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","autoGLMPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","systemPrompt","getAutoGLMPlanPrompt","imagePayload","msgs","rawResponse","usage","callAIWithStringResponse","parsedResponse","parseAutoGLMResponse","transformedActions","parsedAction","parseAction","transformAutoGLMAction","shouldContinuePlanning","JSON","undefined"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACIA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,gBACpBC,eAAuB,EACvBC,OAKC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IAErE,MAAMK,eACJC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBH,YAAY,WAAW,IAC3CC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,0BAA0B,CAAC,GACrE,EAAC;IAEP,MAAMG,eAAeL,QAAQ,UAAU;IAEvCD,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAQ,MAAMF;YAAgB;SAAE;IACpD;IACAE,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKM,aAAa,MAAM;gBAAC;YAAE;SAAE;IAC3E;IAEA,MAAMC,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASH;QAAa;WACrCJ,oBAAoB,QAAQ,CAAC;KACjC;IAED,MAAM,EAAE,SAASQ,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC5CH,MACAL;IAGFP,MAAM,gCAAgCa;IAEtC,MAAMG,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBJ;IAC5Cb,MAAM,yBAAyBgB,eAAe,KAAK;IACnDhB,MAAM,uBAAuBgB,eAAe,OAAO;IAEnD,IAAIE,qBAAqB,EAAE;IAE3B,MAAMC,eAAeC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYJ;IACjChB,MAAM,yBAAyBmB;IAC/BD,qBAAqBG,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EAAuBF,cAAcb,QAAQ,IAAI;IACtEN,MAAM,wBAAwBkB;IAE9Bb,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS,CAAC,OAAO,EAAEW,eAAe,KAAK,CAAC,gBAAgB,EAAEA,eAAe,OAAO,CAAC,SAAS,CAAC;IAC7F;IAEA,MAAMM,yBAAyB,CAACN,eAAe,OAAO,CAAC,UAAU,CAAC;IAElE,OAAO;QACL,SAASE;QACT,KAAKL;QACLC;QACAQ;QACA,aAAaC,KAAK,SAAS,CAACV,aAAaW,QAAW;IACtD;AACF"}

package/dist/lib/ai-model/auto-glm/prompt.js

@@ -189,13 +189,13 @@ const getAutoGLMChinesePlanPrompt = ()=>`
 17. 如果没有合适的搜索结果，可能是因为搜索页面不对，请返回到搜索页面的上一级尝试重新搜索，如果尝试三次返回上一级搜索后仍然没有符合要求的结果，执行 finish(message="原因")。
 18. 在结束任务前请一定要仔细检查任务是否完整准确的完成，如果出现错选、漏选、多选的情况，请返回之前的步骤进行纠正。
 `;
-const getAutoGLMPlanPrompt = (vlMode)=>{
-    if ('auto-glm-multilingual' === vlMode) return getAutoGLMMultilingualPlanPrompt();
-    if ('auto-glm' === vlMode) return getAutoGLMChinesePlanPrompt();
-    throw new Error(`Unsupported vlMode for Auto-GLM plan prompt: ${vlMode}`);
+const getAutoGLMPlanPrompt = (modelFamily)=>{
+    if ('auto-glm-multilingual' === modelFamily) return getAutoGLMMultilingualPlanPrompt();
+    if ('auto-glm' === modelFamily) return getAutoGLMChinesePlanPrompt();
+    throw new Error(`Unsupported modelFamily for Auto-GLM plan prompt: ${modelFamily}`);
 };
-const getAutoGLMLocatePrompt = (vlMode)=>{
-    if ('auto-glm-multilingual' === vlMode) return `
+const getAutoGLMLocatePrompt = (modelFamily)=>{
+    if ('auto-glm-multilingual' === modelFamily) return `
 The current date: ${getMultilingualFormattedDate()}
 
 # Setup
@@ -225,7 +225,7 @@ Your output should STRICTLY follow the format:
 REMEMBER:
 - Your goal is to locate and tap the UI element specified by the user (e.g., button, icon, link, etc.). Do not attempt any other actions.
   `;
-    if ('auto-glm' === vlMode) return `
+    if ('auto-glm' === modelFamily) return `
 今天的日期是: ${getChineseFormattedDate()}
 
 你是一个智能体分析专家，可以根据操作历史和当前状态图执行一系列操作来完成任务。
@@ -244,7 +244,7 @@ REMEMBER:
 必须遵循的规则：
 - 你的目标是定位并点击用户指定的UI元素（例如按钮、图标、链接等），请不要尝试任何其他的操作。
     `;
-    throw new Error(`Unsupported vlMode for Auto-GLM locate prompt: ${vlMode}`);
+    throw new Error(`Unsupported modelFamily for Auto-GLM locate prompt: ${modelFamily}`);
 };
 exports.getAutoGLMLocatePrompt = __webpack_exports__.getAutoGLMLocatePrompt;
 exports.getAutoGLMPlanPrompt = __webpack_exports__.getAutoGLMPlanPrompt;

package/dist/lib/ai-model/auto-glm/prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/prompt.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/prompt.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/**\n * Auto-GLM Prompt Templates\n *\n * Portions of this file are derived from Open-AutoGLM\n * Copyright (c) 2024 zai-org\n * Licensed under the Apache License, Version 2.0\n *\n * Source: https://github.com/zai-org/Open-AutoGLM\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n * Modifications:\n * - Adapted prompts for Midscene.js integration\n */\n\nimport type { TVlModeTypes } from '@midscene/shared/env';\n\n/**\n * Get formatted date string for system prompts\n * @returns Formatted date string like \"2026-01-12, Sunday\"\n */\nfunction getMultilingualFormattedDate(): string {\n  const today = new Date();\n  const year = today.getFullYear();\n  const month = String(today.getMonth() + 1).padStart(2, '0');\n  const date = String(today.getDate()).padStart(2, '0');\n  const dayOfWeek = [\n    'Sunday',\n    'Monday',\n    'Tuesday',\n    'Wednesday',\n    'Thursday',\n    'Friday',\n    'Saturday',\n  ][today.getDay()];\n\n  return `${year}-${month}-${date}, ${dayOfWeek}`;\n}\n\n/**\n * Get formatted Chinese date (e.g., \"2026年01月13日 星期一\")\n */\nfunction getChineseFormattedDate(): string {\n  const today = new Date();\n  const year = today.getFullYear();\n  const month = String(today.getMonth() + 1).padStart(2, '0');\n  const date = String(today.getDate()).padStart(2, '0');\n  const weekdayNames = [\n    '星期日',\n    '星期一',\n    '星期二',\n    '星期三',\n    '星期四',\n    '星期五',\n    '星期六',\n  ];\n  const weekday = weekdayNames[today.getDay()];\n\n  return `${year}年${month}月${date}日 ${weekday}`;\n}\n\nconst getAutoGLMMultilingualPlanPrompt = (): string => {\n  return `\nThe current date: ${getMultilingualFormattedDate()}\n\n# Setup\nYou are a professional Android operation agent assistant that can fulfill the user's high-level instructions. Given a screenshot of the Android interface at each step, you first analyze the situation, then plan the best course of action using Python-style pseudo-code.\n\n# More details about the code\nYour response format must be structured as follows:\n\nThink first: Use <think>...</think> to analyze the current screen, identify key elements, and determine the most efficient action.\nProvide the action: Use <answer>...</answer> to return a single line of pseudo-code representing the operation.\n\nYour output should STRICTLY follow the format:\n<think>\n[Your thought]\n</think>\n<answer>\n[Your operation code]\n</answer>\n\n- **Tap**\n  Perform a tap action on a specified screen area. The element is a list of 2 integers, representing the coordinates of the tap point.\n  **Example**:\n  <answer>\n  do(action=\"Tap\", element=[x,y])\n  </answer>\n- **Type**\n  Enter text into the currently focused input field.\n  **Example**:\n  <answer>\n  do(action=\"Type\", text=\"Hello World\")\n  </answer>\n- **Swipe**\n  Perform a swipe action with start point and end point.\n  **Examples**:\n  <answer>\n  do(action=\"Swipe\", start=[x1,y1], end=[x2,y2])\n  </answer>\n- **Long Press**\n  Perform a long press action on a specified screen area.\n  You can add the element to the action to specify the long press area. The element is a list of 2 integers, representing the coordinates of the long press point.\n  **Example**:\n  <answer>\n  do(action=\"Long Press\", element=[x,y])\n  </answer>\n- **Launch**\n  Launch an app. Try to use launch action when you need to launch an app. Check the instruction to choose the right app before you use this action.\n  **Example**:\n  <answer>\n  do(action=\"Launch\", app=\"Settings\")\n  </answer>\n- **Back**\n  Press the Back button to navigate to the previous screen.\n  **Example**:\n  <answer>\n  do(action=\"Back\")\n  </answer>\n- **Finish**\n  Terminate the program and optionally print a message.\n  **Example**:\n  <answer>\n  finish(message=\"Task completed.\")\n  </answer>\n\n\nREMEMBER:\n- Think before you act: Always analyze the current UI and the best course of action before executing any step, and output in <think> part.\n- Only ONE LINE of action in <answer> part per response: Each step must contain exactly one line of executable code.\n- Generate execution code strictly according to format requirements.\n  `;\n};\n\nconst getAutoGLMChinesePlanPrompt = (): string => {\n  return `\n今天的日期是: ${getChineseFormattedDate()}\n\n你是一个智能体分析专家，可以根据操作历史和当前状态图执行一系列操作来完成任务。\n你必须严格按照要求输出以下格式：\n<think>{think}</think>\n<answer>{action}</answer>\n\n其中：\n- {think} 是对你为什么选择这个操作的简短推理说明。\n- {action} 是本次执行的具体操作指令，必须严格遵循下方定义的指令格式。\n\n操作指令及其作用如下：\n- do(action=\"Launch\", app=\"xxx\")  \n    Launch是启动目标app的操作，这比通过主屏幕导航更快。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Tap\", element=[x,y])  \n    Tap是点击操作，点击屏幕上的特定点。可用此操作点击按钮、选择项目、从主屏幕打开应用程序，或与任何可点击的用户界面元素进行交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Tap\", element=[x,y], message=\"重要操作\")  \n    基本功能同Tap，点击涉及财产、支付、隐私等敏感按钮时触发。\n- do(action=\"Type\", text=\"xxx\")  \n    Type是输入操作，在当前聚焦的输入框中输入文本。使用此操作前，请确保输入框已被聚焦（先点击它）。输入的文本将像使用键盘输入一样输入。重要提示：手机可能正在使用 ADB 键盘，该键盘不会像普通键盘那样占用屏幕空间。要确认键盘已激活，请查看屏幕底部是否显示 'ADB Keyboard {ON}' 类似的文本，或者检查输入框是否处于激活/高亮状态。不要仅仅依赖视觉上的键盘显示。自动清除文本：当你使用输入操作时，输入框中现有的任何文本（包括占位符文本和实际输入）都会在输入新文本前自动清除。你无需在输入前手动清除文本——直接使用输入操作输入所需文本即可。操作完成后，你将自动收到结果状态的截图。\n- do(action=\"Type_Name\", text=\"xxx\")  \n    Type_Name是输入人名的操作，基本功能同Type。\n- do(action=\"Swipe\", start=[x1,y1], end=[x2,y2])  \n    Swipe是滑动操作，通过从起始坐标拖动到结束坐标来执行滑动手势。可用于滚动内容、在屏幕之间导航、下拉通知栏以及项目栏或进行基于手势的导航。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。滑动持续时间会自动调整以实现自然的移动。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Long Press\", element=[x,y])  \n    Long Pres是长按操作，在屏幕上的特定点长按指定时间。可用于触发上下文菜单、选择文本或激活长按交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的屏幕截图。\n- do(action=\"Double Tap\", element=[x,y])  \n    Double Tap在屏幕上的特定点快速连续点按两次。使用此操作可以激活双击交互，如缩放、选择文本或打开项目。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Back\")  \n    导航返回到上一个屏幕或关闭当前对话框。相当于按下 Android 的返回按钮。使用此操作可以从更深的屏幕返回、关闭弹出窗口或退出当前上下文。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Home\") \n    Home是回到系统桌面的操作，相当于按下 Android 主屏幕按钮。使用此操作可退出当前应用并返回启动器，或从已知状态启动新任务。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Wait\", duration=\"x seconds\")  \n    等待页面加载，x为需要等待多少秒。\n- finish(message=\"xxx\")  \n    finish是结束任务的操作，表示准确完整完成任务，message是终止信息。 \n\n必须遵循的规则：\n0. 严禁调用 Interact、Take_over、Note、Call_API 这四个操作，这些操作暂不支持。\n1. 在执行任何操作前，先检查当前app是否是目标app，如果不是，先执行 Launch。\n2. 如果进入到了无关页面，先执行 Back。如果执行Back后页面没有变化，请点击页面左上角的返回键进行返回，或者右上角的X号关闭。\n3. 如果页面未加载出内容，最多连续 Wait 三次，否则执行 Back重新进入。\n4. 如果页面显示网络问题，需要重新加载，请点击重新加载。\n5. 如果当前页面找不到目标联系人、商品、店铺等信息，可以尝试 Swipe 滑动查找。\n6. 遇到价格区间、时间区间等筛选条件，如果没有完全符合的，可以放宽要求。\n7. 在做小红书总结类任务时一定要筛选图文笔记。\n8. 购物车全选后再点击全选可以把状态设为全不选，在做购物车任务时，如果购物车里已经有商品被选中时，你需要点击全选后再点击取消全选，再去找需要购买或者删除的商品。\n9. 在做外卖任务时，如果相应店铺购物车里已经有其他商品你需要先把购物车清空再去购买用户指定的外卖。\n10. 在做点外卖任务时，如果用户需要点多个外卖，请尽量在同一店铺进行购买，如果无法找到可以下单，并说明某个商品未找到。\n11. 请严格遵循用户意图执行任务，用户的特殊要求可以执行多次搜索，滑动查找。比如（i）用户要求点一杯咖啡，要咸的，你可以直接搜索咸咖啡，或者搜索咖啡后滑动查找咸的咖啡，比如海盐咖啡。（ii）用户要找到XX群，发一条消息，你可以先搜索XX群，找不到结果后，将\"群\"字去掉，搜索XX重试。（iii）用户要找到宠物友好的餐厅，你可以搜索餐厅，找到筛选，找到设施，选择可带宠物，或者直接搜索可带宠物，必要时可以使用AI搜索。\n12. 在选择日期时，如果原滑动方向与预期日期越来越远，请向反方向滑动查找。\n13. 执行任务过程中如果有多个可选择的项目栏，请逐个查找每个项目栏，直到完成任务，一定不要在同一项目栏多次查找，从而陷入死循环。\n14. 在执行下一步操作前请一定要检查上一步的操作是否生效，如果点击没生效，可能因为app反应较慢，请先稍微等待一下，如果还是不生效请调整一下点击位置重试，如果仍然不生效请跳过这一步继续任务，并在finish message说明点击不生效。\n15. 在执行任务中如果遇到滑动不生效的情况，请调整一下起始点位置，增大滑动距离重试，如果还是不生效，有可能是已经滑到底了，请继续向反方向滑动，直到顶部或底部，如果仍然没有符合要求的结果，请跳过这一步继续任务，并在finish message说明但没找到要求的项目。\n16. 在做游戏任务时如果在战斗页面如果有自动战斗一定要开启自动战斗，如果多轮历史状态相似要检查自动战斗是否开启。\n17. 如果没有合适的搜索结果，可能是因为搜索页面不对，请返回到搜索页面的上一级尝试重新搜索，如果尝试三次返回上一级搜索后仍然没有符合要求的结果，执行 finish(message=\"原因\")。\n18. 在结束任务前请一定要仔细检查任务是否完整准确的完成，如果出现错选、漏选、多选的情况，请返回之前的步骤进行纠正。\n`;\n};\n\nexport const getAutoGLMPlanPrompt = (\n  vlMode: TVlModeTypes | undefined,\n): string => {\n  if (vlMode === 'auto-glm-multilingual') {\n    return getAutoGLMMultilingualPlanPrompt();\n  } else if (vlMode === 'auto-glm') {\n    return getAutoGLMChinesePlanPrompt();\n  }\n  throw new Error(`Unsupported vlMode for Auto-GLM plan prompt: ${vlMode}`);\n};\n\nexport const getAutoGLMLocatePrompt = (\n  vlMode: TVlModeTypes | undefined,\n): string => {\n  if (vlMode === 'auto-glm-multilingual') {\n    return `\nThe current date: ${getMultilingualFormattedDate()}\n\n# Setup\nYou are a professional Android operation agent assistant that can fulfill the user's high-level instructions. Given a screenshot of the Android interface at each step, you first analyze the situation, then plan the best course of action using Python-style pseudo-code.\n\n# More details about the code\nYour response format must be structured as follows:\n\nThink first: Use <think>...</think> to analyze the current screen, identify key elements, and determine the most efficient action.\nProvide the action: Use <answer>...</answer> to return a single line of pseudo-code representing the operation.\n\nYour output should STRICTLY follow the format:\n<think>\n[Your thought]\n</think>\n<answer>\n[Your operation code]\n</answer>\n\n- **Tap**\n  Perform a tap action on a specified screen area. The element is a list of 2 integers, representing the coordinates of the tap point.\n  **Example**:\n  <answer>\n  do(action=\"Tap\", element=[x,y])\n  </answer>\n\nREMEMBER:\n- Your goal is to locate and tap the UI element specified by the user (e.g., button, icon, link, etc.). Do not attempt any other actions.\n  `;\n  } else if (vlMode === 'auto-glm') {\n    return `\n今天的日期是: ${getChineseFormattedDate()}\n\n你是一个智能体分析专家，可以根据操作历史和当前状态图执行一系列操作来完成任务。\n你必须严格按照要求输出以下格式：\n<think>{think}</think>\n<answer>{action}</answer>\n\n其中：\n- {think} 是对你为什么选择这个操作的简短推理说明。\n- {action} 是本次执行的具体操作指令，必须严格遵循下方定义的指令格式。\n\n操作指令及其作用如下：\n- do(action=\"Tap\", element=[x,y])  \n    Tap是点击操作，点击屏幕上的特定点。可用此操作点击按钮、选择项目、从主屏幕打开应用程序，或与任何可点击的用户界面元素进行交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的截图。\n\n必须遵循的规则：\n- 你的目标是定位并点击用户指定的UI元素（例如按钮、图标、链接等），请不要尝试任何其他的操作。\n    `;\n  }\n  throw new Error(`Unsupported vlMode for Auto-GLM locate prompt: ${vlMode}`);\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getMultilingualFormattedDate","today","Date","year","month","String","date","dayOfWeek","getChineseFormattedDate","weekdayNames","weekday","getAutoGLMMultilingualPlanPrompt","getAutoGLMChinesePlanPrompt","getAutoGLMPlanPrompt","vlMode","Error","getAutoGLMLocatePrompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACyBA,SAASI;IACP,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,OAAOF,MAAM,WAAW;IAC9B,MAAMG,QAAQC,OAAOJ,MAAM,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACvD,MAAMK,OAAOD,OAAOJ,MAAM,OAAO,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,YAAY;QAChB;QACA;QACA;QACA;QACA;QACA;QACA;KACD,CAACN,MAAM,MAAM,GAAG;IAEjB,OAAO,GAAGE,KAAK,CAAC,EAAEC,MAAM,CAAC,EAAEE,KAAK,EAAE,EAAEC,WAAW;AACjD;AAKA,SAASC;IACP,MAAMP,QAAQ,IAAIC;IAClB,MAAMC,OAAOF,MAAM,WAAW;IAC9B,MAAMG,QAAQC,OAAOJ,MAAM,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACvD,MAAMK,OAAOD,OAAOJ,MAAM,OAAO,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMQ,eAAe;QACnB;QACA;QACA;QACA;QACA;QACA;QACA;KACD;IACD,MAAMC,UAAUD,YAAY,CAACR,MAAM,MAAM,GAAG;IAE5C,OAAO,GAAGE,KAAK,CAAC,EAAEC,MAAM,CAAC,EAAEE,KAAK,EAAE,EAAEI,SAAS;AAC/C;AAEA,MAAMC,mCAAmC,IAChC,CAAC;kBACQ,EAAEX,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoEjD,CAAC;AAGH,MAAMY,8BAA8B,IAC3B,CAAC;QACF,EAAEJ,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDpC,CAAC;AAGM,MAAMK,uBAAuB,CAClCC;IAEA,IAAIA,AAAW,4BAAXA,QACF,OAAOH;IACF,IAAIG,AAAW,eAAXA,QACT,OAAOF;IAET,MAAM,IAAIG,MAAM,CAAC,6CAA6C,EAAED,QAAQ;AAC1E;AAEO,MAAME,yBAAyB,CACpCF;IAEA,IAAIA,AAAW,4BAAXA,QACF,OAAO,CAAC;kBACM,EAAEd,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4BjD,CAAC;IACM,IAAIc,AAAW,eAAXA,QACT,OAAO,CAAC;QACJ,EAAEN,0BAA0B;;;;;;;;;;;;;;;;;IAiBhC,CAAC;IAEH,MAAM,IAAIO,MAAM,CAAC,+CAA+C,EAAED,QAAQ;AAC5E"}
+{"version":3,"file":"ai-model/auto-glm/prompt.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/prompt.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/**\n * Auto-GLM Prompt Templates\n *\n * Portions of this file are derived from Open-AutoGLM\n * Copyright (c) 2024 zai-org\n * Licensed under the Apache License, Version 2.0\n *\n * Source: https://github.com/zai-org/Open-AutoGLM\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n * Modifications:\n * - Adapted prompts for Midscene.js integration\n */\n\nimport type { TModelFamily } from '@midscene/shared/env';\n\n/**\n * Get formatted date string for system prompts\n * @returns Formatted date string like \"2026-01-12, Sunday\"\n */\nfunction getMultilingualFormattedDate(): string {\n  const today = new Date();\n  const year = today.getFullYear();\n  const month = String(today.getMonth() + 1).padStart(2, '0');\n  const date = String(today.getDate()).padStart(2, '0');\n  const dayOfWeek = [\n    'Sunday',\n    'Monday',\n    'Tuesday',\n    'Wednesday',\n    'Thursday',\n    'Friday',\n    'Saturday',\n  ][today.getDay()];\n\n  return `${year}-${month}-${date}, ${dayOfWeek}`;\n}\n\n/**\n * Get formatted Chinese date (e.g., \"2026年01月13日 星期一\")\n */\nfunction getChineseFormattedDate(): string {\n  const today = new Date();\n  const year = today.getFullYear();\n  const month = String(today.getMonth() + 1).padStart(2, '0');\n  const date = String(today.getDate()).padStart(2, '0');\n  const weekdayNames = [\n    '星期日',\n    '星期一',\n    '星期二',\n    '星期三',\n    '星期四',\n    '星期五',\n    '星期六',\n  ];\n  const weekday = weekdayNames[today.getDay()];\n\n  return `${year}年${month}月${date}日 ${weekday}`;\n}\n\nconst getAutoGLMMultilingualPlanPrompt = (): string => {\n  return `\nThe current date: ${getMultilingualFormattedDate()}\n\n# Setup\nYou are a professional Android operation agent assistant that can fulfill the user's high-level instructions. Given a screenshot of the Android interface at each step, you first analyze the situation, then plan the best course of action using Python-style pseudo-code.\n\n# More details about the code\nYour response format must be structured as follows:\n\nThink first: Use <think>...</think> to analyze the current screen, identify key elements, and determine the most efficient action.\nProvide the action: Use <answer>...</answer> to return a single line of pseudo-code representing the operation.\n\nYour output should STRICTLY follow the format:\n<think>\n[Your thought]\n</think>\n<answer>\n[Your operation code]\n</answer>\n\n- **Tap**\n  Perform a tap action on a specified screen area. The element is a list of 2 integers, representing the coordinates of the tap point.\n  **Example**:\n  <answer>\n  do(action=\"Tap\", element=[x,y])\n  </answer>\n- **Type**\n  Enter text into the currently focused input field.\n  **Example**:\n  <answer>\n  do(action=\"Type\", text=\"Hello World\")\n  </answer>\n- **Swipe**\n  Perform a swipe action with start point and end point.\n  **Examples**:\n  <answer>\n  do(action=\"Swipe\", start=[x1,y1], end=[x2,y2])\n  </answer>\n- **Long Press**\n  Perform a long press action on a specified screen area.\n  You can add the element to the action to specify the long press area. The element is a list of 2 integers, representing the coordinates of the long press point.\n  **Example**:\n  <answer>\n  do(action=\"Long Press\", element=[x,y])\n  </answer>\n- **Launch**\n  Launch an app. Try to use launch action when you need to launch an app. Check the instruction to choose the right app before you use this action.\n  **Example**:\n  <answer>\n  do(action=\"Launch\", app=\"Settings\")\n  </answer>\n- **Back**\n  Press the Back button to navigate to the previous screen.\n  **Example**:\n  <answer>\n  do(action=\"Back\")\n  </answer>\n- **Finish**\n  Terminate the program and optionally print a message.\n  **Example**:\n  <answer>\n  finish(message=\"Task completed.\")\n  </answer>\n\n\nREMEMBER:\n- Think before you act: Always analyze the current UI and the best course of action before executing any step, and output in <think> part.\n- Only ONE LINE of action in <answer> part per response: Each step must contain exactly one line of executable code.\n- Generate execution code strictly according to format requirements.\n  `;\n};\n\nconst getAutoGLMChinesePlanPrompt = (): string => {\n  return `\n今天的日期是: ${getChineseFormattedDate()}\n\n你是一个智能体分析专家，可以根据操作历史和当前状态图执行一系列操作来完成任务。\n你必须严格按照要求输出以下格式：\n<think>{think}</think>\n<answer>{action}</answer>\n\n其中：\n- {think} 是对你为什么选择这个操作的简短推理说明。\n- {action} 是本次执行的具体操作指令，必须严格遵循下方定义的指令格式。\n\n操作指令及其作用如下：\n- do(action=\"Launch\", app=\"xxx\")  \n    Launch是启动目标app的操作，这比通过主屏幕导航更快。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Tap\", element=[x,y])  \n    Tap是点击操作，点击屏幕上的特定点。可用此操作点击按钮、选择项目、从主屏幕打开应用程序，或与任何可点击的用户界面元素进行交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Tap\", element=[x,y], message=\"重要操作\")  \n    基本功能同Tap，点击涉及财产、支付、隐私等敏感按钮时触发。\n- do(action=\"Type\", text=\"xxx\")  \n    Type是输入操作，在当前聚焦的输入框中输入文本。使用此操作前，请确保输入框已被聚焦（先点击它）。输入的文本将像使用键盘输入一样输入。重要提示：手机可能正在使用 ADB 键盘，该键盘不会像普通键盘那样占用屏幕空间。要确认键盘已激活，请查看屏幕底部是否显示 'ADB Keyboard {ON}' 类似的文本，或者检查输入框是否处于激活/高亮状态。不要仅仅依赖视觉上的键盘显示。自动清除文本：当你使用输入操作时，输入框中现有的任何文本（包括占位符文本和实际输入）都会在输入新文本前自动清除。你无需在输入前手动清除文本——直接使用输入操作输入所需文本即可。操作完成后，你将自动收到结果状态的截图。\n- do(action=\"Type_Name\", text=\"xxx\")  \n    Type_Name是输入人名的操作，基本功能同Type。\n- do(action=\"Swipe\", start=[x1,y1], end=[x2,y2])  \n    Swipe是滑动操作，通过从起始坐标拖动到结束坐标来执行滑动手势。可用于滚动内容、在屏幕之间导航、下拉通知栏以及项目栏或进行基于手势的导航。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。滑动持续时间会自动调整以实现自然的移动。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Long Press\", element=[x,y])  \n    Long Pres是长按操作，在屏幕上的特定点长按指定时间。可用于触发上下文菜单、选择文本或激活长按交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的屏幕截图。\n- do(action=\"Double Tap\", element=[x,y])  \n    Double Tap在屏幕上的特定点快速连续点按两次。使用此操作可以激活双击交互，如缩放、选择文本或打开项目。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Back\")  \n    导航返回到上一个屏幕或关闭当前对话框。相当于按下 Android 的返回按钮。使用此操作可以从更深的屏幕返回、关闭弹出窗口或退出当前上下文。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Home\") \n    Home是回到系统桌面的操作，相当于按下 Android 主屏幕按钮。使用此操作可退出当前应用并返回启动器，或从已知状态启动新任务。此操作完成后，您将自动收到结果状态的截图。\n- do(action=\"Wait\", duration=\"x seconds\")  \n    等待页面加载，x为需要等待多少秒。\n- finish(message=\"xxx\")  \n    finish是结束任务的操作，表示准确完整完成任务，message是终止信息。 \n\n必须遵循的规则：\n0. 严禁调用 Interact、Take_over、Note、Call_API 这四个操作，这些操作暂不支持。\n1. 在执行任何操作前，先检查当前app是否是目标app，如果不是，先执行 Launch。\n2. 如果进入到了无关页面，先执行 Back。如果执行Back后页面没有变化，请点击页面左上角的返回键进行返回，或者右上角的X号关闭。\n3. 如果页面未加载出内容，最多连续 Wait 三次，否则执行 Back重新进入。\n4. 如果页面显示网络问题，需要重新加载，请点击重新加载。\n5. 如果当前页面找不到目标联系人、商品、店铺等信息，可以尝试 Swipe 滑动查找。\n6. 遇到价格区间、时间区间等筛选条件，如果没有完全符合的，可以放宽要求。\n7. 在做小红书总结类任务时一定要筛选图文笔记。\n8. 购物车全选后再点击全选可以把状态设为全不选，在做购物车任务时，如果购物车里已经有商品被选中时，你需要点击全选后再点击取消全选，再去找需要购买或者删除的商品。\n9. 在做外卖任务时，如果相应店铺购物车里已经有其他商品你需要先把购物车清空再去购买用户指定的外卖。\n10. 在做点外卖任务时，如果用户需要点多个外卖，请尽量在同一店铺进行购买，如果无法找到可以下单，并说明某个商品未找到。\n11. 请严格遵循用户意图执行任务，用户的特殊要求可以执行多次搜索，滑动查找。比如（i）用户要求点一杯咖啡，要咸的，你可以直接搜索咸咖啡，或者搜索咖啡后滑动查找咸的咖啡，比如海盐咖啡。（ii）用户要找到XX群，发一条消息，你可以先搜索XX群，找不到结果后，将\"群\"字去掉，搜索XX重试。（iii）用户要找到宠物友好的餐厅，你可以搜索餐厅，找到筛选，找到设施，选择可带宠物，或者直接搜索可带宠物，必要时可以使用AI搜索。\n12. 在选择日期时，如果原滑动方向与预期日期越来越远，请向反方向滑动查找。\n13. 执行任务过程中如果有多个可选择的项目栏，请逐个查找每个项目栏，直到完成任务，一定不要在同一项目栏多次查找，从而陷入死循环。\n14. 在执行下一步操作前请一定要检查上一步的操作是否生效，如果点击没生效，可能因为app反应较慢，请先稍微等待一下，如果还是不生效请调整一下点击位置重试，如果仍然不生效请跳过这一步继续任务，并在finish message说明点击不生效。\n15. 在执行任务中如果遇到滑动不生效的情况，请调整一下起始点位置，增大滑动距离重试，如果还是不生效，有可能是已经滑到底了，请继续向反方向滑动，直到顶部或底部，如果仍然没有符合要求的结果，请跳过这一步继续任务，并在finish message说明但没找到要求的项目。\n16. 在做游戏任务时如果在战斗页面如果有自动战斗一定要开启自动战斗，如果多轮历史状态相似要检查自动战斗是否开启。\n17. 如果没有合适的搜索结果，可能是因为搜索页面不对，请返回到搜索页面的上一级尝试重新搜索，如果尝试三次返回上一级搜索后仍然没有符合要求的结果，执行 finish(message=\"原因\")。\n18. 在结束任务前请一定要仔细检查任务是否完整准确的完成，如果出现错选、漏选、多选的情况，请返回之前的步骤进行纠正。\n`;\n};\n\nexport const getAutoGLMPlanPrompt = (\n  modelFamily: TModelFamily | undefined,\n): string => {\n  if (modelFamily === 'auto-glm-multilingual') {\n    return getAutoGLMMultilingualPlanPrompt();\n  } else if (modelFamily === 'auto-glm') {\n    return getAutoGLMChinesePlanPrompt();\n  }\n  throw new Error(\n    `Unsupported modelFamily for Auto-GLM plan prompt: ${modelFamily}`,\n  );\n};\n\nexport const getAutoGLMLocatePrompt = (\n  modelFamily: TModelFamily | undefined,\n): string => {\n  if (modelFamily === 'auto-glm-multilingual') {\n    return `\nThe current date: ${getMultilingualFormattedDate()}\n\n# Setup\nYou are a professional Android operation agent assistant that can fulfill the user's high-level instructions. Given a screenshot of the Android interface at each step, you first analyze the situation, then plan the best course of action using Python-style pseudo-code.\n\n# More details about the code\nYour response format must be structured as follows:\n\nThink first: Use <think>...</think> to analyze the current screen, identify key elements, and determine the most efficient action.\nProvide the action: Use <answer>...</answer> to return a single line of pseudo-code representing the operation.\n\nYour output should STRICTLY follow the format:\n<think>\n[Your thought]\n</think>\n<answer>\n[Your operation code]\n</answer>\n\n- **Tap**\n  Perform a tap action on a specified screen area. The element is a list of 2 integers, representing the coordinates of the tap point.\n  **Example**:\n  <answer>\n  do(action=\"Tap\", element=[x,y])\n  </answer>\n\nREMEMBER:\n- Your goal is to locate and tap the UI element specified by the user (e.g., button, icon, link, etc.). Do not attempt any other actions.\n  `;\n  } else if (modelFamily === 'auto-glm') {\n    return `\n今天的日期是: ${getChineseFormattedDate()}\n\n你是一个智能体分析专家，可以根据操作历史和当前状态图执行一系列操作来完成任务。\n你必须严格按照要求输出以下格式：\n<think>{think}</think>\n<answer>{action}</answer>\n\n其中：\n- {think} 是对你为什么选择这个操作的简短推理说明。\n- {action} 是本次执行的具体操作指令，必须严格遵循下方定义的指令格式。\n\n操作指令及其作用如下：\n- do(action=\"Tap\", element=[x,y])  \n    Tap是点击操作，点击屏幕上的特定点。可用此操作点击按钮、选择项目、从主屏幕打开应用程序，或与任何可点击的用户界面元素进行交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的截图。\n\n必须遵循的规则：\n- 你的目标是定位并点击用户指定的UI元素（例如按钮、图标、链接等），请不要尝试任何其他的操作。\n    `;\n  }\n  throw new Error(\n    `Unsupported modelFamily for Auto-GLM locate prompt: ${modelFamily}`,\n  );\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getMultilingualFormattedDate","today","Date","year","month","String","date","dayOfWeek","getChineseFormattedDate","weekdayNames","weekday","getAutoGLMMultilingualPlanPrompt","getAutoGLMChinesePlanPrompt","getAutoGLMPlanPrompt","modelFamily","Error","getAutoGLMLocatePrompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACyBA,SAASI;IACP,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,OAAOF,MAAM,WAAW;IAC9B,MAAMG,QAAQC,OAAOJ,MAAM,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACvD,MAAMK,OAAOD,OAAOJ,MAAM,OAAO,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,YAAY;QAChB;QACA;QACA;QACA;QACA;QACA;QACA;KACD,CAACN,MAAM,MAAM,GAAG;IAEjB,OAAO,GAAGE,KAAK,CAAC,EAAEC,MAAM,CAAC,EAAEE,KAAK,EAAE,EAAEC,WAAW;AACjD;AAKA,SAASC;IACP,MAAMP,QAAQ,IAAIC;IAClB,MAAMC,OAAOF,MAAM,WAAW;IAC9B,MAAMG,QAAQC,OAAOJ,MAAM,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACvD,MAAMK,OAAOD,OAAOJ,MAAM,OAAO,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMQ,eAAe;QACnB;QACA;QACA;QACA;QACA;QACA;QACA;KACD;IACD,MAAMC,UAAUD,YAAY,CAACR,MAAM,MAAM,GAAG;IAE5C,OAAO,GAAGE,KAAK,CAAC,EAAEC,MAAM,CAAC,EAAEE,KAAK,EAAE,EAAEI,SAAS;AAC/C;AAEA,MAAMC,mCAAmC,IAChC,CAAC;kBACQ,EAAEX,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoEjD,CAAC;AAGH,MAAMY,8BAA8B,IAC3B,CAAC;QACF,EAAEJ,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDpC,CAAC;AAGM,MAAMK,uBAAuB,CAClCC;IAEA,IAAIA,AAAgB,4BAAhBA,aACF,OAAOH;IACF,IAAIG,AAAgB,eAAhBA,aACT,OAAOF;IAET,MAAM,IAAIG,MACR,CAAC,kDAAkD,EAAED,aAAa;AAEtE;AAEO,MAAME,yBAAyB,CACpCF;IAEA,IAAIA,AAAgB,4BAAhBA,aACF,OAAO,CAAC;kBACM,EAAEd,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4BjD,CAAC;IACM,IAAIc,AAAgB,eAAhBA,aACT,OAAO,CAAC;QACJ,EAAEN,0BAA0B;;;;;;;;;;;;;;;;;IAiBhC,CAAC;IAEH,MAAM,IAAIO,MACR,CAAC,oDAAoD,EAAED,aAAa;AAExE"}

package/dist/lib/ai-model/auto-glm/util.js

@@ -24,14 +24,20 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    isAutoGLM: ()=>isAutoGLM
+    isAutoGLM: ()=>isAutoGLM,
+    isUITars: ()=>isUITars
 });
-function isAutoGLM(vlMode) {
-    return 'auto-glm' === vlMode || 'auto-glm-multilingual' === vlMode;
+function isAutoGLM(modelFamily) {
+    return 'auto-glm' === modelFamily || 'auto-glm-multilingual' === modelFamily;
+}
+function isUITars(modelFamily) {
+    return 'vlm-ui-tars' === modelFamily || 'vlm-ui-tars-doubao' === modelFamily || 'vlm-ui-tars-doubao-1.5' === modelFamily;
 }
 exports.isAutoGLM = __webpack_exports__.isAutoGLM;
+exports.isUITars = __webpack_exports__.isUITars;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
-    "isAutoGLM"
+    "isAutoGLM",
+    "isUITars"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true

package/dist/lib/ai-model/auto-glm/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/util.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/util.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TVlModeTypes } from '@midscene/shared/env';\n\n/**\n * Check if the vlMode is auto-glm or auto-glm-multilingual\n * @param vlMode The VL mode to check\n * @returns true if vlMode is auto-glm or auto-glm-multilingual\n */\nexport function isAutoGLM(vlMode: TVlModeTypes | undefined): boolean {\n  return vlMode === 'auto-glm' || vlMode === 'auto-glm-multilingual';\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","isAutoGLM","vlMode"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACCO,SAASI,UAAUC,MAAgC;IACxD,OAAOA,AAAW,eAAXA,UAAyBA,AAAW,4BAAXA;AAClC"}
+{"version":3,"file":"ai-model/auto-glm/util.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/util.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\n\n/**\n * Check if the modelFamily is auto-glm or auto-glm-multilingual\n * @param modelFamily The model family to check\n * @returns true if modelFamily is auto-glm or auto-glm-multilingual\n */\nexport function isAutoGLM(modelFamily: TModelFamily | undefined): boolean {\n  return modelFamily === 'auto-glm' || modelFamily === 'auto-glm-multilingual';\n}\n\n/**\n * Check if the modelFamily is a UI-TARS variant\n * @param modelFamily The model family to check\n * @returns true if modelFamily is any UI-TARS variant\n */\nexport function isUITars(modelFamily: TModelFamily | undefined): boolean {\n  return (\n    modelFamily === 'vlm-ui-tars' ||\n    modelFamily === 'vlm-ui-tars-doubao' ||\n    modelFamily === 'vlm-ui-tars-doubao-1.5'\n  );\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","isAutoGLM","modelFamily","isUITars"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACCO,SAASI,UAAUC,WAAqC;IAC7D,OAAOA,AAAgB,eAAhBA,eAA8BA,AAAgB,4BAAhBA;AACvC;AAOO,SAASC,SAASD,WAAqC;IAC5D,OACEA,AAAgB,kBAAhBA,eACAA,AAAgB,yBAAhBA,eACAA,AAAgB,6BAAhBA;AAEJ"}

package/dist/lib/ai-model/index.js

@@ -24,7 +24,6 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    resizeImageForUiTars: ()=>external_ui_tars_planning_js_namespaceObject.resizeImageForUiTars,
     callAIWithObjectResponse: ()=>index_js_namespaceObject.callAIWithObjectResponse,
     TUserPromptSchema: ()=>external_common_js_namespaceObject.TUserPromptSchema,
     generatePlaywrightTest: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTest,
@@ -89,7 +88,6 @@ exports.generateYamlTestStream = __webpack_exports__.generateYamlTestStream;
 exports.getMidsceneLocationSchema = __webpack_exports__.getMidsceneLocationSchema;
 exports.parseActionParam = __webpack_exports__.parseActionParam;
 exports.plan = __webpack_exports__.plan;
-exports.resizeImageForUiTars = __webpack_exports__.resizeImageForUiTars;
 exports.systemPromptToLocateElement = __webpack_exports__.systemPromptToLocateElement;
 exports.uiTarsPlanning = __webpack_exports__.uiTarsPlanning;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
@@ -118,7 +116,6 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "getMidsceneLocationSchema",
     "parseActionParam",
     "plan",
-    "resizeImageForUiTars",
     "systemPromptToLocateElement",
     "uiTarsPlanning"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];

package/dist/lib/ai-model/inspect.js

@@ -89,12 +89,12 @@ const promptsToChatParam = async (multimodalPrompt)=>{
 };
 async function AiLocateElement(options) {
     const { context, targetElementDescription, callAIFn, modelConfig } = options;
-    const { vlMode } = modelConfig;
+    const { modelFamily } = modelConfig;
     const screenshotBase64 = context.screenshot.base64;
     (0, utils_namespaceObject.assert)(targetElementDescription, "cannot find the target element description");
     const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
     const userInstructionPrompt = (0, llm_locator_js_namespaceObject.findElementPrompt)(targetElementDescriptionText);
-    const systemPrompt = (0, util_js_namespaceObject.isAutoGLM)(vlMode) ? (0, prompt_js_namespaceObject.getAutoGLMLocatePrompt)(vlMode) : (0, llm_locator_js_namespaceObject.systemPromptToLocateElement)(vlMode);
+    const systemPrompt = (0, util_js_namespaceObject.isAutoGLM)(modelFamily) ? (0, prompt_js_namespaceObject.getAutoGLMLocatePrompt)(modelFamily) : (0, llm_locator_js_namespaceObject.systemPromptToLocateElement)(modelFamily);
     let imagePayload = screenshotBase64;
     let imageWidth = context.size.width;
     let imageHeight = context.size.height;
@@ -108,7 +108,7 @@ async function AiLocateElement(options) {
         imageHeight = options.searchConfig.rect?.height;
         originalImageWidth = imageWidth;
         originalImageHeight = imageHeight;
-    } else if ('qwen2.5-vl' === vlMode) {
+    } else if ('qwen2.5-vl' === modelFamily) {
         const paddedResult = await (0, img_namespaceObject.paddingToMatchBlockByBase64)(imagePayload);
         imageWidth = paddedResult.width;
         imageHeight = paddedResult.height;
@@ -131,7 +131,7 @@ async function AiLocateElement(options) {
                 },
                 {
                     type: 'text',
-                    text: (0, util_js_namespaceObject.isAutoGLM)(vlMode) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
+                    text: (0, util_js_namespaceObject.isAutoGLM)(modelFamily) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
                 }
             ]
         }
@@ -143,7 +143,7 @@ async function AiLocateElement(options) {
         });
         msgs.push(...addOns);
     }
-    if ((0, util_js_namespaceObject.isAutoGLM)(vlMode)) {
+    if ((0, util_js_namespaceObject.isAutoGLM)(modelFamily)) {
         const { content: rawResponseContent, usage } = await (0, index_js_namespaceObject.callAIWithStringResponse)(msgs, modelConfig);
         debugInspect('auto-glm rawResponse:', rawResponseContent);
         const parsed = (0, parser_js_namespaceObject.parseAutoGLMLocateResponse)(rawResponseContent);
@@ -212,7 +212,7 @@ async function AiLocateElement(options) {
     let errors = 'errors' in res.content ? res.content.errors : [];
     try {
         if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
-            resRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, vlMode);
+            resRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, modelFamily);
             debugInspect('resRect', resRect);
             const rectCenter = {
                 x: resRect.left + resRect.width / 2,
@@ -244,9 +244,9 @@ async function AiLocateElement(options) {
 }
 async function AiLocateSection(options) {
     const { context, sectionDescription, modelConfig } = options;
-    const { vlMode } = modelConfig;
+    const { modelFamily } = modelConfig;
     const screenshotBase64 = context.screenshot.base64;
-    const systemPrompt = (0, llm_section_locator_js_namespaceObject.systemPromptToLocateSection)(vlMode);
+    const systemPrompt = (0, llm_section_locator_js_namespaceObject.systemPromptToLocateSection)(modelFamily);
     const sectionLocatorInstructionText = (0, llm_section_locator_js_namespaceObject.sectionLocatorInstruction)(extraTextFromUserPrompt(sectionDescription));
     const msgs = [
         {
@@ -281,23 +281,23 @@ async function AiLocateSection(options) {
     let sectionRect;
     const sectionBbox = result.content.bbox;
     if (sectionBbox) {
-        const targetRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(sectionBbox, context.size.width, context.size.height, 0, 0, context.size.width, context.size.height, vlMode);
+        const targetRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(sectionBbox, context.size.width, context.size.height, 0, 0, context.size.width, context.size.height, modelFamily);
         debugSection('original targetRect %j', targetRect);
         const referenceBboxList = result.content.references_bbox || [];
         debugSection('referenceBboxList %j', referenceBboxList);
-        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>(0, external_common_js_namespaceObject.adaptBboxToRect)(bbox, context.size.width, context.size.height, 0, 0, context.size.width, context.size.height, vlMode));
+        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>(0, external_common_js_namespaceObject.adaptBboxToRect)(bbox, context.size.width, context.size.height, 0, 0, context.size.width, context.size.height, modelFamily));
         debugSection('referenceRects %j', referenceRects);
         const mergedRect = (0, external_common_js_namespaceObject.mergeRects)([
             targetRect,
             ...referenceRects
         ]);
         debugSection('mergedRect %j', mergedRect);
-        sectionRect = (0, external_common_js_namespaceObject.expandSearchArea)(mergedRect, context.size, vlMode);
+        sectionRect = (0, external_common_js_namespaceObject.expandSearchArea)(mergedRect, context.size, modelFamily);
         debugSection('expanded sectionRect %j', sectionRect);
     }
     let imageBase64 = screenshotBase64;
     if (sectionRect) {
-        const croppedResult = await (0, img_namespaceObject.cropByRect)(screenshotBase64, sectionRect, 'qwen2.5-vl' === vlMode);
+        const croppedResult = await (0, img_namespaceObject.cropByRect)(screenshotBase64, sectionRect, 'qwen2.5-vl' === modelFamily);
         imageBase64 = croppedResult.imageBase64;
         sectionRect.width = croppedResult.width;
         sectionRect.height = croppedResult.height;

package/dist/lib/ai-model/inspect.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/inspect.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport { parseAutoGLMLocateResponse } from './auto-glm/parser';\nimport { getAutoGLMLocatePrompt } from './auto-glm/prompt';\nimport { isAutoGLM } from './auto-glm/util';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  callAIWithObjectResponse,\n  callAIWithStringResponse,\n} from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const screenshotBase64 = context.screenshot.base64;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = isAutoGLM(vlMode)\n    ? getAutoGLMLocatePrompt(vlMode)\n    : systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: isAutoGLM(vlMode)\n            ? `Tap: ${userInstructionPrompt}`\n            : userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  if (isAutoGLM(vlMode)) {\n    const { content: rawResponseContent, usage } =\n      await callAIWithStringResponse(msgs, modelConfig);\n\n    debugInspect('auto-glm rawResponse:', rawResponseContent);\n\n    const parsed = parseAutoGLMLocateResponse(rawResponseContent);\n\n    debugInspect('auto-glm thinking:', parsed.think);\n    debugInspect('auto-glm coordinates:', parsed.coordinates);\n\n    let resRect: Rect | undefined;\n    let matchedElements: LocateResultElement[] = [];\n    let errors: string[] = [];\n\n    if (parsed.error || !parsed.coordinates) {\n      errors = [parsed.error || 'Failed to parse auto-glm response'];\n      debugInspect('auto-glm parse error:', errors[0]);\n    } else {\n      const { x, y } = parsed.coordinates;\n\n      debugInspect('auto-glm coordinates [0-999]:', { x, y });\n\n      // Convert auto-glm coordinates [0,999] to pixel bbox\n      // Map from [0,999] to pixel coordinates\n      const pixelX = Math.round((x * imageWidth) / 1000);\n      const pixelY = Math.round((y * imageHeight) / 1000);\n\n      debugInspect('auto-glm pixel coordinates:', { pixelX, pixelY });\n\n      // Create a small bbox around the point\n      const bboxSize = 10;\n      const x1 = Math.max(pixelX - bboxSize / 2, 0);\n      const y1 = Math.max(pixelY - bboxSize / 2, 0);\n      const x2 = Math.min(pixelX + bboxSize / 2, imageWidth);\n      const y2 = Math.min(pixelY + bboxSize / 2, imageHeight);\n\n      // Convert to Rect format\n      resRect = {\n        left: x1,\n        top: y1,\n        width: x2 - x1,\n        height: y2 - y1,\n      };\n\n      // Apply offset if searching in a cropped area\n      if (options.searchConfig?.rect) {\n        resRect.left += options.searchConfig.rect.left;\n        resRect.top += options.searchConfig.rect.top;\n      }\n\n      debugInspect('auto-glm resRect:', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n\n    return {\n      rect: resRect,\n      parseResult: {\n        elements: matchedElements,\n        errors,\n      },\n      rawResponse: rawResponseContent,\n      usage,\n      reasoning_content: parsed.think,\n    };\n  }\n\n  const res = await callAIFn(msgs, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: LocateResultElement[] = [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n    reasoning_content: result.reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(msgs, modelConfig);\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","isAutoGLM","getAutoGLMLocatePrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","rawResponseContent","usage","callAIWithStringResponse","parsed","parseAutoGLMLocateResponse","resRect","matchedElements","errors","x","y","pixelX","Math","pixelY","bboxSize","x1","y1","x2","y2","rectCenter","element","generateElementByPosition","res","rawResponse","JSON","Array","adaptBboxToRect","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;ACiDA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAC9B,MAAMC,eAAeD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OAQrC;IAUC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAME,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElDM,IAAAA,sBAAAA,MAAAA,AAAAA,EACEL,0BACA;IAEF,MAAMM,+BAA+BjB,wBACnCW;IAEF,MAAMO,wBAAwBC,AAAAA,IAAAA,+BAAAA,iBAAAA,AAAAA,EAAkBF;IAChD,MAAMG,eAAeC,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUP,UAC3BQ,AAAAA,IAAAA,0BAAAA,sBAAAA,AAAAA,EAAuBR,UACvBS,AAAAA,IAAAA,+BAAAA,2BAAAA,AAAAA,EAA4BT;IAEhC,IAAIU,eAAeT;IACnB,IAAIU,aAAaf,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIgB,cAAchB,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIiB,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIjB,QAAQ,YAAY,EAAE;QACxBO,IAAAA,sBAAAA,MAAAA,AAAAA,EACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,IAAAA,sBAAAA,MAAAA,AAAAA,EACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFe,eAAef,QAAQ,YAAY,CAAC,WAAW;QAC/CgB,aAAahB,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCiB,cAAcjB,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCkB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIZ,AAAW,iBAAXA,QAAyB;QAClC,MAAMe,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMH,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUP,UACZ,CAAC,KAAK,EAAEI,uBAAuB,GAC/BA;gBACN;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMoB,SAAS,MAAM7B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,IAAIV,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUP,SAAS;QACrB,MAAM,EAAE,SAASkB,kBAAkB,EAAEC,KAAK,EAAE,GAC1C,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB9B,MAAMS;QAEvChB,aAAa,yBAAyBmC;QAEtC,MAAMG,SAASC,AAAAA,IAAAA,0BAAAA,0BAAAA,AAAAA,EAA2BJ;QAE1CnC,aAAa,sBAAsBsC,OAAO,KAAK;QAC/CtC,aAAa,yBAAyBsC,OAAO,WAAW;QAExD,IAAIE;QACJ,IAAIC,kBAAyC,EAAE;QAC/C,IAAIC,SAAmB,EAAE;QAEzB,IAAIJ,OAAO,KAAK,IAAI,CAACA,OAAO,WAAW,EAAE;YACvCI,SAAS;gBAACJ,OAAO,KAAK,IAAI;aAAoC;YAC9DtC,aAAa,yBAAyB0C,MAAM,CAAC,EAAE;QACjD,OAAO;YACL,MAAM,EAAEC,CAAC,EAAEC,CAAC,EAAE,GAAGN,OAAO,WAAW;YAEnCtC,aAAa,iCAAiC;gBAAE2C;gBAAGC;YAAE;YAIrD,MAAMC,SAASC,KAAK,KAAK,CAAEH,IAAIf,aAAc;YAC7C,MAAMmB,SAASD,KAAK,KAAK,CAAEF,IAAIf,cAAe;YAE9C7B,aAAa,+BAA+B;gBAAE6C;gBAAQE;YAAO;YAG7D,MAAMC,WAAW;YACjB,MAAMC,KAAKH,KAAK,GAAG,CAACD,SAASG,WAAW,GAAG;YAC3C,MAAME,KAAKJ,KAAK,GAAG,CAACC,SAASC,WAAW,GAAG;YAC3C,MAAMG,KAAKL,KAAK,GAAG,CAACD,SAASG,WAAW,GAAGpB;YAC3C,MAAMwB,KAAKN,KAAK,GAAG,CAACC,SAASC,WAAW,GAAGnB;YAG3CW,UAAU;gBACR,MAAMS;gBACN,KAAKC;gBACL,OAAOC,KAAKF;gBACZ,QAAQG,KAAKF;YACf;YAGA,IAAItC,QAAQ,YAAY,EAAE,MAAM;gBAC9B4B,QAAQ,IAAI,IAAI5B,QAAQ,YAAY,CAAC,IAAI,CAAC,IAAI;gBAC9C4B,QAAQ,GAAG,IAAI5B,QAAQ,YAAY,CAAC,IAAI,CAAC,GAAG;YAC9C;YAEAZ,aAAa,qBAAqBwC;YAElC,MAAMa,aAAa;gBACjB,GAAGb,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMc,UAA+BC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,EACnCF,YACAjC;YAGF,IAAIkC,SACFb,kBAAkB;gBAACa;aAAQ;QAE/B;QAEA,OAAO;YACL,MAAMd;YACN,aAAa;gBACX,UAAUC;gBACVC;YACF;YACA,aAAaP;YACbC;YACA,mBAAmBE,OAAO,KAAK;QACjC;IACF;IAEA,MAAMkB,MAAM,MAAMzC,SAASR,MAAMS;IAEjC,MAAMyC,cAAcC,KAAK,SAAS,CAACF,IAAI,OAAO;IAE9C,IAAIhB;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYc,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBG,MAAM,OAAO,CAACH,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAhB,UAAUoB,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EACRJ,IAAI,OAAO,CAAC,IAAI,EAChB5B,YACAC,aACAjB,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BkB,oBACAC,qBACAd;YAGFjB,aAAa,WAAWwC;YAExB,MAAMa,aAAa;gBACjB,GAAGb,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMc,UAA+BC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,EACnCF,YACAjC;YAEFsB,SAAS,EAAE;YAEX,IAAIY,SACFb,kBAAkB;gBAACa;aAAQ;QAE/B;IACF,EAAE,OAAOO,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACnB,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEoB,IAAI,CAAC,CAAC;aAFtBpB,SAAS;YAACoB;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMtB;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAe;QACA,OAAOD,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAeQ,gBAAgBpD,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEoD,kBAAkB,EAAEjD,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAME,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMU,eAAe2C,AAAAA,IAAAA,uCAAAA,2BAAAA,AAAAA,EAA4BjD;IACjD,MAAMkD,gCAAgCC,AAAAA,IAAAA,uCAAAA,yBAAAA,AAAAA,EACpCjE,wBAAwB8D;IAE1B,MAAM1D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMiD;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM/B,SAAS,MAAM7B,mBAAmB;YACtC,QAAQ4D,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA1D,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMmC,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACnB/D,MACAS;IAGF,IAAIuD;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAab,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EACjBY,aACA3D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFf,aAAa,0BAA0BuE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DnE,aAAa,wBAAwBwE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASjB,MAAM,OAAO,CAACiB,OAC/B,GAAG,CAAC,CAACA,OACGhB,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EACLgB,MACA/D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNf,aAAa,qBAAqByE;QAGlC,MAAME,aAAaC,AAAAA,IAAAA,mCAAAA,UAAAA,AAAAA,EAAW;YAACL;eAAeE;SAAe;QAC7DzE,aAAa,iBAAiB2E;QAG9BN,cAAcQ,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBF,YAAYhE,QAAQ,IAAI,EAAEI;QACzDf,aAAa,2BAA2BqE;IAC1C;IAEA,IAAIS,cAAc9D;IAClB,IAAIqD,aAAa;QACf,MAAMU,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BhE,kBACAqD,aACAtD,AAAW,iBAAXA;QAEF+D,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAaX,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwBvE,OAO7C;IACC,MAAM,EAAEwE,SAAS,EAAEvE,OAAO,EAAEwE,aAAa,EAAE/E,gBAAgB,EAAEU,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAe+D,AAAAA,IAAAA,8BAAAA,qBAAAA,AAAAA;IACrB,MAAMpE,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM0E,wBAAwBC,AAAAA,IAAAA,8BAAAA,sBAAAA,AAAAA,EAC5B5E,QAAQ,eAAe,IAAI,IAC3BwE;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKvE;YACL,QAAQ;QACV;IACF;IAGFuE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMhF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASkE;QACX;KACD;IAED,IAAInF,kBAAkB;QACpB,MAAM4B,SAAS,MAAM7B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMmC,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACnB/D,MACAS;IAEF,OAAO;QACL,aAAaqD,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;QACnB,mBAAmBA,OAAO,iBAAiB;IAC7C;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnB5E,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeqE,AAAAA,IAAAA,yCAAAA,iCAAAA,AAAAA;IACrB,MAAMC,aAAaC,AAAAA,IAAAA,yCAAAA,yBAAAA,AAAAA,EAA0BH;IAE7C,MAAMpF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASsE;QACX;KACD;IAED,MAAMxB,SAAS,MAAMtD,SAASR,MAAMS;IAEpC,OAAO;QACL,kBAAkBqD,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/inspect.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport { parseAutoGLMLocateResponse } from './auto-glm/parser';\nimport { getAutoGLMLocatePrompt } from './auto-glm/prompt';\nimport { isAutoGLM } from './auto-glm/util';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  callAIWithObjectResponse,\n  callAIWithStringResponse,\n} from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { modelFamily } = modelConfig;\n  const screenshotBase64 = context.screenshot.base64;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = isAutoGLM(modelFamily)\n    ? getAutoGLMLocatePrompt(modelFamily)\n    : systemPromptToLocateElement(modelFamily);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (modelFamily === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: isAutoGLM(modelFamily)\n            ? `Tap: ${userInstructionPrompt}`\n            : userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  if (isAutoGLM(modelFamily)) {\n    const { content: rawResponseContent, usage } =\n      await callAIWithStringResponse(msgs, modelConfig);\n\n    debugInspect('auto-glm rawResponse:', rawResponseContent);\n\n    const parsed = parseAutoGLMLocateResponse(rawResponseContent);\n\n    debugInspect('auto-glm thinking:', parsed.think);\n    debugInspect('auto-glm coordinates:', parsed.coordinates);\n\n    let resRect: Rect | undefined;\n    let matchedElements: LocateResultElement[] = [];\n    let errors: string[] = [];\n\n    if (parsed.error || !parsed.coordinates) {\n      errors = [parsed.error || 'Failed to parse auto-glm response'];\n      debugInspect('auto-glm parse error:', errors[0]);\n    } else {\n      const { x, y } = parsed.coordinates;\n\n      debugInspect('auto-glm coordinates [0-999]:', { x, y });\n\n      // Convert auto-glm coordinates [0,999] to pixel bbox\n      // Map from [0,999] to pixel coordinates\n      const pixelX = Math.round((x * imageWidth) / 1000);\n      const pixelY = Math.round((y * imageHeight) / 1000);\n\n      debugInspect('auto-glm pixel coordinates:', { pixelX, pixelY });\n\n      // Create a small bbox around the point\n      const bboxSize = 10;\n      const x1 = Math.max(pixelX - bboxSize / 2, 0);\n      const y1 = Math.max(pixelY - bboxSize / 2, 0);\n      const x2 = Math.min(pixelX + bboxSize / 2, imageWidth);\n      const y2 = Math.min(pixelY + bboxSize / 2, imageHeight);\n\n      // Convert to Rect format\n      resRect = {\n        left: x1,\n        top: y1,\n        width: x2 - x1,\n        height: y2 - y1,\n      };\n\n      // Apply offset if searching in a cropped area\n      if (options.searchConfig?.rect) {\n        resRect.left += options.searchConfig.rect.left;\n        resRect.top += options.searchConfig.rect.top;\n      }\n\n      debugInspect('auto-glm resRect:', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n\n    return {\n      rect: resRect,\n      parseResult: {\n        elements: matchedElements,\n        errors,\n      },\n      rawResponse: rawResponseContent,\n      usage,\n      reasoning_content: parsed.think,\n    };\n  }\n\n  const res = await callAIFn(msgs, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: LocateResultElement[] = [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        modelFamily,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { modelFamily } = modelConfig;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = systemPromptToLocateSection(modelFamily);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      modelFamily,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          modelFamily,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, modelFamily);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      modelFamily === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n    reasoning_content: result.reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(msgs, modelConfig);\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","modelFamily","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","isAutoGLM","getAutoGLMLocatePrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","rawResponseContent","usage","callAIWithStringResponse","parsed","parseAutoGLMLocateResponse","resRect","matchedElements","errors","x","y","pixelX","Math","pixelY","bboxSize","x1","y1","x2","y2","rectCenter","element","generateElementByPosition","res","rawResponse","JSON","Array","adaptBboxToRect","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;ACiDA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAC9B,MAAMC,eAAeD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OAQrC;IAUC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,WAAW,EAAE,GAAGD;IACxB,MAAME,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElDM,IAAAA,sBAAAA,MAAAA,AAAAA,EACEL,0BACA;IAEF,MAAMM,+BAA+BjB,wBACnCW;IAEF,MAAMO,wBAAwBC,AAAAA,IAAAA,+BAAAA,iBAAAA,AAAAA,EAAkBF;IAChD,MAAMG,eAAeC,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUP,eAC3BQ,AAAAA,IAAAA,0BAAAA,sBAAAA,AAAAA,EAAuBR,eACvBS,AAAAA,IAAAA,+BAAAA,2BAAAA,AAAAA,EAA4BT;IAEhC,IAAIU,eAAeT;IACnB,IAAIU,aAAaf,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIgB,cAAchB,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIiB,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIjB,QAAQ,YAAY,EAAE;QACxBO,IAAAA,sBAAAA,MAAAA,AAAAA,EACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,IAAAA,sBAAAA,MAAAA,AAAAA,EACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFe,eAAef,QAAQ,YAAY,CAAC,WAAW;QAC/CgB,aAAahB,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCiB,cAAcjB,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCkB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIZ,AAAgB,iBAAhBA,aAA8B;QACvC,MAAMe,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMH,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUP,eACZ,CAAC,KAAK,EAAEI,uBAAuB,GAC/BA;gBACN;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMoB,SAAS,MAAM7B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,IAAIV,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUP,cAAc;QAC1B,MAAM,EAAE,SAASkB,kBAAkB,EAAEC,KAAK,EAAE,GAC1C,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB9B,MAAMS;QAEvChB,aAAa,yBAAyBmC;QAEtC,MAAMG,SAASC,AAAAA,IAAAA,0BAAAA,0BAAAA,AAAAA,EAA2BJ;QAE1CnC,aAAa,sBAAsBsC,OAAO,KAAK;QAC/CtC,aAAa,yBAAyBsC,OAAO,WAAW;QAExD,IAAIE;QACJ,IAAIC,kBAAyC,EAAE;QAC/C,IAAIC,SAAmB,EAAE;QAEzB,IAAIJ,OAAO,KAAK,IAAI,CAACA,OAAO,WAAW,EAAE;YACvCI,SAAS;gBAACJ,OAAO,KAAK,IAAI;aAAoC;YAC9DtC,aAAa,yBAAyB0C,MAAM,CAAC,EAAE;QACjD,OAAO;YACL,MAAM,EAAEC,CAAC,EAAEC,CAAC,EAAE,GAAGN,OAAO,WAAW;YAEnCtC,aAAa,iCAAiC;gBAAE2C;gBAAGC;YAAE;YAIrD,MAAMC,SAASC,KAAK,KAAK,CAAEH,IAAIf,aAAc;YAC7C,MAAMmB,SAASD,KAAK,KAAK,CAAEF,IAAIf,cAAe;YAE9C7B,aAAa,+BAA+B;gBAAE6C;gBAAQE;YAAO;YAG7D,MAAMC,WAAW;YACjB,MAAMC,KAAKH,KAAK,GAAG,CAACD,SAASG,WAAW,GAAG;YAC3C,MAAME,KAAKJ,KAAK,GAAG,CAACC,SAASC,WAAW,GAAG;YAC3C,MAAMG,KAAKL,KAAK,GAAG,CAACD,SAASG,WAAW,GAAGpB;YAC3C,MAAMwB,KAAKN,KAAK,GAAG,CAACC,SAASC,WAAW,GAAGnB;YAG3CW,UAAU;gBACR,MAAMS;gBACN,KAAKC;gBACL,OAAOC,KAAKF;gBACZ,QAAQG,KAAKF;YACf;YAGA,IAAItC,QAAQ,YAAY,EAAE,MAAM;gBAC9B4B,QAAQ,IAAI,IAAI5B,QAAQ,YAAY,CAAC,IAAI,CAAC,IAAI;gBAC9C4B,QAAQ,GAAG,IAAI5B,QAAQ,YAAY,CAAC,IAAI,CAAC,GAAG;YAC9C;YAEAZ,aAAa,qBAAqBwC;YAElC,MAAMa,aAAa;gBACjB,GAAGb,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMc,UAA+BC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,EACnCF,YACAjC;YAGF,IAAIkC,SACFb,kBAAkB;gBAACa;aAAQ;QAE/B;QAEA,OAAO;YACL,MAAMd;YACN,aAAa;gBACX,UAAUC;gBACVC;YACF;YACA,aAAaP;YACbC;YACA,mBAAmBE,OAAO,KAAK;QACjC;IACF;IAEA,MAAMkB,MAAM,MAAMzC,SAASR,MAAMS;IAEjC,MAAMyC,cAAcC,KAAK,SAAS,CAACF,IAAI,OAAO;IAE9C,IAAIhB;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYc,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBG,MAAM,OAAO,CAACH,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAhB,UAAUoB,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EACRJ,IAAI,OAAO,CAAC,IAAI,EAChB5B,YACAC,aACAjB,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BkB,oBACAC,qBACAd;YAGFjB,aAAa,WAAWwC;YAExB,MAAMa,aAAa;gBACjB,GAAGb,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMc,UAA+BC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,EACnCF,YACAjC;YAEFsB,SAAS,EAAE;YAEX,IAAIY,SACFb,kBAAkB;gBAACa;aAAQ;QAE/B;IACF,EAAE,OAAOO,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACnB,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEoB,IAAI,CAAC,CAAC;aAFtBpB,SAAS;YAACoB;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMtB;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAe;QACA,OAAOD,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAeQ,gBAAgBpD,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEoD,kBAAkB,EAAEjD,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,WAAW,EAAE,GAAGD;IACxB,MAAME,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMU,eAAe2C,AAAAA,IAAAA,uCAAAA,2BAAAA,AAAAA,EAA4BjD;IACjD,MAAMkD,gCAAgCC,AAAAA,IAAAA,uCAAAA,yBAAAA,AAAAA,EACpCjE,wBAAwB8D;IAE1B,MAAM1D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMiD;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM/B,SAAS,MAAM7B,mBAAmB;YACtC,QAAQ4D,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA1D,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMmC,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACnB/D,MACAS;IAGF,IAAIuD;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAab,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EACjBY,aACA3D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFf,aAAa,0BAA0BuE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DnE,aAAa,wBAAwBwE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASjB,MAAM,OAAO,CAACiB,OAC/B,GAAG,CAAC,CAACA,OACGhB,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EACLgB,MACA/D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNf,aAAa,qBAAqByE;QAGlC,MAAME,aAAaC,AAAAA,IAAAA,mCAAAA,UAAAA,AAAAA,EAAW;YAACL;eAAeE;SAAe;QAC7DzE,aAAa,iBAAiB2E;QAG9BN,cAAcQ,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBF,YAAYhE,QAAQ,IAAI,EAAEI;QACzDf,aAAa,2BAA2BqE;IAC1C;IAEA,IAAIS,cAAc9D;IAClB,IAAIqD,aAAa;QACf,MAAMU,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BhE,kBACAqD,aACAtD,AAAgB,iBAAhBA;QAEF+D,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAaX,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwBvE,OAO7C;IACC,MAAM,EAAEwE,SAAS,EAAEvE,OAAO,EAAEwE,aAAa,EAAE/E,gBAAgB,EAAEU,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAe+D,AAAAA,IAAAA,8BAAAA,qBAAAA,AAAAA;IACrB,MAAMpE,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM0E,wBAAwBC,AAAAA,IAAAA,8BAAAA,sBAAAA,AAAAA,EAC5B5E,QAAQ,eAAe,IAAI,IAC3BwE;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKvE;YACL,QAAQ;QACV;IACF;IAGFuE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMhF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASkE;QACX;KACD;IAED,IAAInF,kBAAkB;QACpB,MAAM4B,SAAS,MAAM7B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMmC,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACnB/D,MACAS;IAEF,OAAO;QACL,aAAaqD,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;QACnB,mBAAmBA,OAAO,iBAAiB;IAC7C;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnB5E,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeqE,AAAAA,IAAAA,yCAAAA,iCAAAA,AAAAA;IACrB,MAAMC,aAAaC,AAAAA,IAAAA,yCAAAA,yBAAAA,AAAAA,EAA0BH;IAE7C,MAAMpF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASsE;QACX;KACD;IAED,MAAMxB,SAAS,MAAMtD,SAASR,MAAMS;IAEpC,OAAO;QACL,kBAAkBqD,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/lib/ai-model/llm-planning.js

@@ -24,6 +24,7 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    parseXMLPlanningResponse: ()=>parseXMLPlanningResponse,
     plan: ()=>plan
 });
 const img_namespaceObject = require("@midscene/shared/img");
@@ -33,14 +34,57 @@ const external_common_js_namespaceObject = require("../common.js");
 const llm_planning_js_namespaceObject = require("./prompt/llm-planning.js");
 const index_js_namespaceObject = require("./service-caller/index.js");
 const debug = (0, logger_namespaceObject.getDebug)('planning');
+function parseXMLPlanningResponse(xmlString) {
+    const extractTag = (tagName)=>{
+        const regex = new RegExp(`<${tagName}>([\\s\\S]*?)</${tagName}>`, 'i');
+        const match = xmlString.match(regex);
+        return match ? match[1].trim() : void 0;
+    };
+    const thought = extractTag('thought');
+    const note = extractTag('note');
+    const log = extractTag('log');
+    const error = extractTag('error');
+    const actionType = extractTag('action-type');
+    const actionParamStr = extractTag('action-param-json');
+    if (!log) throw new Error('Missing required field: log');
+    let action = null;
+    if (actionType && 'null' !== actionType.toLowerCase()) {
+        const type = actionType.trim();
+        let param;
+        if (actionParamStr) try {
+            param = (0, index_js_namespaceObject.safeParseJson)(actionParamStr, void 0);
+        } catch (e) {
+            throw new Error(`Failed to parse action-param-json: ${e}`);
+        }
+        action = {
+            type,
+            ...void 0 !== param ? {
+                param
+            } : {}
+        };
+    }
+    return {
+        ...thought ? {
+            thought
+        } : {},
+        ...note ? {
+            note
+        } : {},
+        log,
+        ...error ? {
+            error
+        } : {},
+        action
+    };
+}
 async function plan(userInstruction, opts) {
     const { context, modelConfig, conversationHistory } = opts;
     const { size } = context;
     const screenshotBase64 = context.screenshot.base64;
-    const { vlMode } = modelConfig;
+    const { modelFamily } = modelConfig;
     const systemPrompt = await (0, llm_planning_js_namespaceObject.systemPromptToTaskPlanning)({
         actionSpace: opts.actionSpace,
-        vlMode,
+        modelFamily,
         includeBbox: opts.includeBbox,
         includeThought: true !== opts.deepThink
     });
@@ -49,7 +93,7 @@ async function plan(userInstruction, opts) {
     let imageHeight = size.height;
     const rightLimit = imageWidth;
     const bottomLimit = imageHeight;
-    if ('qwen2.5-vl' === vlMode) {
+    if ('qwen2.5-vl' === modelFamily) {
         const paddedResult = await (0, img_namespaceObject.paddingToMatchBlockByBase64)(imagePayload);
         imageWidth = paddedResult.width;
         imageHeight = paddedResult.height;
@@ -112,9 +156,10 @@ async function plan(userInstruction, opts) {
         ...instruction,
         ...historyLog
     ];
-    const { content: planFromAI, contentString: rawResponse, usage, reasoning_content } = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelConfig, {
+    const { content: rawResponse, usage, reasoning_content } = await (0, index_js_namespaceObject.callAI)(msgs, modelConfig, {
         deepThink: 'unset' === opts.deepThink ? void 0 : opts.deepThink
     });
+    const planFromAI = parseXMLPlanningResponse(rawResponse);
     const actions = planFromAI.action ? [
         planFromAI.action
     ] : [];
@@ -141,7 +186,7 @@ async function plan(userInstruction, opts) {
         debug('locateFields', locateFields);
         locateFields.forEach((field)=>{
             const locateResult = action.param[field];
-            if (locateResult && void 0 !== vlMode) action.param[field] = (0, external_common_js_namespaceObject.fillBboxParam)(locateResult, imageWidth, imageHeight, rightLimit, bottomLimit, vlMode);
+            if (locateResult && void 0 !== modelFamily) action.param[field] = (0, external_common_js_namespaceObject.fillBboxParam)(locateResult, imageWidth, imageHeight, rightLimit, bottomLimit, modelFamily);
         });
     });
     conversationHistory.append({
@@ -155,8 +200,10 @@ async function plan(userInstruction, opts) {
     });
     return returnValue;
 }
+exports.parseXMLPlanningResponse = __webpack_exports__.parseXMLPlanningResponse;
 exports.plan = __webpack_exports__.plan;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "parseXMLPlanningResponse",
     "plan"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {