@midscene/core 0.28.11-beta-20250922065131.0 → 0.28.12-beta-20250923080328.0

package/dist/lib/agent/ui-utils.js

@@ -37,8 +37,20 @@ function typeStr(task) {
 function locateParamStr(locate) {
     if (!locate) return '';
     if ('string' == typeof locate) return locate;
-    if ('string' == typeof locate.prompt) return locate.prompt;
-    if ('object' == typeof locate.prompt && locate.prompt.prompt) return locate.prompt.prompt;
+    if ('object' == typeof locate) {
+        if ('string' == typeof locate.prompt) return locate.prompt;
+        if ('object' == typeof locate.prompt && locate.prompt.prompt) {
+            const prompt = locate.prompt.prompt;
+            const images = locate.prompt.images || [];
+            if (0 === images.length) return prompt;
+            const imagesStr = images.map((image)=>{
+                let url = image.url;
+                if (url.startsWith('data:image/') || url.startsWith('data:') && url.includes('base64')) url = `${url.substring(0, 15)}...`;
+                return `[${image.name}](${url})`;
+            }).join(', ');
+            return `${prompt}, ${imagesStr}`;
+        }
+    }
     return '';
 }
 function scrollParamStr(scrollParam) {
@@ -64,27 +76,29 @@ function paramStr(task) {
         value = null == task ? void 0 : null == (_task_param = task.param) ? void 0 : _task_param.userInstruction;
     }
     if ('Insight' === task.type) {
-        var _task_param1, _task_param2, _task_param3, _task_param4;
-        value = (null == task ? void 0 : null == (_task_param1 = task.param) ? void 0 : _task_param1.prompt) || (null == task ? void 0 : null == (_task_param2 = task.param) ? void 0 : _task_param2.id) || (null == task ? void 0 : null == (_task_param3 = task.param) ? void 0 : _task_param3.dataDemand) || (null == task ? void 0 : null == (_task_param4 = task.param) ? void 0 : _task_param4.assertion);
+        var _task_param1, _task_param2, _task_param3;
+        value = locateParamStr(null == task ? void 0 : task.param) || (null == task ? void 0 : null == (_task_param1 = task.param) ? void 0 : _task_param1.id) || (null == task ? void 0 : null == (_task_param2 = task.param) ? void 0 : _task_param2.dataDemand) || (null == task ? void 0 : null == (_task_param3 = task.param) ? void 0 : _task_param3.assertion);
     }
     if ('Action' === task.type) {
-        var _task_param5, _task_param6, _task_param7, _task_param8;
+        var _task_param4, _task_param5, _task_param6, _task_param7;
         const locate = null == task ? void 0 : task.locate;
         const locateStr = locate ? locateParamStr(locate) : '';
         value = task.thought || '';
-        if ('number' == typeof (null == task ? void 0 : null == (_task_param5 = task.param) ? void 0 : _task_param5.timeMs)) {
+        if ('number' == typeof (null == task ? void 0 : null == (_task_param4 = task.param) ? void 0 : _task_param4.timeMs)) {
+            var _task_param8;
+            value = `${null == task ? void 0 : null == (_task_param8 = task.param) ? void 0 : _task_param8.timeMs}ms`;
+        } else if ('string' == typeof (null == task ? void 0 : null == (_task_param5 = task.param) ? void 0 : _task_param5.scrollType)) value = scrollParamStr(null == task ? void 0 : task.param);
+        else if ('string' == typeof (null == task ? void 0 : null == (_task_param6 = task.param) ? void 0 : _task_param6.direction) && (null == task ? void 0 : task.subType) === 'AndroidPull') value = pullParamStr(null == task ? void 0 : task.param);
+        else if (void 0 !== (null == task ? void 0 : null == (_task_param7 = task.param) ? void 0 : _task_param7.value)) {
             var _task_param9;
-            value = `${null == task ? void 0 : null == (_task_param9 = task.param) ? void 0 : _task_param9.timeMs}ms`;
-        } else if ('string' == typeof (null == task ? void 0 : null == (_task_param6 = task.param) ? void 0 : _task_param6.scrollType)) value = scrollParamStr(null == task ? void 0 : task.param);
-        else if ('string' == typeof (null == task ? void 0 : null == (_task_param7 = task.param) ? void 0 : _task_param7.direction) && (null == task ? void 0 : task.subType) === 'AndroidPull') value = pullParamStr(null == task ? void 0 : task.param);
-        else if (void 0 !== (null == task ? void 0 : null == (_task_param8 = task.param) ? void 0 : _task_param8.value)) {
-            var _task_param10;
-            value = null == task ? void 0 : null == (_task_param10 = task.param) ? void 0 : _task_param10.value;
+            value = null == task ? void 0 : null == (_task_param9 = task.param) ? void 0 : _task_param9.value;
         }
         if (locateStr) value = value ? `${locateStr} - ${value}` : locateStr;
     }
     if (void 0 === value) return '';
-    return 'string' == typeof value ? value : JSON.stringify(value, void 0, 2);
+    if ('string' == typeof value) return value;
+    if ('object' == typeof value && locateParamStr(value)) return locateParamStr(value);
+    return JSON.stringify(value, void 0, 2);
 }
 exports.locateParamStr = __webpack_exports__.locateParamStr;
 exports.paramStr = __webpack_exports__.paramStr;

package/dist/lib/agent/ui-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AndroidPullParam,\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ScrollParam,\n} from '@/index';\n\nexport function typeStr(task: ExecutionTask) {\n  return task.subType && task.subType !== 'Plan'\n    ? `${task.type} / ${task.subType || ''}`\n    : task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam) {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate.prompt === 'string') {\n    return locate.prompt;\n  }\n\n  if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n    return locate.prompt.prompt;\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: AndroidPullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      (task as ExecutionTaskInsightLocate)?.param?.prompt ||\n      (task as ExecutionTaskInsightLocate)?.param?.id ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n  return typeof value === 'string'\n    ? value\n    : JSON.stringify(value, undefined, 2);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","prompt","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","_task_param7","_task_param8","locateStr","_task_param9","_task_param10","JSON","undefined"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACMO,SAASI,QAAQC,IAAmB;IACzC,OAAOA,KAAK,OAAO,IAAIA,AAAiB,WAAjBA,KAAK,OAAO,GAC/B,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,IAAI,IAAI,GACtCA,KAAK,IAAI;AACf;AAEO,SAASC,eAAeC,MAA4B;IACzD,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;IAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAC3D,OAAOA,OAAO,MAAM,CAAC,MAAM;IAG7B,OAAO;AACT;AAEO,SAASC,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAA4B;IACvD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZC,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGD,KAAK,GAAG,EAAEC,QAAQ;IAE9B,OAAOD;AACT;AAEO,SAASE,SAASX,IAAmB;IAC1C,IAAIY;IACJ,IAAIZ,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBa;QAARD,QAASZ,QAAAA,OAAAA,KAAAA,IAAAA,QAADa,CAAAA,cAACb,KAAgC,KAAK,AAAD,IAArCa,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIb,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAEzBc,cACAC,cACAC,cACAC;QAJFL,QACEE,AAACd,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADc,CAAAA,eAACd,KAAqC,KAAK,AAAD,IAA1Cc,KAAAA,IAAAA,aAA6C,MAAM,AAAD,KACjDd,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADe,CAAAA,eAACf,KAAqC,KAAK,AAAD,IAA1Ce,KAAAA,IAAAA,aAA6C,EAAE,AAAD,KAC7Cf,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADgB,CAAAA,eAAChB,KAAoC,KAAK,AAAD,IAAzCgB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDhB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,eAACjB,KAAwC,KAAK,AAAD,IAA7CiB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAIjB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfkB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMnB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMsB,YAAYpB,SAASD,eAAeC,UAAU;QAEpDU,QAAQZ,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAA8B,KAAK,AAAD,IAAnCkB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXX,QAAQ,GAAIZ,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQvB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAA8B,KAAK,AAAD,IAAnCmB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDP,QAAQT,eAAgBH,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAA8B,KAAK,AAAD,IAAnCoB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACpB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CY,QAAQP,aAAcL,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARZ,QAASZ,QAAAA,OAAAA,KAAAA,IAAAA,QAADwB,CAAAA,gBAACxB,KAA8B,KAAK,AAAD,IAAnCwB,KAAAA,IAAAA,cAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAV,QADEA,QACM,GAAGU,UAAU,GAAG,EAAEV,OAAO,GAEzBU;IAGd;IAEA,IAAI,AAAiB,WAAVV,OAAuB,OAAO;IACzC,OAAO,AAAiB,YAAjB,OAAOA,QACVA,QACAa,KAAK,SAAS,CAACb,OAAOc,QAAW;AACvC"}
+{"version":3,"file":"agent/ui-utils.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AndroidPullParam,\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ScrollParam,\n} from '@/index';\n\nexport function typeStr(task: ExecutionTask) {\n  return task.subType && task.subType !== 'Plan'\n    ? `${task.type} / ${task.subType || ''}`\n    : task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: AndroidPullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||\n      (task as ExecutionTaskInsightLocate)?.param?.id ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","_task_param7","locateStr","_task_param8","_task_param9","JSON","undefined"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACMO,SAASI,QAAQC,IAAmB;IACzC,OAAOA,KAAK,OAAO,IAAIA,AAAiB,WAAjBA,KAAK,OAAO,GAC/B,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,IAAI,IAAI,GACtCA,KAAK,IAAI;AACf;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAA4B;IACvD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAGzBkB,cACAC,cACAC;QAJFJ,QACEf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAqC,KAAK,KACzDA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAqC,KAAK,AAAD,IAA1CkB,KAAAA,IAAAA,aAA6C,EAAE,AAAD,KAC7ClB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAoC,KAAK,AAAD,IAAzCmB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDnB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAAwC,KAAK,AAAD,IAA7CoB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAIpB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfqB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMtB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMyB,YAAYvB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXV,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQ1B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDN,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACvB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADwB,CAAAA,eAACxB,KAA8B,KAAK,AAAD,IAAnCwB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARX,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD2B,CAAAA,eAAC3B,KAA8B,KAAK,AAAD,IAAnC2B,KAAAA,IAAAA,aAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAT,QADEA,QACM,GAAGS,UAAU,GAAG,EAAET,OAAO,GAEzBS;IAGd;IAEA,IAAI,AAAiB,WAAVT,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOY,KAAK,SAAS,CAACZ,OAAOa,QAAW;AAC1C"}

package/dist/lib/agent/utils.js

@@ -186,7 +186,7 @@ function trimContextByViewport(execution) {
         }) : execution.tasks
     };
 }
-const getMidsceneVersion = ()=>"0.28.11-beta-20250922065131.0";
+const getMidsceneVersion = ()=>"0.28.12-beta-20250923080328.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/conversation-history.js

@@ -0,0 +1,92 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    ConversationHistory: ()=>ConversationHistory
+});
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+var _computedKey;
+_computedKey = Symbol.iterator;
+let _computedKey1 = _computedKey;
+class ConversationHistory {
+    append(message) {
+        if ('user' === message.role) this.pruneOldestUserMessageIfNecessary();
+        this.messages.push(message);
+    }
+    seed(messages) {
+        this.reset();
+        messages.forEach((message)=>{
+            this.append(message);
+        });
+    }
+    reset() {
+        this.messages.length = 0;
+    }
+    snapshot() {
+        return [
+            ...this.messages
+        ];
+    }
+    get length() {
+        return this.messages.length;
+    }
+    [_computedKey1]() {
+        return this.messages[Symbol.iterator]();
+    }
+    toJSON() {
+        return this.snapshot();
+    }
+    pruneOldestUserMessageIfNecessary() {
+        const userMessages = this.messages.filter((item)=>'user' === item.role);
+        if (userMessages.length < this.maxUserImageMessages) return;
+        const firstUserMessageIndex = this.messages.findIndex((item)=>'user' === item.role);
+        if (firstUserMessageIndex >= 0) this.messages.splice(firstUserMessageIndex, 1);
+    }
+    constructor(options){
+        var _options_initialMessages;
+        _define_property(this, "maxUserImageMessages", void 0);
+        _define_property(this, "messages", []);
+        this.maxUserImageMessages = (null == options ? void 0 : options.maxUserImageMessages) ?? 4;
+        if (null == options ? void 0 : null == (_options_initialMessages = options.initialMessages) ? void 0 : _options_initialMessages.length) this.seed(options.initialMessages);
+    }
+}
+exports.ConversationHistory = __webpack_exports__.ConversationHistory;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "ConversationHistory"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=conversation-history.js.map

package/dist/lib/ai-model/conversation-history.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/conversation-history.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/conversation-history.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  maxUserImageMessages?: number;\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly maxUserImageMessages: number;\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  constructor(options?: ConversationHistoryOptions) {\n    this.maxUserImageMessages = options?.maxUserImageMessages ?? 4;\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    if (message.role === 'user') {\n      this.pruneOldestUserMessageIfNecessary();\n    }\n\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  snapshot(): ChatCompletionMessageParam[] {\n    return [...this.messages];\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n\n  private pruneOldestUserMessageIfNecessary() {\n    const userMessages = this.messages.filter((item) => item.role === 'user');\n    if (userMessages.length < this.maxUserImageMessages) {\n      return;\n    }\n\n    const firstUserMessageIndex = this.messages.findIndex(\n      (item) => item.role === 'user',\n    );\n\n    if (firstUserMessageIndex >= 0) {\n      this.messages.splice(firstUserMessageIndex, 1);\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ConversationHistory","message","messages","userMessages","item","firstUserMessageIndex","options","_options_initialMessages"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;eCuCGG,OAAO,QAAQ;;AAtCX,MAAMC;IAWX,OAAOC,OAAmC,EAAE;QAC1C,IAAIA,AAAiB,WAAjBA,QAAQ,IAAI,EACd,IAAI,CAAC,iCAAiC;QAGxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAEA,WAAyC;QACvC,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACF,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAEQ,oCAAoC;QAC1C,MAAMI,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAACC,OAASA,AAAc,WAAdA,KAAK,IAAI;QAC7D,IAAID,aAAa,MAAM,GAAG,IAAI,CAAC,oBAAoB,EACjD;QAGF,MAAME,wBAAwB,IAAI,CAAC,QAAQ,CAAC,SAAS,CACnD,CAACD,OAASA,AAAc,WAAdA,KAAK,IAAI;QAGrB,IAAIC,yBAAyB,GAC3B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAACA,uBAAuB;IAEhD;IAvDA,YAAYC,OAAoC,CAAE;YAE5CC;QALN,uBAAiB,wBAAjB;QACA,uBAAiB,YAAyC,EAAE;QAG1D,IAAI,CAAC,oBAAoB,GAAGD,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,oBAAoB,AAAD,KAAK;QAC7D,IAAIC,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,QAAS,eAAe,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAClC,IAAI,CAAC,IAAI,CAACD,QAAQ,eAAe;IAErC;AAmDF"}

package/dist/lib/ai-model/index.js

@@ -32,19 +32,20 @@ __webpack_require__.d(__webpack_exports__, {
     AiLocateSection: ()=>external_inspect_js_namespaceObject.AiLocateSection,
     AIActionType: ()=>external_common_js_namespaceObject.AIActionType,
     TUserPromptSchema: ()=>external_common_js_namespaceObject.TUserPromptSchema,
+    ConversationHistory: ()=>external_conversation_history_js_namespaceObject.ConversationHistory,
     callAI: ()=>index_js_namespaceObject.callAI,
-    findAllMidsceneLocatorField: ()=>external_common_js_namespaceObject.findAllMidsceneLocatorField,
     AiExtractElementInfo: ()=>external_inspect_js_namespaceObject.AiExtractElementInfo,
+    findAllMidsceneLocatorField: ()=>external_common_js_namespaceObject.findAllMidsceneLocatorField,
     generatePlaywrightTest: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTest,
     generateYamlTestStream: ()=>yaml_generator_js_namespaceObject.generateYamlTestStream,
     loadActionParam: ()=>external_common_js_namespaceObject.loadActionParam,
     resizeImageForUiTars: ()=>external_ui_tars_planning_js_namespaceObject.resizeImageForUiTars,
     systemPromptToLocateElement: ()=>llm_locator_js_namespaceObject.systemPromptToLocateElement,
-    vlmPlanning: ()=>external_ui_tars_planning_js_namespaceObject.vlmPlanning,
     RectSchema: ()=>external_common_js_namespaceObject.RectSchema,
     PointSchema: ()=>external_common_js_namespaceObject.PointSchema,
     adaptBboxToRect: ()=>external_common_js_namespaceObject.adaptBboxToRect,
     callAIWithObjectResponse: ()=>index_js_namespaceObject.callAIWithObjectResponse,
+    vlmPlanning: ()=>external_ui_tars_planning_js_namespaceObject.vlmPlanning,
     AiLocateElement: ()=>external_inspect_js_namespaceObject.AiLocateElement,
     getMidsceneLocationSchema: ()=>external_common_js_namespaceObject.getMidsceneLocationSchema,
     callAIWithStringResponse: ()=>index_js_namespaceObject.callAIWithStringResponse,
@@ -61,10 +62,12 @@ const external_inspect_js_namespaceObject = require("./inspect.js");
 const external_llm_planning_js_namespaceObject = require("./llm-planning.js");
 const external_common_js_namespaceObject = require("./common.js");
 const external_ui_tars_planning_js_namespaceObject = require("./ui-tars-planning.js");
+const external_conversation_history_js_namespaceObject = require("./conversation-history.js");
 exports.AIActionType = __webpack_exports__.AIActionType;
 exports.AiExtractElementInfo = __webpack_exports__.AiExtractElementInfo;
 exports.AiLocateElement = __webpack_exports__.AiLocateElement;
 exports.AiLocateSection = __webpack_exports__.AiLocateSection;
+exports.ConversationHistory = __webpack_exports__.ConversationHistory;
 exports.PointSchema = __webpack_exports__.PointSchema;
 exports.RectSchema = __webpack_exports__.RectSchema;
 exports.SizeSchema = __webpack_exports__.SizeSchema;
@@ -93,6 +96,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "AiExtractElementInfo",
     "AiLocateElement",
     "AiLocateSection",
+    "ConversationHistory",
     "PointSchema",
     "RectSchema",
     "SizeSchema",

package/dist/lib/ai-model/llm-planning.js

@@ -35,8 +35,8 @@ const util_js_namespaceObject = require("./prompt/util.js");
 const index_js_namespaceObject = require("./service-caller/index.js");
 const debug = (0, logger_namespaceObject.getDebug)('planning');
 async function plan(userInstruction, opts) {
-    var _planFromAI_action;
-    const { context, modelConfig } = opts;
+    var _opts_conversationHistory, _planFromAI_action;
+    const { context, modelConfig, conversationHistory } = opts;
     const { screenshotBase64, size } = context;
     const { modelName, vlMode } = modelConfig;
     const { description: pageDescription, elementById } = await (0, util_js_namespaceObject.describeUserPage)(context, {
@@ -46,7 +46,9 @@ async function plan(userInstruction, opts) {
         actionSpace: opts.actionSpace,
         vlMode: vlMode
     });
-    const taskBackgroundContextText = (0, llm_planning_js_namespaceObject.generateTaskBackgroundContext)(userInstruction, opts.log, opts.actionContext);
+    const historyLog = getLogFromConversationHistory(null == (_opts_conversationHistory = opts.conversationHistory) ? void 0 : _opts_conversationHistory.snapshot());
+    const logForContext = historyLog ?? opts.log;
+    const taskBackgroundContextText = (0, llm_planning_js_namespaceObject.generateTaskBackgroundContext)(userInstruction, logForContext, opts.actionContext);
     const userInstructionPrompt = await (0, llm_planning_js_namespaceObject.automationUserPrompt)(vlMode).format({
         pageDescription,
         taskBackgroundContext: taskBackgroundContextText
@@ -108,8 +110,26 @@ async function plan(userInstruction, opts) {
     });
     (0, utils_namespaceObject.assert)(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);
     if (0 === actions.length && returnValue.more_actions_needed_by_instruction && !returnValue.sleep) console.warn('No actions planned for the prompt, but model said more actions are needed:', userInstruction);
+    null == conversationHistory || conversationHistory.append({
+        role: 'assistant',
+        content: returnValue.log
+    });
     return returnValue;
 }
+function getLogFromConversationHistory(conversationHistory) {
+    if (!(null == conversationHistory ? void 0 : conversationHistory.length)) return;
+    const assistantMessages = conversationHistory.filter((message)=>'assistant' === message.role).map((message)=>{
+        if ('string' == typeof message.content) return message.content.trim();
+        if (Array.isArray(message.content)) return message.content.map((part)=>{
+            if ('text' === part.type && 'string' == typeof part.text) return part.text.trim();
+            if ('string' == typeof part.text) return part.text.trim();
+            return '';
+        }).filter(Boolean).join('\n');
+        return '';
+    }).map((content)=>content.trim()).filter(Boolean);
+    if (0 === assistantMessages.length) return;
+    return `- ${assistantMessages.join('\n- ')}`;
+}
 exports.plan = __webpack_exports__.plan;
 for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "plan"

package/dist/lib/ai-model/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    log?: string;\n    actionContext?: string;\n    modelConfig: IModelConfig;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { modelName, vlMode } = modelConfig;\n\n  const { description: pageDescription, elementById } = await describeUserPage(\n    context,\n    { vlMode },\n  );\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlMode,\n  });\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    opts.log,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(vlMode).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlMode === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlMode) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size, modelName);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const { content, usage } = await callAIWithObjectResponse<PlanningAIResponse>(\n    msgs,\n    AIActionType.PLAN,\n    modelConfig,\n  );\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: use zod.parse to parse the action.param, and then fill the bbox param.\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult) {\n        if (vlMode) {\n          action.param[field] = fillBboxParam(\n            locateResult,\n            size.width,\n            size.height,\n            vlMode,\n          );\n        } else {\n          const element = elementById(locateResult);\n          if (element) {\n            action.param[field].id = element.id;\n          }\n        }\n      }\n    });\n  });\n  // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n  assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","plan","userInstruction","opts","_planFromAI_action","context","modelConfig","screenshotBase64","size","modelName","vlMode","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","content","usage","callAIWithObjectResponse","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","element","console"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACqBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAOC;QAmEEC;IAjEH,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGH;IACjC,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGH;IAEnC,MAAM,EAAEI,SAAS,EAAEC,MAAM,EAAE,GAAGJ;IAE9B,MAAM,EAAE,aAAaK,eAAe,EAAEC,WAAW,EAAE,GAAG,MAAMC,AAAAA,IAAAA,wBAAAA,gBAAAA,AAAAA,EAC1DR,SACA;QAAEK;IAAO;IAEX,MAAMI,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaZ,KAAK,WAAW;QAC7B,QAAQO;IACV;IACA,MAAMM,4BAA4BC,AAAAA,IAAAA,gCAAAA,6BAAAA,AAAAA,EAChCf,iBACAC,KAAK,GAAG,EACRA,KAAK,aAAa;IAEpB,MAAMe,wBAAwB,MAAMC,AAAAA,IAAAA,gCAAAA,oBAAAA,AAAAA,EAAqBT,QAAQ,MAAM,CAAC;QACtEC;QACA,uBAAuBK;IACzB;IAEA,IAAII,eAAeb;IACnB,IAAIG,AAAW,cAAXA,QACFU,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BD;SAC5C,IAAI,CAACV,QACVU,eAAe,MAAME,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EACnBf,kBACAF,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBkB,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBf,MAAMC;IAEzB,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASV;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKM;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAM,EAAEO,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC/BH,MACAI,mCAAAA,YAAAA,CAAAA,IAAiB,EACjBtB;IAEF,MAAMuB,cAAcC,KAAK,SAAS,CAACL,SAASM,QAAW;IACvD,MAAMC,aAAaP;IAEnB,MAAMQ,UACH7B,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAAC4B,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAH;QACA,UAAUS,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EACRF,SACA9B,KAAK,WAAW,EAChB6B,WAAW,KAAK;IAEpB;IAEAI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,YAAY;IAGnBC,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBpC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACkC,SAAWA,OAAO,IAAI,KAAKC;QAG9BvC,MAAM,+BAA+BwC;QACrC,MAAMC,eAAeD,sBACjBE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENxC,MAAM,gBAAgByC;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,cACF,IAAIjC,QACF2B,OAAO,KAAK,CAACK,MAAM,GAAGE,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EACpBD,cACAnC,KAAK,KAAK,EACVA,KAAK,MAAM,EACXE;iBAEG;gBACL,MAAMmC,UAAUjC,YAAY+B;gBAC5B,IAAIE,SACFR,OAAO,KAAK,CAACK,MAAM,CAAC,EAAE,GAAGG,QAAQ,EAAE;YAEvC;QAEJ;IACF;IAEAT,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IAEvE,IACEC,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACA5C;IAIJ,OAAOgC;AACT"}
+{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DeviceAction,\n  InterfaceType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport type { ConversationHistory } from './conversation-history';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    interfaceType: InterfaceType;\n    actionSpace: DeviceAction<any>[];\n    log?: string;\n    actionContext?: string;\n    modelConfig: IModelConfig;\n    conversationHistory?: ConversationHistory;\n  },\n): Promise<PlanningAIResponse> {\n  const { context, modelConfig, conversationHistory } = opts;\n  const { screenshotBase64, size } = context;\n\n  const { modelName, vlMode } = modelConfig;\n\n  const { description: pageDescription, elementById } = await describeUserPage(\n    context,\n    { vlMode },\n  );\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlMode,\n  });\n  const historyLog = getLogFromConversationHistory(\n    opts.conversationHistory?.snapshot(),\n  );\n  const logForContext = historyLog ?? opts.log;\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    logForContext,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(vlMode).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlMode === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlMode) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size, modelName);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const { content, usage } = await callAIWithObjectResponse<PlanningAIResponse>(\n    msgs,\n    AIActionType.PLAN,\n    modelConfig,\n  );\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: use zod.parse to parse the action.param, and then fill the bbox param.\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult) {\n        if (vlMode) {\n          action.param[field] = fillBboxParam(\n            locateResult,\n            size.width,\n            size.height,\n            vlMode,\n          );\n        } else {\n          const element = elementById(locateResult);\n          if (element) {\n            action.param[field].id = element.id;\n          }\n        }\n      }\n    });\n  });\n  // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n  assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  conversationHistory?.append({\n    role: 'assistant',\n    content: returnValue.log,\n  });\n\n  return returnValue;\n}\n\nfunction getLogFromConversationHistory(\n  conversationHistory?: ChatCompletionMessageParam[],\n): string | undefined {\n  if (!conversationHistory?.length) {\n    return undefined;\n  }\n\n  const assistantMessages = conversationHistory\n    .filter((message) => message.role === 'assistant')\n    .map((message) => {\n      if (typeof message.content === 'string') {\n        return message.content.trim();\n      }\n\n      if (Array.isArray(message.content)) {\n        return message.content\n          .map((part) => {\n            if (part.type === 'text' && typeof part.text === 'string') {\n              return part.text.trim();\n            }\n\n            if (typeof (part as any).text === 'string') {\n              return (part as any).text.trim();\n            }\n\n            return '';\n          })\n          .filter(Boolean)\n          .join('\\n');\n      }\n\n      return '';\n    })\n    .map((content) => content.trim())\n    .filter(Boolean);\n\n  if (assistantMessages.length === 0) {\n    return undefined;\n  }\n\n  return `- ${assistantMessages.join('\\n- ')}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","plan","userInstruction","opts","_opts_conversationHistory","_planFromAI_action","context","modelConfig","conversationHistory","screenshotBase64","size","modelName","vlMode","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","historyLog","getLogFromConversationHistory","logForContext","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","content","usage","callAIWithObjectResponse","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","element","console","assistantMessages","message","Array","part","Boolean"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACuBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAQC;QAgBCC,2BAuDCC;IArEH,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGL;IACtD,MAAM,EAAEM,gBAAgB,EAAEC,IAAI,EAAE,GAAGJ;IAEnC,MAAM,EAAEK,SAAS,EAAEC,MAAM,EAAE,GAAGL;IAE9B,MAAM,EAAE,aAAaM,eAAe,EAAEC,WAAW,EAAE,GAAG,MAAMC,AAAAA,IAAAA,wBAAAA,gBAAAA,AAAAA,EAC1DT,SACA;QAAEM;IAAO;IAEX,MAAMI,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAad,KAAK,WAAW;QAC7B,QAAQS;IACV;IACA,MAAMM,aAAaC,8BAA8B,QAC/Cf,CAAAA,4BAAAA,KAAK,mBAAmB,AAAD,IAAvBA,KAAAA,IAAAA,0BAA0B,QAAQ;IAEpC,MAAMgB,gBAAgBF,cAAcf,KAAK,GAAG;IAC5C,MAAMkB,4BAA4BC,AAAAA,IAAAA,gCAAAA,6BAAAA,AAAAA,EAChCpB,iBACAkB,eACAjB,KAAK,aAAa;IAEpB,MAAMoB,wBAAwB,MAAMC,AAAAA,IAAAA,gCAAAA,oBAAAA,AAAAA,EAAqBZ,QAAQ,MAAM,CAAC;QACtEC;QACA,uBAAuBQ;IACzB;IAEA,IAAII,eAAehB;IACnB,IAAIG,AAAW,cAAXA,QACFa,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,2BAAAA,AAAAA,EAA4BD;SAC5C,IAAI,CAACb,QACVa,eAAe,MAAME,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EACnBlB,kBACAH,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBsB,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBlB,MAAMC;IAEzB,MAAMkB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASb;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKS;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAM,EAAEO,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC/BH,MACAI,mCAAAA,YAAAA,CAAAA,IAAiB,EACjB1B;IAEF,MAAM2B,cAAcC,KAAK,SAAS,CAACL,SAASM,QAAW;IACvD,MAAMC,aAAaP;IAEnB,MAAMQ,UACHjC,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAACgC,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAH;QACA,UAAUS,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EACRF,SACAnC,KAAK,WAAW,EAChBkC,WAAW,KAAK;IAEpB;IAEAI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,YAAY;IAGnBC,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBzC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACuC,SAAWA,OAAO,IAAI,KAAKC;QAG9B5C,MAAM,+BAA+B6C;QACrC,MAAMC,eAAeD,sBACjBE,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAEN7C,MAAM,gBAAgB8C;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,cACF,IAAIpC,QACF8B,OAAO,KAAK,CAACK,MAAM,GAAGE,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EACpBD,cACAtC,KAAK,KAAK,EACVA,KAAK,MAAM,EACXE;iBAEG;gBACL,MAAMsC,UAAUpC,YAAYkC;gBAC5B,IAAIE,SACFR,OAAO,KAAK,CAACK,MAAM,CAAC,EAAE,GAAGG,QAAQ,EAAE;YAEvC;QAEJ;IACF;IAEAT,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IAEvE,IACEC,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACAjD;IAIJM,QAAAA,uBAAAA,oBAAqB,MAAM,CAAC;QAC1B,MAAM;QACN,SAAS+B,YAAY,GAAG;IAC1B;IAEA,OAAOA;AACT;AAEA,SAASpB,8BACPX,mBAAkD;IAElD,IAAI,CAACA,CAAAA,QAAAA,sBAAAA,KAAAA,IAAAA,oBAAqB,MAAM,AAAD,GAC7B;IAGF,MAAM4C,oBAAoB5C,oBACvB,MAAM,CAAC,CAAC6C,UAAYA,AAAiB,gBAAjBA,QAAQ,IAAI,EAChC,GAAG,CAAC,CAACA;QACJ,IAAI,AAA2B,YAA3B,OAAOA,QAAQ,OAAO,EACxB,OAAOA,QAAQ,OAAO,CAAC,IAAI;QAG7B,IAAIC,MAAM,OAAO,CAACD,QAAQ,OAAO,GAC/B,OAAOA,QAAQ,OAAO,CACnB,GAAG,CAAC,CAACE;YACJ,IAAIA,AAAc,WAAdA,KAAK,IAAI,IAAe,AAAqB,YAArB,OAAOA,KAAK,IAAI,EAC1C,OAAOA,KAAK,IAAI,CAAC,IAAI;YAGvB,IAAI,AAA8B,YAA9B,OAAQA,KAAa,IAAI,EAC3B,OAAQA,KAAa,IAAI,CAAC,IAAI;YAGhC,OAAO;QACT,GACC,MAAM,CAACC,SACP,IAAI,CAAC;QAGV,OAAO;IACT,GACC,GAAG,CAAC,CAAC1B,UAAYA,QAAQ,IAAI,IAC7B,MAAM,CAAC0B;IAEV,IAAIJ,AAA6B,MAA7BA,kBAAkB,MAAM,EAC1B;IAGF,OAAO,CAAC,EAAE,EAAEA,kBAAkB,IAAI,CAAC,SAAS;AAC9C"}

package/dist/lib/ai-model/ui-tars-planning.js

@@ -44,18 +44,31 @@ const pointToBbox = (point, width, height)=>[
         Math.round(Math.min(point.x + bboxSize / 2, width)),
         Math.round(Math.min(point.y + bboxSize / 2, height))
     ];
-async function vlmPlanning(options) {
-    const { conversationHistory, userInstruction, size, modelConfig } = options;
+async function vlmPlanning(userInstruction, options) {
+    const { conversationHistory, context, modelConfig } = options;
     const { uiTarsModelVersion } = modelConfig;
     const systemPrompt = (0, ui_tars_planning_js_namespaceObject.getUiTarsPlanningPrompt)() + userInstruction;
+    const imagePayload = await resizeImageForUiTars(context.screenshotBase64, context.size, uiTarsModelVersion);
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload
+                }
+            }
+        ]
+    });
     const res = await (0, index_js_namespaceObject.callAIWithStringResponse)([
         {
             role: 'user',
             content: systemPrompt
         },
-        ...conversationHistory
+        ...conversationHistory.snapshot()
     ], external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig);
     const convertedText = convertBboxToCoordinates(res.content);
+    const { size } = context;
     const { parsed } = (0, action_parser_namespaceObject.actionParser)({
         prediction: convertedText,
         factor: [
@@ -70,6 +83,7 @@ async function vlmPlanning(options) {
     });
     debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
     const transformActions = [];
+    let shouldContinue = true;
     parsed.forEach((action)=>{
         if ('click' === action.action_type) {
             (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
@@ -147,6 +161,7 @@ async function vlmPlanning(options) {
             },
             thought: action.thought || ''
         });
+        else if ('finished' === action.action_type) shouldContinue = false;
     });
     if (0 === transformActions.length) throw new Error(`No actions found, response: ${res.content}`, {
         cause: {
@@ -155,12 +170,17 @@ async function vlmPlanning(options) {
         }
     });
     debug('transformActions', JSON.stringify(transformActions, null, 2));
+    const log = (0, ui_tars_planning_js_namespaceObject.getSummary)(res.content);
+    conversationHistory.append({
+        role: 'assistant',
+        content: log
+    });
     return {
         actions: transformActions,
-        actionsFromModel: parsed,
-        action_summary: (0, ui_tars_planning_js_namespaceObject.getSummary)(res.content),
+        log,
         usage: res.usage,
-        rawResponse: JSON.stringify(res.content, void 0, 2)
+        rawResponse: JSON.stringify(res.content, void 0, 2),
+        more_actions_needed_by_instruction: shouldContinue
     };
 }
 function convertBboxToCoordinates(text) {

package/dist/lib/ai-model/ui-tars-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/ui-tars-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIUsageInfo,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Size,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType } from './common';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { callAIWithStringResponse } from './service-caller/index';\ntype ActionType =\n  | 'click'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function vlmPlanning(options: {\n  userInstruction: string;\n  conversationHistory: ChatCompletionMessageParam[];\n  size: { width: number; height: number };\n  modelConfig: IModelConfig;\n}): Promise<{\n  actions: PlanningAction<any>[];\n  actionsFromModel: ReturnType<typeof actionParser>['parsed'];\n  action_summary: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const { conversationHistory, userInstruction, size, modelConfig } = options;\n  const { uiTarsModelVersion } = modelConfig;\n  const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory,\n    ],\n    AIActionType.INSPECT_ELEMENT,\n    modelConfig,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  parsed.forEach((action) => {\n    if (action.action_type === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n      });\n    } else if (action.action_type === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys,\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (action.action_type === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(`No actions found, response: ${res.content}`, {\n      cause: {\n        prediction: res.content,\n        parsed,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n\n  return {\n    actions: transformActions,\n    actionsFromModel: parsed,\n    action_summary: getSummary(res.content),\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  uiTarsVersion: UITarsModelVersion | undefined,\n) {\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","vlmPlanning","options","conversationHistory","userInstruction","size","modelConfig","uiTarsModelVersion","systemPrompt","getUiTarsPlanningPrompt","res","callAIWithStringResponse","AIActionType","convertedText","convertBboxToCoordinates","parsed","actionParser","JSON","transformActions","action","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","Error","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","resizeImageForUiTars","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACmBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,YAAYC,OAKjC;IAQC,MAAM,EAAEC,mBAAmB,EAAEC,eAAe,EAAEC,IAAI,EAAEC,WAAW,EAAE,GAAGJ;IACpE,MAAM,EAAEK,kBAAkB,EAAE,GAAGD;IAC/B,MAAME,eAAeC,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,MAA4BL;IAEjD,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChB;QACE;YACE,MAAM;YACN,SAASH;QACX;WACGL;KACJ,EACDS,mCAAAA,YAAAA,CAAAA,eAA4B,EAC5BN;IAEF,MAAMO,gBAAgBC,yBAAyBJ,IAAI,OAAO;IAE1D,MAAM,EAAEK,MAAM,EAAE,GAAGC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC9B,YAAYH;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOR,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUE;IACZ;IAEAd,MACE,oBACAc,oBACA,YACAU,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7CH,OAAO,OAAO,CAAC,CAACI;QACd,IAAIA,AAAuB,YAAvBA,OAAO,WAAW,EAAc;YAClCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMtB,QAAQwB,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEd;YACvDa,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BQ,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIc,AAAuB,WAAvBA,OAAO,WAAW,EAAa;YACxCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMG,aAAaD,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEd;YAC5D,MAAMkB,WAAWF,SAASF,OAAO,aAAa,CAAC,OAAO,EAAEd;YACxDa,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAG0B,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCjB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQc,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAG2B,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjClB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASc,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIA,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOC,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWC,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,eAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASC,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3B,IAAKA,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMK,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBN,OAAO,aAAa,CAAC,GAAG;YAE1DD,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASM;gBACX;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEO,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASC,OAAO,OAAO,IAAI;QAC7B;IAEJ;IAEA,IAAID,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIS,MAAM,CAAC,4BAA4B,EAAEjB,IAAI,OAAO,EAAE,EAAE;QAC5D,OAAO;YACL,YAAYA,IAAI,OAAO;YACvBK;QACF;IACF;IAGFtB,MAAM,oBAAoBwB,KAAK,SAAS,CAACC,kBAAkB,MAAM;IAEjE,OAAO;QACL,SAASA;QACT,kBAAkBH;QAClB,gBAAgBa,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWlB,IAAI,OAAO;QACtC,OAAOA,IAAI,KAAK;QAChB,aAAaO,KAAK,SAAS,CAACP,IAAI,OAAO,EAAEmB,QAAW;IACtD;AACF;AAOA,SAASf,yBAAyBgB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAI3C,KAAK,KAAK,CAAEsC,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAI5C,KAAK,KAAK,CAAEwC,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASX,SAASyB,QAAgB,EAAEzC,IAAuC;IACzE,MAAM,CAACsC,GAAGC,EAAE,GAAG3B,KAAK,KAAK,CAAC6B;IAC1B,OAAO;QAACH,IAAItC,KAAK,KAAK;QAAEuC,IAAIvC,KAAK,MAAM;KAAC;AAC1C;AAkEO,eAAe0C,qBACpBC,WAAmB,EACnB3C,IAAU,EACV4C,aAA6C;IAE7C,IAAIA,kBAAkBC,oBAAAA,kBAAAA,CAAAA,IAAuB,EAAE;QAC7CzD,MAAM,uCAAuCY;QAC7C,MAAM8C,gBAAgB9C,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAM+C,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAerD,KAAK,IAAI,CAACoD,YAAYD;YAC3C,MAAMG,WAAWtD,KAAK,KAAK,CAACK,KAAK,KAAK,GAAGgD;YACzC,MAAME,YAAYvD,KAAK,KAAK,CAACK,KAAK,MAAM,GAAGgD;YAC3C5D,MACE,2DACA6D,UACAC;YAEF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}
+{"version":3,"file":"ai-model/ui-tars-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  PlanningAIResponse,\n  PlanningAction,\n  Size,\n  UIContext,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport { AIActionType } from './common';\nimport type { ConversationHistory } from './conversation-history';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { callAIWithStringResponse } from './service-caller/index';\ntype ActionType =\n  | 'click'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function vlmPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig } = options;\n  const { uiTarsModelVersion } = modelConfig;\n  const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;\n\n  const imagePayload = await resizeImageForUiTars(\n    context.screenshotBase64,\n    context.size,\n    uiTarsModelVersion,\n  );\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: {\n          url: imagePayload,\n        },\n      },\n    ],\n  });\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory.snapshot(),\n    ],\n    AIActionType.INSPECT_ELEMENT,\n    modelConfig,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const { size } = context;\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  let shouldContinue = true;\n  parsed.forEach((action) => {\n    if (action.action_type === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n      });\n    } else if (action.action_type === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys,\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (action.action_type === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'finished') {\n      shouldContinue = false;\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(`No actions found, response: ${res.content}`, {\n      cause: {\n        prediction: res.content,\n        parsed,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  const log = getSummary(res.content);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: log,\n  });\n\n  return {\n    actions: transformActions,\n    log,\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n    more_actions_needed_by_instruction: shouldContinue,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  uiTarsVersion: UITarsModelVersion | undefined,\n) {\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","vlmPlanning","userInstruction","options","conversationHistory","context","modelConfig","uiTarsModelVersion","systemPrompt","getUiTarsPlanningPrompt","imagePayload","resizeImageForUiTars","res","callAIWithStringResponse","AIActionType","convertedText","convertBboxToCoordinates","size","parsed","actionParser","JSON","transformActions","shouldContinue","action","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","Error","log","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACmBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,YACpBC,eAAuB,EACvBC,OAIC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGH;IACtD,MAAM,EAAEI,kBAAkB,EAAE,GAAGD;IAC/B,MAAME,eAAeC,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,MAA4BP;IAEjD,MAAMQ,eAAe,MAAMC,qBACzBN,QAAQ,gBAAgB,EACxBA,QAAQ,IAAI,EACZE;IAGFH,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKM;gBACP;YACF;SACD;IACH;IAEA,MAAME,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChB;QACE;YACE,MAAM;YACN,SAASL;QACX;WACGJ,oBAAoB,QAAQ;KAChC,EACDU,mCAAAA,YAAAA,CAAAA,eAA4B,EAC5BR;IAEF,MAAMS,gBAAgBC,yBAAyBJ,IAAI,OAAO;IAE1D,MAAM,EAAEK,IAAI,EAAE,GAAGZ;IACjB,MAAM,EAAEa,MAAM,EAAE,GAAGC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC9B,YAAYJ;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOE,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUV;IACZ;IAEAd,MACE,oBACAc,oBACA,YACAa,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7C,IAAIC,iBAAiB;IACrBJ,OAAO,OAAO,CAAC,CAACK;QACd,IAAIA,AAAuB,YAAvBA,OAAO,WAAW,EAAc;YAClCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM1B,QAAQ4B,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEN;YACvDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQE,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BoB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIM,AAAuB,WAAvBA,OAAO,WAAW,EAAa;YACxCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMG,aAAaD,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEN;YAC5D,MAAMU,WAAWF,SAASF,OAAO,aAAa,CAAC,OAAO,EAAEN;YACxDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQE,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAG8B,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCT,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQM,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAG+B,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjCV,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASM,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIA,AAAuB,WAAvBA,OAAO,WAAW,EAC3BF,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOE,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3BF,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWE,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,eAAvBA,OAAO,WAAW,EAC3BF,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASE,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3B,IAAKA,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMK,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBN,OAAO,aAAa,CAAC,GAAG;YAE1DF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASO;gBACX;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEO,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAuB,WAAvBA,OAAO,WAAW,EAC3BF,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASE,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,eAAvBA,OAAO,WAAW,EAC3BD,iBAAiB;IAErB;IAEA,IAAID,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIU,MAAM,CAAC,4BAA4B,EAAEnB,IAAI,OAAO,EAAE,EAAE;QAC5D,OAAO;YACL,YAAYA,IAAI,OAAO;YACvBM;QACF;IACF;IAGFzB,MAAM,oBAAoB2B,KAAK,SAAS,CAACC,kBAAkB,MAAM;IACjE,MAAMW,MAAMC,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWrB,IAAI,OAAO;IAElCR,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS4B;IACX;IAEA,OAAO;QACL,SAASX;QACTW;QACA,OAAOpB,IAAI,KAAK;QAChB,aAAaQ,KAAK,SAAS,CAACR,IAAI,OAAO,EAAEsB,QAAW;QACpD,oCAAoCZ;IACtC;AACF;AAOA,SAASN,yBAAyBmB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAIhD,KAAK,KAAK,CAAE2C,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAIjD,KAAK,KAAK,CAAE6C,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASZ,SAAS0B,QAAgB,EAAElC,IAAuC;IACzE,MAAM,CAAC+B,GAAGC,EAAE,GAAG7B,KAAK,KAAK,CAAC+B;IAC1B,OAAO;QAACH,IAAI/B,KAAK,KAAK;QAAEgC,IAAIhC,KAAK,MAAM;KAAC;AAC1C;AAkEO,eAAeN,qBACpByC,WAAmB,EACnBnC,IAAU,EACVoC,aAA6C;IAE7C,IAAIA,kBAAkBC,oBAAAA,kBAAAA,CAAAA,IAAuB,EAAE;QAC7C7D,MAAM,uCAAuCwB;QAC7C,MAAMsC,gBAAgBtC,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAMuC,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAezD,KAAK,IAAI,CAACwD,YAAYD;YAC3C,MAAMG,WAAW1D,KAAK,KAAK,CAACiB,KAAK,KAAK,GAAGwC;YACzC,MAAME,YAAY3D,KAAK,KAAK,CAACiB,KAAK,MAAM,GAAGwC;YAC3ChE,MACE,2DACAiE,UACAC;YAEF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}