@midscene/core 1.9.8 → 1.10.0

package/dist/lib/ai-model/models/ui-tars/actions.js

@@ -0,0 +1,181 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    transformUiTarsActions: ()=>transformUiTarsActions
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const us_keyboard_layout_namespaceObject = require("@midscene/shared/us-keyboard-layout");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const debug = (0, logger_namespaceObject.getDebug)('ui-tars-planning');
+const warnLog = (0, logger_namespaceObject.getDebug)('ui-tars-planning', {
+    console: true
+});
+function transformUiTarsActions(parsedPlanningResponse) {
+    const transformActions = [];
+    const unhandledActions = [];
+    parsedPlanningResponse.actions.forEach((action)=>{
+        const actionType = (action.action_type || '').toLowerCase();
+        if ('click' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'Tap',
+                param: {
+                    locate
+                }
+            });
+        } else if ('left_double' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'DoubleClick',
+                param: {
+                    locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('right_single' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'RightClick',
+                param: {
+                    locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('drag' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            (0, utils_namespaceObject.assert)(action.action_inputs.end_box, 'end_box is required');
+            const startPoint = getPoint(action.action_inputs.start_box);
+            const endPoint = getPoint(action.action_inputs.end_box);
+            transformActions.push({
+                type: 'DragAndDrop',
+                param: {
+                    from: {
+                        point: startPoint,
+                        prompt: action.thought || ''
+                    },
+                    to: {
+                        point: endPoint,
+                        prompt: action.thought || ''
+                    }
+                },
+                thought: action.thought || ''
+            });
+        } else if ('type' === actionType) transformActions.push({
+            type: 'Input',
+            param: {
+                value: action.action_inputs.content
+            },
+            thought: action.thought || ''
+        });
+        else if ('scroll' === actionType) transformActions.push({
+            type: 'Scroll',
+            param: {
+                direction: action.action_inputs.direction
+            },
+            thought: action.thought || ''
+        });
+        else if ('finished' === actionType) transformActions.push({
+            type: 'Finished',
+            param: {},
+            thought: action.action_inputs.content || action.thought || ''
+        });
+        else if ('hotkey' === actionType) if (action.action_inputs.key) {
+            const keys = (0, us_keyboard_layout_namespaceObject.transformHotkeyInput)(action.action_inputs.key);
+            transformActions.push({
+                type: 'KeyboardPress',
+                param: {
+                    keyName: keys.join('+')
+                },
+                thought: action.thought || ''
+            });
+        } else warnLog('No key found in action: hotkey. Will not perform action.');
+        else if ('wait' === actionType) transformActions.push({
+            type: 'Sleep',
+            param: {
+                timeMs: 1000
+            },
+            thought: action.thought || ''
+        });
+        else if (actionType) {
+            unhandledActions.push({
+                type: actionType,
+                thought: action.thought || ''
+            });
+            debug('Unhandled action type:', actionType, 'thought:', action.thought);
+        }
+    });
+    if (0 === transformActions.length) throw new Error(buildNoUiTarsActionsError(parsedPlanningResponse.rawResponse, parsedPlanningResponse.actions, unhandledActions));
+    debug('transformActions', JSON.stringify(transformActions, null, 2));
+    return transformActions;
+}
+function getPoint(startBox) {
+    const [x, y] = JSON.parse(startBox);
+    (0, utils_namespaceObject.assert)('number' == typeof x && Number.isFinite(x) && 'number' == typeof y && Number.isFinite(y), `invalid point data for ui-tars planning: ${startBox}`);
+    return [
+        x,
+        y
+    ];
+}
+function buildNoUiTarsActionsError(rawResponse, actions, unhandledActions) {
+    const errorDetails = [];
+    if (0 === actions.length) {
+        errorDetails.push('Action parser returned no actions');
+        if (rawResponse.includes('Thought:') && !rawResponse.includes('Action:')) errorDetails.push('Response contains "Thought:" but missing "Action:" line');
+        else errorDetails.push('Response may be malformed or empty');
+    }
+    if (unhandledActions.length > 0) {
+        const types = unhandledActions.map((a)=>a.type).join(', ');
+        errorDetails.push(`Unhandled action types: ${types}`);
+    }
+    return [
+        'No actions found in UI-TARS response.',
+        ...errorDetails
+    ].join('\n');
+}
+exports.transformUiTarsActions = __webpack_exports__.transformUiTarsActions;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "transformUiTarsActions"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=actions.js.map

package/dist/lib/ai-model/models/ui-tars/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  DragAndDropPlanningAction,\n  LocatePlanningAction,\n} from '../../shared/planning-action';\nimport type { UiTarsParsedPlanningResponse } from './parser';\n\nconst debug = getDebug('ui-tars-planning');\nconst warnLog = getDebug('ui-tars-planning', { console: true });\n\nexport function transformUiTarsActions(\n  parsedPlanningResponse: UiTarsParsedPlanningResponse,\n): PlanningAction[] {\n  const transformActions: PlanningAction[] = [];\n  const unhandledActions: Array<{ type: string; thought: string }> = [];\n\n  parsedPlanningResponse.actions.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate,\n        },\n      } satisfies LocatePlanningAction<'Tap'>);\n    } else if (actionType === 'left_double') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'DoubleClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'DoubleClick'>);\n    } else if (actionType === 'right_single') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'RightClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'RightClick'>);\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box);\n      const endPoint = getPoint(action.action_inputs.end_box);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            point: startPoint,\n            prompt: action.thought || '',\n          },\n          to: {\n            point: endPoint,\n            prompt: action.thought || '',\n          },\n        },\n        thought: action.thought || '',\n      } satisfies DragAndDropPlanningAction);\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.action_inputs.content || action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        warnLog('No key found in action: hotkey. Will not perform action.');\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys.join('+'),\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType) {\n      unhandledActions.push({\n        type: actionType,\n        thought: action.thought || '',\n      });\n      debug('Unhandled action type:', actionType, 'thought:', action.thought);\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(\n      buildNoUiTarsActionsError(\n        parsedPlanningResponse.rawResponse,\n        parsedPlanningResponse.actions,\n        unhandledActions,\n      ),\n    );\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  return transformActions;\n}\n\nfunction getPoint(startBox: string): [number, number] {\n  const [x, y] = JSON.parse(startBox);\n  assert(\n    typeof x === 'number' &&\n      Number.isFinite(x) &&\n      typeof y === 'number' &&\n      Number.isFinite(y),\n    `invalid point data for ui-tars planning: ${startBox}`,\n  );\n  return [x, y];\n}\n\nfunction buildNoUiTarsActionsError(\n  rawResponse: string,\n  actions: UiTarsParsedPlanningResponse['actions'],\n  unhandledActions: Array<{ type: string; thought: string }>,\n): string {\n  const errorDetails: string[] = [];\n\n  if (actions.length === 0) {\n    errorDetails.push('Action parser returned no actions');\n\n    if (rawResponse.includes('Thought:') && !rawResponse.includes('Action:')) {\n      errorDetails.push(\n        'Response contains \"Thought:\" but missing \"Action:\" line',\n      );\n    } else {\n      errorDetails.push('Response may be malformed or empty');\n    }\n  }\n\n  if (unhandledActions.length > 0) {\n    const types = unhandledActions.map((a) => a.type).join(', ');\n    errorDetails.push(`Unhandled action types: ${types}`);\n  }\n\n  return ['No actions found in UI-TARS response.', ...errorDetails].join('\\n');\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","transformUiTarsActions","parsedPlanningResponse","transformActions","unhandledActions","action","actionType","assert","point","getPoint","locate","startPoint","endPoint","keys","transformHotkeyInput","Error","buildNoUiTarsActionsError","JSON","startBox","x","y","Number","rawResponse","actions","errorDetails","types","a"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACIA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,oBAAoB;IAAE,SAAS;AAAK;AAEtD,SAASE,uBACdC,sBAAoD;IAEpD,MAAMC,mBAAqC,EAAE;IAC7C,MAAMC,mBAA6D,EAAE;IAErEF,uBAAuB,OAAO,CAAC,OAAO,CAAC,CAACG;QACtC,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;YACF;QACF,OAAO,IAAIJ,AAAe,kBAAfA,YAA8B;YACvCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,mBAAfA,YAA+B;YACxCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YAAuB;YAChCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMM,aAAaF,SAASJ,OAAO,aAAa,CAAC,SAAS;YAC1D,MAAMO,WAAWH,SAASJ,OAAO,aAAa,CAAC,OAAO;YACtDF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,OAAOQ;wBACP,QAAQN,OAAO,OAAO,IAAI;oBAC5B;oBACA,IAAI;wBACF,OAAOO;wBACP,QAAQP,OAAO,OAAO,IAAI;oBAC5B;gBACF;gBACA,SAASA,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOE,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWE,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASE,OAAO,aAAa,CAAC,OAAO,IAAIA,OAAO,OAAO,IAAI;QAC7D;aACK,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAEtB;YACL,MAAMQ,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBT,OAAO,aAAa,CAAC,GAAG;YAE1DF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASU,KAAK,IAAI,CAAC;gBACrB;gBACA,SAASR,OAAO,OAAO,IAAI;YAC7B;QACF,OAXEL,QAAQ;aAYL,IAAIM,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASE,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,YAAY;YACrBF,iBAAiB,IAAI,CAAC;gBACpB,MAAME;gBACN,SAASD,OAAO,OAAO,IAAI;YAC7B;YACAP,MAAM,0BAA0BQ,YAAY,YAAYD,OAAO,OAAO;QACxE;IACF;IAEA,IAAIF,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIY,MACRC,0BACEd,uBAAuB,WAAW,EAClCA,uBAAuB,OAAO,EAC9BE;IAKNN,MAAM,oBAAoBmB,KAAK,SAAS,CAACd,kBAAkB,MAAM;IACjE,OAAOA;AACT;AAEA,SAASM,SAASS,QAAgB;IAChC,MAAM,CAACC,GAAGC,EAAE,GAAGH,KAAK,KAAK,CAACC;IAC1BX,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAa,YAAb,OAAOY,KACLE,OAAO,QAAQ,CAACF,MAChB,AAAa,YAAb,OAAOC,KACPC,OAAO,QAAQ,CAACD,IAClB,CAAC,yCAAyC,EAAEF,UAAU;IAExD,OAAO;QAACC;QAAGC;KAAE;AACf;AAEA,SAASJ,0BACPM,WAAmB,EACnBC,OAAgD,EAChDnB,gBAA0D;IAE1D,MAAMoB,eAAyB,EAAE;IAEjC,IAAID,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;QACxBC,aAAa,IAAI,CAAC;QAElB,IAAIF,YAAY,QAAQ,CAAC,eAAe,CAACA,YAAY,QAAQ,CAAC,YAC5DE,aAAa,IAAI,CACf;aAGFA,aAAa,IAAI,CAAC;IAEtB;IAEA,IAAIpB,iBAAiB,MAAM,GAAG,GAAG;QAC/B,MAAMqB,QAAQrB,iBAAiB,GAAG,CAAC,CAACsB,IAAMA,EAAE,IAAI,EAAE,IAAI,CAAC;QACvDF,aAAa,IAAI,CAAC,CAAC,wBAAwB,EAAEC,OAAO;IACtD;IAEA,OAAO;QAAC;WAA4CD;KAAa,CAAC,IAAI,CAAC;AACzE"}

package/dist/lib/ai-model/models/ui-tars/adapter.js

@@ -150,7 +150,7 @@ function createUiTarsAdapter(uiTarsModelVersion) {
             kind: 'custom',
             cacheEnabled: false,
             defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,
-            planFn: (userInstruction, options)=>(0, external_planning_js_namespaceObject.uiTarsPlanning)(userInstruction, options, uiTarsModelVersion)
+            planner: (0, external_planning_js_namespaceObject.createUiTarsPlanner)(uiTarsModelVersion)
         },
         locate: {
             resultAdapter: {

package/dist/lib/ai-model/models/ui-tars/adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/ui-tars/adapter.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/adapter.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TModelFamily, UITarsModelVersion } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport {\n  extractJSONFromCodeBlock,\n  safeParseJson,\n} from '../../service-caller/json';\nimport {\n  type LocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../../shared/model-locate-result';\nimport type {\n  JsonParserContext,\n  JsonParserSource,\n  ModelAdapterDefinition,\n} from '../types';\nimport { uiTarsPlanning } from './planning';\n\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nfunction normalizeJsonObject(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item, context));\n  }\n\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : normalizeJsonObject(value, context);\n      normalized[trimmedKey] = normalizedValue;\n    }\n    return normalized;\n  }\n\n  return typeof obj === 'string' ? obj.trim() : obj;\n}\n\nfunction shouldRepairUiTarsLocateJson(source: JsonParserSource) {\n  return (\n    source === 'locate' ||\n    source === 'section-locator' ||\n    source === 'planning-action-param'\n  );\n}\n\nfunction preprocessUiTarsLocateJson(input: string) {\n  if (input.includes('bbox')) {\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nconst uiTarsJsonParser: ModelAdapterDefinition['jsonParser'] = (\n  raw,\n  context = { source: 'generic-object' },\n) => {\n  const { source } = context;\n  try {\n    return safeParseJson(raw, context);\n  } catch (firstError) {\n    if (!shouldRepairUiTarsLocateJson(source)) {\n      throw firstError;\n    }\n\n    const jsonString = preprocessUiTarsLocateJson(\n      extractJSONFromCodeBlock(raw),\n    );\n    try {\n      return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context);\n    } catch (error) {\n      throw Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          error ?? firstError ?? 'unknown error',\n        )}. Response - \\n ${raw}`,\n      );\n    }\n  }\n};\n\n// UI-TARS has not received active updates for a long time, so this parser is\n// intentionally kept separate from Doubao even though the current logic is the\n// same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.\nfunction parseUiTarsRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for ui-tars mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return {\n        type: 'bbox',\n        coordinates: [\n          Number(splitted[0]),\n          Number(splitted[1]),\n          Number(splitted[2]),\n          Number(splitted[3]),\n        ],\n      };\n    }\n    throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return {\n      type: 'bbox',\n      coordinates: [bboxList[0], bboxList[1], bboxList[2], bboxList[3]],\n    };\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return { type: 'point', coordinates: [bboxList[0], bboxList[1]] };\n  }\n\n  if (bbox.length === 8) {\n    return {\n      type: 'bbox',\n      coordinates: [bboxList[0], bboxList[1], bboxList[4], bboxList[5]],\n    };\n  }\n\n  const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction createUiTarsAdapter(\n  uiTarsModelVersion: UITarsModelVersion,\n): ModelAdapterDefinition {\n  return {\n    jsonParser: uiTarsJsonParser,\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,\n      planFn: (userInstruction, options) =>\n        uiTarsPlanning(userInstruction, options, uiTarsModelVersion),\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n        parseRawLocateValue: parseUiTarsRawLocateValue,\n      },\n    },\n  };\n}\n\nconst uiTarsDoubao15Adapter = createUiTarsAdapter(\n  UITarsModelVersion.DOUBAO_1_5_20B,\n);\n\nexport const uiTarsAdapters = {\n  'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),\n  'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,\n  'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5'\n>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultVlmUiTarsReplanningCycleLimit","normalizeJsonObject","context","Array","item","normalized","value","trimmedKey","preserveStringValue","normalizedValue","shouldRepairUiTarsLocateJson","source","preprocessUiTarsLocateJson","input","uiTarsJsonParser","raw","safeParseJson","firstError","jsonString","extractJSONFromCodeBlock","JSON","jsonrepair","error","Error","String","parseUiTarsRawLocateValue","bbox","unwrapCoordinateListLikeInput","assert","splitted","Number","bboxList","x","y","msg","createUiTarsAdapter","uiTarsModelVersion","midsceneDefaults","userConfig","commonOverrideConfig","undefined","userInstruction","options","uiTarsPlanning","uiTarsDoubao15Adapter","UITarsModelVersion","uiTarsAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACYA,MAAMI,uCAAuC;AAE7C,SAASC,oBACPJ,GAAQ,EACRK,UAA8D,CAAC,CAAC;IAEhE,IAAIL,QAAAA,KACF,OAAOA;IAGT,IAAIM,MAAM,OAAO,CAACN,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACO,OAASH,oBAAoBG,MAAMF;IAGrD,IAAI,AAAe,YAAf,OAAOL,KAAkB;QAC3B,MAAMQ,aAAkB,CAAC;QACzB,KAAK,MAAM,CAACV,KAAKW,MAAM,IAAIV,OAAO,OAAO,CAACC,KAAM;YAC9C,MAAMU,aAAaZ,IAAI,IAAI;YAC3B,MAAMa,sBACJN,QAAQ,uBAAuB,EAAE,SAASK,eAAe;YAC3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOH,QACHE,sBACEF,QACAA,MAAM,IAAI,KACZL,oBAAoBK,OAAOJ;YACjCG,UAAU,CAACE,WAAW,GAAGE;QAC3B;QACA,OAAOJ;IACT;IAEA,OAAO,AAAe,YAAf,OAAOR,MAAmBA,IAAI,IAAI,KAAKA;AAChD;AAEA,SAASa,6BAA6BC,MAAwB;IAC5D,OACEA,AAAW,aAAXA,UACAA,AAAW,sBAAXA,UACAA,AAAW,4BAAXA;AAEJ;AAEA,SAASC,2BAA2BC,KAAa;IAC/C,IAAIA,MAAM,QAAQ,CAAC,SACjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEA,MAAMC,mBAAyD,CAC7DC,KACAb,UAAU;IAAE,QAAQ;AAAiB,CAAC;IAEtC,MAAM,EAAES,MAAM,EAAE,GAAGT;IACnB,IAAI;QACF,OAAOc,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcD,KAAKb;IAC5B,EAAE,OAAOe,YAAY;QACnB,IAAI,CAACP,6BAA6BC,SAChC,MAAMM;QAGR,MAAMC,aAAaN,2BACjBO,AAAAA,IAAAA,wBAAAA,wBAAAA,AAAAA,EAAyBJ;QAE3B,IAAI;YACF,OAAOd,oBAAoBmB,KAAK,KAAK,CAACC,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWH,cAAchB;QACjE,EAAE,OAAOoB,OAAO;YACd,MAAMC,MACJ,CAAC,gDAAgD,EAAEC,OACjDF,SAASL,cAAc,iBACvB,gBAAgB,EAAEF,KAAK;QAE7B;IACF;AACF;AAKA,SAASU,0BAA0BZ,KAAc;IAC/C,MAAMa,OAAOC,AAAAA,IAAAA,yBAAAA,6BAAAA,AAAAA,EAA8Bd;IAC3C,IAAI,AAAgB,YAAhB,OAAOa,MAAmB;QAC5BE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,2CAA2C,EAAEA,MAAM;QAEtD,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,MAAM;YACN,aAAa;gBACXC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;aACnB;QACH;QAEF,MAAM,IAAIN,MAAM,CAAC,2CAA2C,EAAEG,MAAM;IACtE;IAEA,IAAIK,WAAqB,EAAE;IAC3B,IAAI5B,MAAM,OAAO,CAACuB,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACtB;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAAC4B,GAAGC,EAAE,GAAG7B,KAAK,KAAK,CAAC;YAC1B2B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAO7B,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAAC4B,GAAGC,EAAE,GAAG7B,KAAK,KAAK,CAAC;YAC1B2B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OACEF,SAAS,IAAI,CAACD,OAAO1B;IAEzB;SAEA2B,WAAWL;IAGb,IAAIK,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACL,MAAM;QACN,aAAa;YAACA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IACnE;IAGF,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QAAE,MAAM;QAAS,aAAa;YAACA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IAAC;IAGlE,IAAIL,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACL,MAAM;QACN,aAAa;YAACK,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IACnE;IAGF,MAAMG,MAAM,CAAC,oCAAoC,EAAEd,KAAK,SAAS,CAACM,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAIH,MAAMW;AAClB;AAEA,SAASC,oBACPC,kBAAsC;IAEtC,OAAO;QACL,YAAYtB;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEuB,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,OAAO;oBACL,QAAQ;wBACN,GAAGD,gBAAgB;wBACnB,GAAGE,oBAAoB;oBACzB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BvC;YAC7B,QAAQ,CAACyC,iBAAiBC,UACxBC,AAAAA,IAAAA,qCAAAA,cAAAA,AAAAA,EAAeF,iBAAiBC,SAASN;QAC7C;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;gBAC9D,qBAAqBX;YACvB;QACF;IACF;AACF;AAEA,MAAMmB,wBAAwBT,oBAC5BU,oBAAAA,kBAAAA,CAAAA,cAAiC;AAG5B,MAAMC,iBAAiB;IAC5B,eAAeX,oBAAoBU,oBAAAA,kBAAAA,CAAAA,IAAuB;IAC1D,sBAAsBD;IACtB,0BAA0BA;AAC5B"}
+{"version":3,"file":"ai-model/models/ui-tars/adapter.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/adapter.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TModelFamily, UITarsModelVersion } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport type {\n  JsonParserContext,\n  JsonParserSource,\n  ModelAdapterDefinition,\n} from '../../model-adapter/types';\nimport {\n  extractJSONFromCodeBlock,\n  safeParseJson,\n} from '../../service-caller/json';\nimport {\n  type LocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../../shared/model-locate-result';\nimport { createUiTarsPlanner } from './planning';\n\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nfunction normalizeJsonObject(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item, context));\n  }\n\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : normalizeJsonObject(value, context);\n      normalized[trimmedKey] = normalizedValue;\n    }\n    return normalized;\n  }\n\n  return typeof obj === 'string' ? obj.trim() : obj;\n}\n\nfunction shouldRepairUiTarsLocateJson(source: JsonParserSource) {\n  return (\n    source === 'locate' ||\n    source === 'section-locator' ||\n    source === 'planning-action-param'\n  );\n}\n\nfunction preprocessUiTarsLocateJson(input: string) {\n  if (input.includes('bbox')) {\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nconst uiTarsJsonParser: ModelAdapterDefinition['jsonParser'] = (\n  raw,\n  context = { source: 'generic-object' },\n) => {\n  const { source } = context;\n  try {\n    return safeParseJson(raw, context);\n  } catch (firstError) {\n    if (!shouldRepairUiTarsLocateJson(source)) {\n      throw firstError;\n    }\n\n    const jsonString = preprocessUiTarsLocateJson(\n      extractJSONFromCodeBlock(raw),\n    );\n    try {\n      return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context);\n    } catch (error) {\n      throw Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          error ?? firstError ?? 'unknown error',\n        )}. Response - \\n ${raw}`,\n      );\n    }\n  }\n};\n\n// UI-TARS has not received active updates for a long time, so this parser is\n// intentionally kept separate from Doubao even though the current logic is the\n// same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.\nfunction parseUiTarsRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for ui-tars mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return {\n        type: 'bbox',\n        coordinates: [\n          Number(splitted[0]),\n          Number(splitted[1]),\n          Number(splitted[2]),\n          Number(splitted[3]),\n        ],\n      };\n    }\n    throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return {\n      type: 'bbox',\n      coordinates: [bboxList[0], bboxList[1], bboxList[2], bboxList[3]],\n    };\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return { type: 'point', coordinates: [bboxList[0], bboxList[1]] };\n  }\n\n  if (bbox.length === 8) {\n    return {\n      type: 'bbox',\n      coordinates: [bboxList[0], bboxList[1], bboxList[4], bboxList[5]],\n    };\n  }\n\n  const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction createUiTarsAdapter(\n  uiTarsModelVersion: UITarsModelVersion,\n): ModelAdapterDefinition {\n  return {\n    jsonParser: uiTarsJsonParser,\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,\n      planner: createUiTarsPlanner(uiTarsModelVersion),\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n        parseRawLocateValue: parseUiTarsRawLocateValue,\n      },\n    },\n  };\n}\n\nconst uiTarsDoubao15Adapter = createUiTarsAdapter(\n  UITarsModelVersion.DOUBAO_1_5_20B,\n);\n\nexport const uiTarsAdapters = {\n  'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),\n  'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,\n  'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5'\n>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultVlmUiTarsReplanningCycleLimit","normalizeJsonObject","context","Array","item","normalized","value","trimmedKey","preserveStringValue","normalizedValue","shouldRepairUiTarsLocateJson","source","preprocessUiTarsLocateJson","input","uiTarsJsonParser","raw","safeParseJson","firstError","jsonString","extractJSONFromCodeBlock","JSON","jsonrepair","error","Error","String","parseUiTarsRawLocateValue","bbox","unwrapCoordinateListLikeInput","assert","splitted","Number","bboxList","x","y","msg","createUiTarsAdapter","uiTarsModelVersion","midsceneDefaults","userConfig","commonOverrideConfig","undefined","createUiTarsPlanner","uiTarsDoubao15Adapter","UITarsModelVersion","uiTarsAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACYA,MAAMI,uCAAuC;AAE7C,SAASC,oBACPJ,GAAQ,EACRK,UAA8D,CAAC,CAAC;IAEhE,IAAIL,QAAAA,KACF,OAAOA;IAGT,IAAIM,MAAM,OAAO,CAACN,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACO,OAASH,oBAAoBG,MAAMF;IAGrD,IAAI,AAAe,YAAf,OAAOL,KAAkB;QAC3B,MAAMQ,aAAkB,CAAC;QACzB,KAAK,MAAM,CAACV,KAAKW,MAAM,IAAIV,OAAO,OAAO,CAACC,KAAM;YAC9C,MAAMU,aAAaZ,IAAI,IAAI;YAC3B,MAAMa,sBACJN,QAAQ,uBAAuB,EAAE,SAASK,eAAe;YAC3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOH,QACHE,sBACEF,QACAA,MAAM,IAAI,KACZL,oBAAoBK,OAAOJ;YACjCG,UAAU,CAACE,WAAW,GAAGE;QAC3B;QACA,OAAOJ;IACT;IAEA,OAAO,AAAe,YAAf,OAAOR,MAAmBA,IAAI,IAAI,KAAKA;AAChD;AAEA,SAASa,6BAA6BC,MAAwB;IAC5D,OACEA,AAAW,aAAXA,UACAA,AAAW,sBAAXA,UACAA,AAAW,4BAAXA;AAEJ;AAEA,SAASC,2BAA2BC,KAAa;IAC/C,IAAIA,MAAM,QAAQ,CAAC,SACjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEA,MAAMC,mBAAyD,CAC7DC,KACAb,UAAU;IAAE,QAAQ;AAAiB,CAAC;IAEtC,MAAM,EAAES,MAAM,EAAE,GAAGT;IACnB,IAAI;QACF,OAAOc,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcD,KAAKb;IAC5B,EAAE,OAAOe,YAAY;QACnB,IAAI,CAACP,6BAA6BC,SAChC,MAAMM;QAGR,MAAMC,aAAaN,2BACjBO,AAAAA,IAAAA,wBAAAA,wBAAAA,AAAAA,EAAyBJ;QAE3B,IAAI;YACF,OAAOd,oBAAoBmB,KAAK,KAAK,CAACC,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWH,cAAchB;QACjE,EAAE,OAAOoB,OAAO;YACd,MAAMC,MACJ,CAAC,gDAAgD,EAAEC,OACjDF,SAASL,cAAc,iBACvB,gBAAgB,EAAEF,KAAK;QAE7B;IACF;AACF;AAKA,SAASU,0BAA0BZ,KAAc;IAC/C,MAAMa,OAAOC,AAAAA,IAAAA,yBAAAA,6BAAAA,AAAAA,EAA8Bd;IAC3C,IAAI,AAAgB,YAAhB,OAAOa,MAAmB;QAC5BE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,2CAA2C,EAAEA,MAAM;QAEtD,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,MAAM;YACN,aAAa;gBACXC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;gBAClBC,OAAOD,QAAQ,CAAC,EAAE;aACnB;QACH;QAEF,MAAM,IAAIN,MAAM,CAAC,2CAA2C,EAAEG,MAAM;IACtE;IAEA,IAAIK,WAAqB,EAAE;IAC3B,IAAI5B,MAAM,OAAO,CAACuB,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACtB;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAAC4B,GAAGC,EAAE,GAAG7B,KAAK,KAAK,CAAC;YAC1B2B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAO7B,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAAC4B,GAAGC,EAAE,GAAG7B,KAAK,KAAK,CAAC;YAC1B2B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OACEF,SAAS,IAAI,CAACD,OAAO1B;IAEzB;SAEA2B,WAAWL;IAGb,IAAIK,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACL,MAAM;QACN,aAAa;YAACA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IACnE;IAGF,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QAAE,MAAM;QAAS,aAAa;YAACA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IAAC;IAGlE,IAAIL,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACL,MAAM;QACN,aAAa;YAACK,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;YAAEA,QAAQ,CAAC,EAAE;SAAC;IACnE;IAGF,MAAMG,MAAM,CAAC,oCAAoC,EAAEd,KAAK,SAAS,CAACM,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAIH,MAAMW;AAClB;AAEA,SAASC,oBACPC,kBAAsC;IAEtC,OAAO;QACL,YAAYtB;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEuB,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,OAAO;oBACL,QAAQ;wBACN,GAAGD,gBAAgB;wBACnB,GAAGE,oBAAoB;oBACzB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BvC;YAC7B,SAASyC,AAAAA,IAAAA,qCAAAA,mBAAAA,AAAAA,EAAoBL;QAC/B;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;gBAC9D,qBAAqBX;YACvB;QACF;IACF;AACF;AAEA,MAAMiB,wBAAwBP,oBAC5BQ,oBAAAA,kBAAAA,CAAAA,cAAiC;AAG5B,MAAMC,iBAAiB;IAC5B,eAAeT,oBAAoBQ,oBAAAA,kBAAAA,CAAAA,IAAuB;IAC1D,sBAAsBD;IACtB,0BAA0BA;AAC5B"}

package/dist/lib/ai-model/models/ui-tars/parser.js

@@ -0,0 +1,74 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    parseUiTarsPlanningResponse: ()=>parseUiTarsPlanningResponse
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const action_parser_namespaceObject = require("@ui-tars/action-parser");
+const debug = (0, logger_namespaceObject.getDebug)('ui-tars-planning');
+function parseUiTarsPlanningResponse(rawResponse, shotSize, uiTarsModelVersion) {
+    const convertedText = convertBboxToCoordinates(rawResponse);
+    const parseResult = (0, action_parser_namespaceObject.actionParser)({
+        prediction: convertedText,
+        factor: [
+            1000,
+            1000
+        ],
+        screenContext: {
+            width: shotSize.width,
+            height: shotSize.height
+        },
+        modelVer: uiTarsModelVersion
+    });
+    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parseResult.parsed));
+    return {
+        rawResponse,
+        actions: parseResult.parsed
+    };
+}
+function convertBboxToCoordinates(text) {
+    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
+    function replaceMatch(match, x1, y1, x2, y2) {
+        const x1Num = Number.parseInt(x1, 10);
+        const y1Num = Number.parseInt(y1, 10);
+        const x2Num = Number.parseInt(x2, 10);
+        const y2Num = Number.parseInt(y2, 10);
+        const x = Math.floor((x1Num + x2Num) / 2);
+        const y = Math.floor((y1Num + y2Num) / 2);
+        return `(${x},${y})`;
+    }
+    const cleanedText = text.replace(/\[EOS\]/g, '').replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');
+    return cleanedText.replace(pattern, replaceMatch).trim();
+}
+exports.parseUiTarsPlanningResponse = __webpack_exports__.parseUiTarsPlanningResponse;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "parseUiTarsPlanningResponse"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=parser.js.map

package/dist/lib/ai-model/models/ui-tars/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/parser.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/parser.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Size } from '@/types';\nimport type { UITarsModelVersion } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { actionParser } from '@ui-tars/action-parser';\n\ntype ActionType =\n  | 'click'\n  | 'left_double'\n  | 'right_single'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\n\nexport interface UiTarsParsedPlanningResponse {\n  rawResponse: string;\n  actions: ReturnType<typeof actionParser>['parsed'];\n}\n\nexport function parseUiTarsPlanningResponse(\n  rawResponse: string,\n  shotSize: Size,\n  uiTarsModelVersion: UITarsModelVersion,\n): UiTarsParsedPlanningResponse {\n  const convertedText = convertBboxToCoordinates(rawResponse);\n  const parseResult = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: shotSize.width,\n      height: shotSize.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parseResult.parsed),\n  );\n\n  return {\n    rawResponse,\n    actions: parseResult.parsed,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points.\n */\nfunction convertBboxToCoordinates(text: string): string {\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    return `(${x},${y})`;\n  }\n\n  const cleanedText = text\n    .replace(/\\[EOS\\]/g, '')\n    .replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string;\n    end_box: string;\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string;\n  };\n}\n\ninterface LeftDoubleAction extends BaseAction {\n  action_type: 'left_double';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface RightSingleAction extends BaseAction {\n  action_type: 'right_single';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: {\n    content?: string;\n  };\n}\n\nexport type Action =\n  | ClickAction\n  | LeftDoubleAction\n  | RightSingleAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","parseUiTarsPlanningResponse","rawResponse","shotSize","uiTarsModelVersion","convertedText","convertBboxToCoordinates","parseResult","actionParser","JSON","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","Math","y","cleanedText"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACUA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAOhB,SAASC,4BACdC,WAAmB,EACnBC,QAAc,EACdC,kBAAsC;IAEtC,MAAMC,gBAAgBC,yBAAyBJ;IAC/C,MAAMK,cAAcC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC/B,YAAYH;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOF,SAAS,KAAK;YACrB,QAAQA,SAAS,MAAM;QACzB;QACA,UAAUC;IACZ;IAEAL,MACE,oBACAK,oBACA,YACAK,KAAK,SAAS,CAACF,YAAY,MAAM;IAGnC,OAAO;QACLL;QACA,SAASK,YAAY,MAAM;IAC7B;AACF;AAKA,SAASD,yBAAyBI,IAAY;IAC5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAEV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAElC,MAAMM,IAAIC,KAAK,KAAK,CAAEN,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMI,IAAID,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAEvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEE,EAAE,CAAC,CAAC;IACtB;IAEA,MAAMC,cAAchB,KACjB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,2BAA2B;IACtC,OAAOgB,YAAY,OAAO,CAACf,SAASC,cAAc,IAAI;AACxD"}

package/dist/lib/ai-model/models/ui-tars/planning.js

@@ -24,244 +24,33 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    uiTarsPlanning: ()=>uiTarsPlanning
+    createUiTarsPlanner: ()=>createUiTarsPlanner
 });
-const external_common_js_namespaceObject = require("../../../common.js");
-const logger_namespaceObject = require("@midscene/shared/logger");
-const us_keyboard_layout_namespaceObject = require("@midscene/shared/us-keyboard-layout");
-const utils_namespaceObject = require("@midscene/shared/utils");
-const action_parser_namespaceObject = require("@ui-tars/action-parser");
-const ui_tars_planning_js_namespaceObject = require("../../prompt/ui-tars-planning.js");
-const index_js_namespaceObject = require("../../service-caller/index.js");
-const bbox_js_namespaceObject = require("../../shared/model-locate-result/bbox.js");
-const pixel_bbox_mapper_js_namespaceObject = require("../../shared/model-locate-result/pixel-bbox-mapper.js");
-const debug = (0, logger_namespaceObject.getDebug)('ui-tars-planning');
-const warnLog = (0, logger_namespaceObject.getDebug)('ui-tars-planning', {
-    console: true
-});
-function pointToLocateParam(point, thought, size) {
-    const ctx = {
-        preparedSize: size
-    };
-    const pixelBbox = (0, pixel_bbox_mapper_js_namespaceObject.mapLocateResultToPixelBboxByCoordinates)({
-        type: 'point',
-        coordinates: point
-    }, ctx, {
-        shape: 'point',
-        order: 'xy',
-        normalizedBy: 1
-    });
+const external_actions_js_namespaceObject = require("./actions.js");
+const external_parser_js_namespaceObject = require("./parser.js");
+const external_prompt_js_namespaceObject = require("./prompt.js");
+function createUiTarsPlanner(uiTarsModelVersion) {
     return {
-        prompt: thought || '',
-        locatedPixelBbox: (0, bbox_js_namespaceObject.finalizePixelBbox)(pixelBbox, point, ctx)
-    };
-}
-async function uiTarsPlanning(userInstruction, options, uiTarsModelVersion) {
-    const { conversationHistory, context, modelRuntime, actionContext } = options;
-    const userInstructionText = (0, external_common_js_namespaceObject.userPromptToString)(userInstruction);
-    let instruction = userInstructionText;
-    if (actionContext) instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n<user_instruction>${userInstructionText}</user_instruction>`;
-    const systemPrompt = (0, ui_tars_planning_js_namespaceObject.getUiTarsPlanningPrompt)() + instruction;
-    const screenshotBase64 = context.screenshot.base64;
-    const referenceImageMessages = options.referenceImageMessages ?? [];
-    conversationHistory.append({
-        role: 'user',
-        content: [
-            {
-                type: 'image_url',
-                image_url: {
-                    url: screenshotBase64
-                }
-            }
-        ]
-    });
-    const res = await (0, index_js_namespaceObject.callAIWithStringResponse)([
-        {
-            role: 'user',
-            content: systemPrompt
+        messages: {
+            systemPromptPlacement: 'user-message',
+            buildSystemPrompt: external_prompt_js_namespaceObject.getUiTarsPlanningPrompt,
+            buildUserInstruction: (instruction)=>`<user_instruction>${instruction}</user_instruction>`,
+            buildAssistantContent: (_parsedResponse, rawResponse)=>(0, external_prompt_js_namespaceObject.getSummary)(rawResponse)
         },
-        ...referenceImageMessages,
-        ...conversationHistory.snapshot()
-    ], modelRuntime, {
-        abortSignal: options.abortSignal
-    });
-    let convertedText;
-    let parsed;
-    try {
-        convertedText = convertBboxToCoordinates(res.content);
-        const { shotSize } = context;
-        const parseResult = (0, action_parser_namespaceObject.actionParser)({
-            prediction: convertedText,
-            factor: [
-                1000,
-                1000
-            ],
-            screenContext: {
-                width: shotSize.width,
-                height: shotSize.height
-            },
-            modelVer: uiTarsModelVersion
-        });
-        parsed = parseResult.parsed;
-    } catch (parseError) {
-        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
-        throw new index_js_namespaceObject.AIResponseParseError(`Parse error: ${errorMessage}`, JSON.stringify(res.content, void 0, 2), res.usage, res.rawChoiceMessage);
-    }
-    const { shotSize } = context;
-    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
-    const transformActions = [];
-    const unhandledActions = [];
-    let shouldContinue = true;
-    parsed.forEach((action)=>{
-        const actionType = (action.action_type || '').toLowerCase();
-        if ('click' === actionType) {
-            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
-            const point = getPoint(action.action_inputs.start_box);
-            const locate = pointToLocateParam(point, action.thought, shotSize);
-            transformActions.push({
-                type: 'Tap',
-                param: {
-                    locate
-                }
-            });
-        } else if ('left_double' === actionType) {
-            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
-            const point = getPoint(action.action_inputs.start_box);
-            const locate = pointToLocateParam(point, action.thought, shotSize);
-            transformActions.push({
-                type: 'DoubleClick',
-                param: {
-                    locate
-                },
-                thought: action.thought || ''
-            });
-        } else if ('right_single' === actionType) {
-            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
-            const point = getPoint(action.action_inputs.start_box);
-            const locate = pointToLocateParam(point, action.thought, shotSize);
-            transformActions.push({
-                type: 'RightClick',
-                param: {
-                    locate
-                },
-                thought: action.thought || ''
-            });
-        } else if ('drag' === actionType) {
-            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
-            (0, utils_namespaceObject.assert)(action.action_inputs.end_box, 'end_box is required');
-            const startPoint = getPoint(action.action_inputs.start_box);
-            const endPoint = getPoint(action.action_inputs.end_box);
-            transformActions.push({
-                type: 'DragAndDrop',
-                param: {
-                    from: pointToLocateParam(startPoint, action.thought, shotSize),
-                    to: pointToLocateParam(endPoint, action.thought, shotSize)
-                },
-                thought: action.thought || ''
-            });
-        } else if ('type' === actionType) transformActions.push({
-            type: 'Input',
-            param: {
-                value: action.action_inputs.content
-            },
-            thought: action.thought || ''
-        });
-        else if ('scroll' === actionType) transformActions.push({
-            type: 'Scroll',
-            param: {
-                direction: action.action_inputs.direction
-            },
-            thought: action.thought || ''
-        });
-        else if ('finished' === actionType) {
-            shouldContinue = false;
-            transformActions.push({
-                type: 'Finished',
-                param: {},
-                thought: action.action_inputs.content || action.thought || ''
-            });
-        } else if ('hotkey' === actionType) if (action.action_inputs.key) {
-            const keys = (0, us_keyboard_layout_namespaceObject.transformHotkeyInput)(action.action_inputs.key);
-            transformActions.push({
-                type: 'KeyboardPress',
-                param: {
-                    keyName: keys.join('+')
-                },
-                thought: action.thought || ''
-            });
-        } else warnLog('No key found in action: hotkey. Will not perform action.');
-        else if ('wait' === actionType) transformActions.push({
-            type: 'Sleep',
-            param: {
-                timeMs: 1000
-            },
-            thought: action.thought || ''
-        });
-        else if (actionType) {
-            unhandledActions.push({
-                type: actionType,
-                thought: action.thought || ''
-            });
-            debug('Unhandled action type:', actionType, 'thought:', action.thought);
-        }
-    });
-    if (0 === transformActions.length) {
-        const errorDetails = [];
-        if (0 === parsed.length) {
-            errorDetails.push('Action parser returned no actions');
-            if (res.content.includes('Thought:') && !res.content.includes('Action:')) errorDetails.push('Response contains "Thought:" but missing "Action:" line');
-            else errorDetails.push('Response may be malformed or empty');
-        }
-        if (unhandledActions.length > 0) {
-            const types = unhandledActions.map((a)=>a.type).join(', ');
-            errorDetails.push(`Unhandled action types: ${types}`);
-        }
-        const errorMessage = [
-            'No actions found in UI-TARS response.',
-            ...errorDetails
-        ].join('\n');
-        throw new index_js_namespaceObject.AIResponseParseError(errorMessage, JSON.stringify(res.content, void 0, 2), res.usage, res.rawChoiceMessage);
-    }
-    debug('transformActions', JSON.stringify(transformActions, null, 2));
-    const log = (0, ui_tars_planning_js_namespaceObject.getSummary)(res.content);
-    conversationHistory.append({
-        role: 'assistant',
-        content: log
-    });
-    return {
-        actions: transformActions,
-        log,
-        usage: res.usage,
-        rawResponse: JSON.stringify(res.content, void 0, 2),
-        rawChoiceMessage: res.rawChoiceMessage,
-        shouldContinuePlanning: shouldContinue
+        coordinates: {
+            shape: 'point',
+            order: 'xy',
+            normalizedBy: 1
+        },
+        parseResponse: (rawResponse, { options })=>(0, external_parser_js_namespaceObject.parseUiTarsPlanningResponse)(rawResponse, options.context.shotSize, uiTarsModelVersion),
+        transformActions: (parsedPlanningResponse)=>(0, external_actions_js_namespaceObject.transformUiTarsActions)(parsedPlanningResponse),
+        shouldContinuePlanning: (_parsedResponse, actions)=>actions.every((action)=>'Finished' !== action.type),
+        buildResponseLog: (_parsedResponse, rawResponse)=>(0, external_prompt_js_namespaceObject.getSummary)(rawResponse)
     };
 }
-function convertBboxToCoordinates(text) {
-    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
-    function replaceMatch(match, x1, y1, x2, y2) {
-        const x1Num = Number.parseInt(x1, 10);
-        const y1Num = Number.parseInt(y1, 10);
-        const x2Num = Number.parseInt(x2, 10);
-        const y2Num = Number.parseInt(y2, 10);
-        const x = Math.floor((x1Num + x2Num) / 2);
-        const y = Math.floor((y1Num + y2Num) / 2);
-        return `(${x},${y})`;
-    }
-    const cleanedText = text.replace(/\[EOS\]/g, '').replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');
-    return cleanedText.replace(pattern, replaceMatch).trim();
-}
-function getPoint(startBox) {
-    const [x, y] = JSON.parse(startBox);
-    (0, utils_namespaceObject.assert)('number' == typeof x && Number.isFinite(x) && 'number' == typeof y && Number.isFinite(y), `invalid point data for ui-tars planning: ${startBox}`);
-    return [
-        x,
-        y
-    ];
-}
-exports.uiTarsPlanning = __webpack_exports__.uiTarsPlanning;
+exports.createUiTarsPlanner = __webpack_exports__.createUiTarsPlanner;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
-    "uiTarsPlanning"
+    "createUiTarsPlanner"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true