@aiscene/core 9.0.6 → 9.0.8

package/dist/lib/ai-model/models/ui-tars/actions.js

@@ -0,0 +1,181 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    transformUiTarsActions: ()=>transformUiTarsActions
+});
+const logger_namespaceObject = require("@aiscene/shared/logger");
+const us_keyboard_layout_namespaceObject = require("@aiscene/shared/us-keyboard-layout");
+const utils_namespaceObject = require("@aiscene/shared/utils");
+const debug = (0, logger_namespaceObject.getDebug)('ui-tars-planning');
+const warnLog = (0, logger_namespaceObject.getDebug)('ui-tars-planning', {
+    console: true
+});
+function transformUiTarsActions(parsedPlanningResponse) {
+    const transformActions = [];
+    const unhandledActions = [];
+    parsedPlanningResponse.actions.forEach((action)=>{
+        const actionType = (action.action_type || '').toLowerCase();
+        if ('click' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'Tap',
+                param: {
+                    locate
+                }
+            });
+        } else if ('left_double' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'DoubleClick',
+                param: {
+                    locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('right_single' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box);
+            const locate = {
+                point,
+                prompt: action.thought || ''
+            };
+            transformActions.push({
+                type: 'RightClick',
+                param: {
+                    locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('drag' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            (0, utils_namespaceObject.assert)(action.action_inputs.end_box, 'end_box is required');
+            const startPoint = getPoint(action.action_inputs.start_box);
+            const endPoint = getPoint(action.action_inputs.end_box);
+            transformActions.push({
+                type: 'DragAndDrop',
+                param: {
+                    from: {
+                        point: startPoint,
+                        prompt: action.thought || ''
+                    },
+                    to: {
+                        point: endPoint,
+                        prompt: action.thought || ''
+                    }
+                },
+                thought: action.thought || ''
+            });
+        } else if ('type' === actionType) transformActions.push({
+            type: 'Input',
+            param: {
+                value: action.action_inputs.content
+            },
+            thought: action.thought || ''
+        });
+        else if ('scroll' === actionType) transformActions.push({
+            type: 'Scroll',
+            param: {
+                direction: action.action_inputs.direction
+            },
+            thought: action.thought || ''
+        });
+        else if ('finished' === actionType) transformActions.push({
+            type: 'Finished',
+            param: {},
+            thought: action.action_inputs.content || action.thought || ''
+        });
+        else if ('hotkey' === actionType) if (action.action_inputs.key) {
+            const keys = (0, us_keyboard_layout_namespaceObject.transformHotkeyInput)(action.action_inputs.key);
+            transformActions.push({
+                type: 'KeyboardPress',
+                param: {
+                    keyName: keys.join('+')
+                },
+                thought: action.thought || ''
+            });
+        } else warnLog('No key found in action: hotkey. Will not perform action.');
+        else if ('wait' === actionType) transformActions.push({
+            type: 'Sleep',
+            param: {
+                timeMs: 1000
+            },
+            thought: action.thought || ''
+        });
+        else if (actionType) {
+            unhandledActions.push({
+                type: actionType,
+                thought: action.thought || ''
+            });
+            debug('Unhandled action type:', actionType, 'thought:', action.thought);
+        }
+    });
+    if (0 === transformActions.length) throw new Error(buildNoUiTarsActionsError(parsedPlanningResponse.rawResponse, parsedPlanningResponse.actions, unhandledActions));
+    debug('transformActions', JSON.stringify(transformActions, null, 2));
+    return transformActions;
+}
+function getPoint(startBox) {
+    const [x, y] = JSON.parse(startBox);
+    (0, utils_namespaceObject.assert)('number' == typeof x && Number.isFinite(x) && 'number' == typeof y && Number.isFinite(y), `invalid point data for ui-tars planning: ${startBox}`);
+    return [
+        x,
+        y
+    ];
+}
+function buildNoUiTarsActionsError(rawResponse, actions, unhandledActions) {
+    const errorDetails = [];
+    if (0 === actions.length) {
+        errorDetails.push('Action parser returned no actions');
+        if (rawResponse.includes('Thought:') && !rawResponse.includes('Action:')) errorDetails.push('Response contains "Thought:" but missing "Action:" line');
+        else errorDetails.push('Response may be malformed or empty');
+    }
+    if (unhandledActions.length > 0) {
+        const types = unhandledActions.map((a)=>a.type).join(', ');
+        errorDetails.push(`Unhandled action types: ${types}`);
+    }
+    return [
+        'No actions found in UI-TARS response.',
+        ...errorDetails
+    ].join('\n');
+}
+exports.transformUiTarsActions = __webpack_exports__.transformUiTarsActions;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "transformUiTarsActions"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=actions.js.map

package/dist/lib/ai-model/models/ui-tars/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAction } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { transformHotkeyInput } from '@aiscene/shared/us-keyboard-layout';\nimport { assert } from '@aiscene/shared/utils';\nimport type {\n  DragAndDropPlanningAction,\n  LocatePlanningAction,\n} from '../../model-adapter/custom-planning-action';\nimport type { UiTarsParsedPlanningResponse } from './parser';\n\nconst debug = getDebug('ui-tars-planning');\nconst warnLog = getDebug('ui-tars-planning', { console: true });\n\nexport function transformUiTarsActions(\n  parsedPlanningResponse: UiTarsParsedPlanningResponse,\n): PlanningAction[] {\n  const transformActions: PlanningAction[] = [];\n  const unhandledActions: Array<{ type: string; thought: string }> = [];\n\n  parsedPlanningResponse.actions.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate,\n        },\n      } satisfies LocatePlanningAction<'Tap'>);\n    } else if (actionType === 'left_double') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'DoubleClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'DoubleClick'>);\n    } else if (actionType === 'right_single') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box);\n\n      const locate = {\n        point,\n        prompt: action.thought || '',\n      };\n\n      transformActions.push({\n        type: 'RightClick',\n        param: {\n          locate,\n        },\n        thought: action.thought || '',\n      } satisfies LocatePlanningAction<'RightClick'>);\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box);\n      const endPoint = getPoint(action.action_inputs.end_box);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            point: startPoint,\n            prompt: action.thought || '',\n          },\n          to: {\n            point: endPoint,\n            prompt: action.thought || '',\n          },\n        },\n        thought: action.thought || '',\n      } satisfies DragAndDropPlanningAction);\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.action_inputs.content || action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        warnLog('No key found in action: hotkey. Will not perform action.');\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys.join('+'),\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType) {\n      unhandledActions.push({\n        type: actionType,\n        thought: action.thought || '',\n      });\n      debug('Unhandled action type:', actionType, 'thought:', action.thought);\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(\n      buildNoUiTarsActionsError(\n        parsedPlanningResponse.rawResponse,\n        parsedPlanningResponse.actions,\n        unhandledActions,\n      ),\n    );\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  return transformActions;\n}\n\nfunction getPoint(startBox: string): [number, number] {\n  const [x, y] = JSON.parse(startBox);\n  assert(\n    typeof x === 'number' &&\n      Number.isFinite(x) &&\n      typeof y === 'number' &&\n      Number.isFinite(y),\n    `invalid point data for ui-tars planning: ${startBox}`,\n  );\n  return [x, y];\n}\n\nfunction buildNoUiTarsActionsError(\n  rawResponse: string,\n  actions: UiTarsParsedPlanningResponse['actions'],\n  unhandledActions: Array<{ type: string; thought: string }>,\n): string {\n  const errorDetails: string[] = [];\n\n  if (actions.length === 0) {\n    errorDetails.push('Action parser returned no actions');\n\n    if (rawResponse.includes('Thought:') && !rawResponse.includes('Action:')) {\n      errorDetails.push(\n        'Response contains \"Thought:\" but missing \"Action:\" line',\n      );\n    } else {\n      errorDetails.push('Response may be malformed or empty');\n    }\n  }\n\n  if (unhandledActions.length > 0) {\n    const types = unhandledActions.map((a) => a.type).join(', ');\n    errorDetails.push(`Unhandled action types: ${types}`);\n  }\n\n  return ['No actions found in UI-TARS response.', ...errorDetails].join('\\n');\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","transformUiTarsActions","parsedPlanningResponse","transformActions","unhandledActions","action","actionType","assert","point","getPoint","locate","startPoint","endPoint","keys","transformHotkeyInput","Error","buildNoUiTarsActionsError","JSON","startBox","x","y","Number","rawResponse","actions","errorDetails","types","a"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACIA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,oBAAoB;IAAE,SAAS;AAAK;AAEtD,SAASE,uBACdC,sBAAoD;IAEpD,MAAMC,mBAAqC,EAAE;IAC7C,MAAMC,mBAA6D,EAAE;IAErEF,uBAAuB,OAAO,CAAC,OAAO,CAAC,CAACG;QACtC,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;YACF;QACF,OAAO,IAAIJ,AAAe,kBAAfA,YAA8B;YACvCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,mBAAfA,YAA+B;YACxCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMG,QAAQC,SAASJ,OAAO,aAAa,CAAC,SAAS;YAErD,MAAMK,SAAS;gBACbF;gBACA,QAAQH,OAAO,OAAO,IAAI;YAC5B;YAEAF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACLO;gBACF;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YAAuB;YAChCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMM,aAAaF,SAASJ,OAAO,aAAa,CAAC,SAAS;YAC1D,MAAMO,WAAWH,SAASJ,OAAO,aAAa,CAAC,OAAO;YACtDF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,OAAOQ;wBACP,QAAQN,OAAO,OAAO,IAAI;oBAC5B;oBACA,IAAI;wBACF,OAAOO;wBACP,QAAQP,OAAO,OAAO,IAAI;oBAC5B;gBACF;gBACA,SAASA,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOE,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWE,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASE,OAAO,aAAa,CAAC,OAAO,IAAIA,OAAO,OAAO,IAAI;QAC7D;aACK,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAEtB;YACL,MAAMQ,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBT,OAAO,aAAa,CAAC,GAAG;YAE1DF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASU,KAAK,IAAI,CAAC;gBACrB;gBACA,SAASR,OAAO,OAAO,IAAI;YAC7B;QACF,OAXEL,QAAQ;aAYL,IAAIM,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASE,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,YAAY;YACrBF,iBAAiB,IAAI,CAAC;gBACpB,MAAME;gBACN,SAASD,OAAO,OAAO,IAAI;YAC7B;YACAP,MAAM,0BAA0BQ,YAAY,YAAYD,OAAO,OAAO;QACxE;IACF;IAEA,IAAIF,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIY,MACRC,0BACEd,uBAAuB,WAAW,EAClCA,uBAAuB,OAAO,EAC9BE;IAKNN,MAAM,oBAAoBmB,KAAK,SAAS,CAACd,kBAAkB,MAAM;IACjE,OAAOA;AACT;AAEA,SAASM,SAASS,QAAgB;IAChC,MAAM,CAACC,GAAGC,EAAE,GAAGH,KAAK,KAAK,CAACC;IAC1BX,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAa,YAAb,OAAOY,KACLE,OAAO,QAAQ,CAACF,MAChB,AAAa,YAAb,OAAOC,KACPC,OAAO,QAAQ,CAACD,IAClB,CAAC,yCAAyC,EAAEF,UAAU;IAExD,OAAO;QAACC;QAAGC;KAAE;AACf;AAEA,SAASJ,0BACPM,WAAmB,EACnBC,OAAgD,EAChDnB,gBAA0D;IAE1D,MAAMoB,eAAyB,EAAE;IAEjC,IAAID,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;QACxBC,aAAa,IAAI,CAAC;QAElB,IAAIF,YAAY,QAAQ,CAAC,eAAe,CAACA,YAAY,QAAQ,CAAC,YAC5DE,aAAa,IAAI,CACf;aAGFA,aAAa,IAAI,CAAC;IAEtB;IAEA,IAAIpB,iBAAiB,MAAM,GAAG,GAAG;QAC/B,MAAMqB,QAAQrB,iBAAiB,GAAG,CAAC,CAACsB,IAAMA,EAAE,IAAI,EAAE,IAAI,CAAC;QACvDF,aAAa,IAAI,CAAC,CAAC,wBAAwB,EAAEC,OAAO;IACtD;IAEA,OAAO;QAAC;WAA4CD;KAAa,CAAC,IAAI,CAAC;AACzE"}

package/dist/lib/ai-model/models/ui-tars/adapter.js

@@ -0,0 +1,136 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    uiTarsAdapters: ()=>uiTarsAdapters
+});
+const env_namespaceObject = require("@aiscene/shared/env");
+const utils_namespaceObject = require("@aiscene/shared/utils");
+const index_js_namespaceObject = require("../../shared/model-locate-result/index.js");
+const external_planning_js_namespaceObject = require("./planning.js");
+const defaultVlmUiTarsReplanningCycleLimit = 40;
+const uiTarsBboxCoordinatesMeta = {
+    shape: 'bbox',
+    order: 'xy',
+    normalizedBy: 1000
+};
+const uiTarsPointCoordinatesMeta = {
+    shape: 'point',
+    order: 'xy',
+    normalizedBy: 1000
+};
+function parseUiTarsRawLocateValue(input) {
+    const bbox = (0, index_js_namespaceObject.unwrapCoordinateListLikeInput)(input);
+    if ('string' == typeof bbox) {
+        (0, utils_namespaceObject.assert)(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for ui-tars mode: ${bbox}`);
+        const splitted = bbox.split(' ');
+        if (4 === splitted.length) return (0, index_js_namespaceObject.createLocateResultValue)(uiTarsBboxCoordinatesMeta, [
+            Number(splitted[0]),
+            Number(splitted[1]),
+            Number(splitted[2]),
+            Number(splitted[3])
+        ]);
+        throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);
+    }
+    let bboxList = [];
+    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
+        if ('string' == typeof item && item.includes(',')) {
+            const [x, y] = item.split(',');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else if ('string' == typeof item && item.includes(' ')) {
+            const [x, y] = item.split(' ');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else bboxList.push(Number(item));
+    });
+    else bboxList = bbox;
+    if (4 === bboxList.length || 5 === bboxList.length) return (0, index_js_namespaceObject.createLocateResultValue)(uiTarsBboxCoordinatesMeta, [
+        bboxList[0],
+        bboxList[1],
+        bboxList[2],
+        bboxList[3]
+    ]);
+    if (6 === bboxList.length || 2 === bboxList.length || 3 === bboxList.length || 7 === bboxList.length) return (0, index_js_namespaceObject.createLocateResultValue)(uiTarsPointCoordinatesMeta, [
+        bboxList[0],
+        bboxList[1]
+    ]);
+    if (8 === bbox.length) return (0, index_js_namespaceObject.createLocateResultValue)(uiTarsBboxCoordinatesMeta, [
+        bboxList[0],
+        bboxList[1],
+        bboxList[4],
+        bboxList[5]
+    ]);
+    const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;
+    throw new Error(msg);
+}
+function createUiTarsAdapter(uiTarsModelVersion) {
+    return {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: ({ midsceneDefaults, userConfig })=>{
+                const commonOverrideConfig = {};
+                if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+                return {
+                    config: {
+                        ...midsceneDefaults,
+                        ...commonOverrideConfig
+                    }
+                };
+            }
+        },
+        planning: {
+            kind: 'custom',
+            cacheEnabled: false,
+            defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,
+            planner: (0, external_planning_js_namespaceObject.createUiTarsPlanner)(uiTarsModelVersion)
+        },
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: uiTarsBboxCoordinatesMeta,
+                    parseRawLocateValue: parseUiTarsRawLocateValue
+                }
+            }
+        }
+    };
+}
+const uiTarsDoubao15Adapter = createUiTarsAdapter(env_namespaceObject.UITarsModelVersion.DOUBAO_1_5_20B);
+const uiTarsAdapters = {
+    'vlm-ui-tars': createUiTarsAdapter(env_namespaceObject.UITarsModelVersion.V1_0),
+    'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,
+    'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter
+};
+exports.uiTarsAdapters = __webpack_exports__.uiTarsAdapters;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "uiTarsAdapters"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=adapter.js.map

package/dist/lib/ai-model/models/ui-tars/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/adapter.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/adapter.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TModelFamily, UITarsModelVersion } from '@aiscene/shared/env';\nimport { assert } from '@aiscene/shared/utils';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../../shared/model-locate-result';\nimport { createUiTarsPlanner } from './planning';\n\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\nconst uiTarsBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst uiTarsPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\n// UI-TARS has not received active updates for a long time, so this parser is\n// intentionally kept separate from Doubao even though the current logic is the\n// same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.\nfunction parseUiTarsRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for ui-tars mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n        Number(splitted[0]),\n        Number(splitted[1]),\n        Number(splitted[2]),\n        Number(splitted[3]),\n      ]);\n    }\n    throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(uiTarsPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bbox.length === 8) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction createUiTarsAdapter(\n  uiTarsModelVersion: UITarsModelVersion,\n): ModelAdapterDefinition {\n  return {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,\n      planner: createUiTarsPlanner(uiTarsModelVersion),\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: uiTarsBboxCoordinatesMeta,\n          parseRawLocateValue: parseUiTarsRawLocateValue,\n        },\n      },\n    },\n  };\n}\n\nconst uiTarsDoubao15Adapter = createUiTarsAdapter(\n  UITarsModelVersion.DOUBAO_1_5_20B,\n);\n\nexport const uiTarsAdapters = {\n  'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),\n  'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,\n  'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5'\n>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultVlmUiTarsReplanningCycleLimit","uiTarsBboxCoordinatesMeta","uiTarsPointCoordinatesMeta","parseUiTarsRawLocateValue","input","bbox","unwrapCoordinateListLikeInput","assert","splitted","createLocateResultValue","Number","Error","bboxList","Array","item","x","y","msg","JSON","createUiTarsAdapter","uiTarsModelVersion","midsceneDefaults","userConfig","commonOverrideConfig","undefined","createUiTarsPlanner","uiTarsDoubao15Adapter","UITarsModelVersion","uiTarsAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACIA,MAAMI,uCAAuC;AAC7C,MAAMC,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAKA,SAASC,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,AAAAA,IAAAA,yBAAAA,6BAAAA,AAAAA,EAA8BF;IAC3C,IAAI,AAAgB,YAAhB,OAAOC,MAAmB;QAC5BE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,2CAA2C,EAAEA,MAAM;QAEtD,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,AAAAA,IAAAA,yBAAAA,uBAAAA,AAAAA,EAAwBR,2BAA2B;YACxDS,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;SACnB;QAEH,MAAM,IAAIG,MAAM,CAAC,2CAA2C,EAAEN,MAAM;IACtE;IAEA,IAAIO,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACR,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACS;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACF,OAAOK,EAAE,IAAI,KAAKL,OAAOM,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACF,OAAOK,EAAE,IAAI,KAAKL,OAAOM,EAAE,IAAI;QAC/C,OACEJ,SAAS,IAAI,CAACF,OAAOI;IAEzB;SAEAF,WAAWP;IAGb,IAAIO,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOH,AAAAA,IAAAA,yBAAAA,uBAAAA,AAAAA,EAAwBR,2BAA2B;QACxDW,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOH,AAAAA,IAAAA,yBAAAA,uBAAAA,AAAAA,EAAwBP,4BAA4B;QACzDU,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIP,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAOI,AAAAA,IAAAA,yBAAAA,uBAAAA,AAAAA,EAAwBR,2BAA2B;QACxDW,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAMK,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACb,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAIM,MAAMM;AAClB;AAEA,SAASE,oBACPC,kBAAsC;IAEtC,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEC,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,OAAO;oBACL,QAAQ;wBACN,GAAGD,gBAAgB;wBACnB,GAAGE,oBAAoB;oBACzB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BvB;YAC7B,SAASyB,AAAAA,IAAAA,qCAAAA,mBAAAA,AAAAA,EAAoBL;QAC/B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAanB;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;AACF;AAEA,MAAMuB,wBAAwBP,oBAC5BQ,oBAAAA,kBAAAA,CAAAA,cAAiC;AAG5B,MAAMC,iBAAiB;IAC5B,eAAeT,oBAAoBQ,oBAAAA,kBAAAA,CAAAA,IAAuB;IAC1D,sBAAsBD;IACtB,0BAA0BA;AAC5B"}

package/dist/lib/ai-model/models/ui-tars/parser.js

@@ -0,0 +1,74 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    parseUiTarsPlanningResponse: ()=>parseUiTarsPlanningResponse
+});
+const logger_namespaceObject = require("@aiscene/shared/logger");
+const action_parser_namespaceObject = require("@ui-tars/action-parser");
+const debug = (0, logger_namespaceObject.getDebug)('ui-tars-planning');
+function parseUiTarsPlanningResponse(rawResponse, shotSize, uiTarsModelVersion) {
+    const convertedText = convertBboxToCoordinates(rawResponse);
+    const parseResult = (0, action_parser_namespaceObject.actionParser)({
+        prediction: convertedText,
+        factor: [
+            1000,
+            1000
+        ],
+        screenContext: {
+            width: shotSize.width,
+            height: shotSize.height
+        },
+        modelVer: uiTarsModelVersion
+    });
+    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parseResult.parsed));
+    return {
+        rawResponse,
+        actions: parseResult.parsed
+    };
+}
+function convertBboxToCoordinates(text) {
+    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
+    function replaceMatch(match, x1, y1, x2, y2) {
+        const x1Num = Number.parseInt(x1, 10);
+        const y1Num = Number.parseInt(y1, 10);
+        const x2Num = Number.parseInt(x2, 10);
+        const y2Num = Number.parseInt(y2, 10);
+        const x = Math.floor((x1Num + x2Num) / 2);
+        const y = Math.floor((y1Num + y2Num) / 2);
+        return `(${x},${y})`;
+    }
+    const cleanedText = text.replace(/\[EOS\]/g, '').replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');
+    return cleanedText.replace(pattern, replaceMatch).trim();
+}
+exports.parseUiTarsPlanningResponse = __webpack_exports__.parseUiTarsPlanningResponse;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "parseUiTarsPlanningResponse"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=parser.js.map

package/dist/lib/ai-model/models/ui-tars/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/parser.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/parser.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Size } from '@/types';\nimport type { UITarsModelVersion } from '@aiscene/shared/env';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { actionParser } from '@ui-tars/action-parser';\n\ntype ActionType =\n  | 'click'\n  | 'left_double'\n  | 'right_single'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\n\nexport interface UiTarsParsedPlanningResponse {\n  rawResponse: string;\n  actions: ReturnType<typeof actionParser>['parsed'];\n}\n\nexport function parseUiTarsPlanningResponse(\n  rawResponse: string,\n  shotSize: Size,\n  uiTarsModelVersion: UITarsModelVersion,\n): UiTarsParsedPlanningResponse {\n  const convertedText = convertBboxToCoordinates(rawResponse);\n  const parseResult = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: shotSize.width,\n      height: shotSize.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parseResult.parsed),\n  );\n\n  return {\n    rawResponse,\n    actions: parseResult.parsed,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points.\n */\nfunction convertBboxToCoordinates(text: string): string {\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    return `(${x},${y})`;\n  }\n\n  const cleanedText = text\n    .replace(/\\[EOS\\]/g, '')\n    .replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string;\n    end_box: string;\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string;\n  };\n}\n\ninterface LeftDoubleAction extends BaseAction {\n  action_type: 'left_double';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface RightSingleAction extends BaseAction {\n  action_type: 'right_single';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: {\n    content?: string;\n  };\n}\n\nexport type Action =\n  | ClickAction\n  | LeftDoubleAction\n  | RightSingleAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","parseUiTarsPlanningResponse","rawResponse","shotSize","uiTarsModelVersion","convertedText","convertBboxToCoordinates","parseResult","actionParser","JSON","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","Math","y","cleanedText"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACUA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAOhB,SAASC,4BACdC,WAAmB,EACnBC,QAAc,EACdC,kBAAsC;IAEtC,MAAMC,gBAAgBC,yBAAyBJ;IAC/C,MAAMK,cAAcC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC/B,YAAYH;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOF,SAAS,KAAK;YACrB,QAAQA,SAAS,MAAM;QACzB;QACA,UAAUC;IACZ;IAEAL,MACE,oBACAK,oBACA,YACAK,KAAK,SAAS,CAACF,YAAY,MAAM;IAGnC,OAAO;QACLL;QACA,SAASK,YAAY,MAAM;IAC7B;AACF;AAKA,SAASD,yBAAyBI,IAAY;IAC5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAEV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAElC,MAAMM,IAAIC,KAAK,KAAK,CAAEN,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMI,IAAID,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAEvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEE,EAAE,CAAC,CAAC;IACtB;IAEA,MAAMC,cAAchB,KACjB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,2BAA2B;IACtC,OAAOgB,YAAY,OAAO,CAACf,SAASC,cAAc,IAAI;AACxD"}

package/dist/lib/ai-model/models/ui-tars/planning.js

@@ -0,0 +1,59 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    createUiTarsPlanner: ()=>createUiTarsPlanner
+});
+const external_actions_js_namespaceObject = require("./actions.js");
+const external_parser_js_namespaceObject = require("./parser.js");
+const external_prompt_js_namespaceObject = require("./prompt.js");
+function createUiTarsPlanner(uiTarsModelVersion) {
+    return {
+        messages: {
+            systemPromptPlacement: 'user-message',
+            buildSystemPrompt: external_prompt_js_namespaceObject.getUiTarsPlanningPrompt,
+            buildUserInstruction: (instruction)=>`<user_instruction>${instruction}</user_instruction>`,
+            buildAssistantContent: (_parsedResponse, rawResponse)=>(0, external_prompt_js_namespaceObject.getSummary)(rawResponse)
+        },
+        coordinates: {
+            shape: 'point',
+            order: 'xy',
+            normalizedBy: 1
+        },
+        parseResponse: (rawResponse, { options })=>(0, external_parser_js_namespaceObject.parseUiTarsPlanningResponse)(rawResponse, options.context.shotSize, uiTarsModelVersion),
+        transformActions: (parsedPlanningResponse)=>(0, external_actions_js_namespaceObject.transformUiTarsActions)(parsedPlanningResponse),
+        shouldContinuePlanning: (_parsedResponse, actions)=>actions.every((action)=>'Finished' !== action.type),
+        buildResponseLog: (_parsedResponse, rawResponse)=>(0, external_prompt_js_namespaceObject.getSummary)(rawResponse)
+    };
+}
+exports.createUiTarsPlanner = __webpack_exports__.createUiTarsPlanner;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "createUiTarsPlanner"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=planning.js.map

package/dist/lib/ai-model/models/ui-tars/planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { UITarsModelVersion } from '@aiscene/shared/env';\nimport type { CustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport { transformUiTarsActions } from './actions';\nimport {\n  type UiTarsParsedPlanningResponse,\n  parseUiTarsPlanningResponse,\n} from './parser';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt';\n\nexport function createUiTarsPlanner(\n  uiTarsModelVersion: UITarsModelVersion,\n): CustomPlanningDefinition<UiTarsParsedPlanningResponse> {\n  return {\n    messages: {\n      systemPromptPlacement: 'user-message',\n      buildSystemPrompt: getUiTarsPlanningPrompt,\n      buildUserInstruction: (instruction) =>\n        `<user_instruction>${instruction}</user_instruction>`,\n      buildAssistantContent: (_parsedResponse, rawResponse) =>\n        getSummary(rawResponse),\n    },\n    coordinates: { shape: 'point', order: 'xy', normalizedBy: 1 },\n    parseResponse: (rawResponse, { options }) => {\n      return parseUiTarsPlanningResponse(\n        rawResponse,\n        options.context.shotSize,\n        uiTarsModelVersion,\n      );\n    },\n    transformActions: (parsedPlanningResponse) => {\n      return transformUiTarsActions(parsedPlanningResponse);\n    },\n    shouldContinuePlanning: (_parsedResponse, actions) =>\n      actions.every((action) => action.type !== 'Finished'),\n    buildResponseLog: (_parsedResponse, rawResponse) => getSummary(rawResponse),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createUiTarsPlanner","uiTarsModelVersion","getUiTarsPlanningPrompt","instruction","_parsedResponse","rawResponse","getSummary","options","parseUiTarsPlanningResponse","parsedPlanningResponse","transformUiTarsActions","actions","action"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACGO,SAASI,oBACdC,kBAAsC;IAEtC,OAAO;QACL,UAAU;YACR,uBAAuB;YACvB,mBAAmBC,mCAAAA,uBAAuBA;YAC1C,sBAAsB,CAACC,cACrB,CAAC,kBAAkB,EAAEA,YAAY,mBAAmB,CAAC;YACvD,uBAAuB,CAACC,iBAAiBC,cACvCC,AAAAA,IAAAA,mCAAAA,UAAAA,AAAAA,EAAWD;QACf;QACA,aAAa;YAAE,OAAO;YAAS,OAAO;YAAM,cAAc;QAAE;QAC5D,eAAe,CAACA,aAAa,EAAEE,OAAO,EAAE,GAC/BC,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EACLH,aACAE,QAAQ,OAAO,CAAC,QAAQ,EACxBN;QAGJ,kBAAkB,CAACQ,yBACVC,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EAAuBD;QAEhC,wBAAwB,CAACL,iBAAiBO,UACxCA,QAAQ,KAAK,CAAC,CAACC,SAAWA,AAAgB,eAAhBA,OAAO,IAAI;QACvC,kBAAkB,CAACR,iBAAiBC,cAAgBC,AAAAA,IAAAA,mCAAAA,UAAAA,AAAAA,EAAWD;IACjE;AACF"}

package/dist/lib/ai-model/models/ui-tars/prompt.js

@@ -0,0 +1,73 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    getUiTarsPlanningPrompt: ()=>getUiTarsPlanningPrompt,
+    getSummary: ()=>getSummary
+});
+const env_namespaceObject = require("@aiscene/shared/env");
+function getUiTarsPlanningPrompt() {
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    return `
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(start_box='[x1, y1, x2, y2]')
+left_double(start_box='[x1, y1, x2, y2]')
+right_single(start_box='[x1, y1, x2, y2]')
+drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')
+hotkey(key='')
+type(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content. 
+scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
+
+
+## Note
+- Use ${preferredLanguage} in \`Thought\` part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in \`Thought\` part.
+
+## User Instruction
+`;
+}
+const getSummary = (prediction)=>prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, '').trim();
+exports.getSummary = __webpack_exports__.getSummary;
+exports.getUiTarsPlanningPrompt = __webpack_exports__.getUiTarsPlanningPrompt;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "getSummary",
+    "getUiTarsPlanningPrompt"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=prompt.js.map

package/dist/lib/ai-model/models/ui-tars/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/prompt.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/ui-tars/prompt.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@aiscene/shared/env';\n\nexport function getUiTarsPlanningPrompt(): string {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n\\`\\`\\`\nThought: ...\nAction: ...\n\\`\\`\\`\n\n## Action Space\n\nclick(start_box='[x1, y1, x2, y2]')\nleft_double(start_box='[x1, y1, x2, y2]')\nright_single(start_box='[x1, y1, x2, y2]')\ndrag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')\nhotkey(key='')\ntype(content='xxx') # Use escape characters \\\\', \\\\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\\\n at the end of content. \nscroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\\\', \\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use ${preferredLanguage} in \\`Thought\\` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in \\`Thought\\` part.\n\n## User Instruction\n`;\n}\n\nexport const getSummary = (prediction: string) =>\n  prediction\n    .replace(/Reflection:[\\s\\S]*?(?=Action_Summary:|Action:|$)/g, '')\n    .trim();\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getUiTarsPlanningPrompt","preferredLanguage","getPreferredLanguage","getSummary","prediction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACJO,SAASI;IACd,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;MAuBJ,EAAED,kBAAkB;;;;AAI1B,CAAC;AACD;AAEO,MAAME,aAAa,CAACC,aACzBA,WACG,OAAO,CAAC,qDAAqD,IAC7D,IAAI"}

package/dist/lib/ai-model/models/utils/intent.js

@@ -0,0 +1,40 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    isLocateIntent: ()=>isLocateIntent
+});
+function isLocateIntent(intent) {
+    return 'default' === intent;
+}
+exports.isLocateIntent = __webpack_exports__.isLocateIntent;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "isLocateIntent"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=intent.js.map