@midscene/core 1.2.1-beta-20260112114129.0 → 1.2.1-beta-20260114072539.0

package/dist/lib/ai-model/auto-glm/actions.js

@@ -0,0 +1,251 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    transformAutoGLMAction: ()=>transformAutoGLMAction
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const external_latest_locate_recorder_js_namespaceObject = require("../latest-locate-recorder.js");
+const external_util_js_namespaceObject = require("./util.js");
+const debug = (0, logger_namespaceObject.getDebug)('auto-glm-actions');
+const lastLocateRecorder = new external_latest_locate_recorder_js_namespaceObject.LatestLocateRecorder();
+function transformAutoGLMAction(action, size) {
+    try {
+        switch(action._metadata){
+            case 'finish':
+                {
+                    const finishAction = action;
+                    debug('Transform finish action:', finishAction);
+                    return [
+                        {
+                            type: 'Finished',
+                            param: {},
+                            thought: finishAction.message
+                        }
+                    ];
+                }
+            case 'do':
+                {
+                    const doAction = action;
+                    switch(doAction.action){
+                        case 'Tap':
+                            {
+                                const tapAction = doAction;
+                                debug('Transform Tap action:', tapAction);
+                                const [x1, y1, x2, y2] = (0, external_util_js_namespaceObject.autoGLMCoordinateToBbox)(tapAction.element[0], tapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                lastLocateRecorder.recordLocate(locate, 'Tap');
+                                return [
+                                    {
+                                        type: 'Tap',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Double Tap':
+                            {
+                                const doubleTapAction = doAction;
+                                debug('Transform Double Tap action:', doubleTapAction);
+                                const [x1, y1, x2, y2] = (0, external_util_js_namespaceObject.autoGLMCoordinateToBbox)(doubleTapAction.element[0], doubleTapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                lastLocateRecorder.recordLocate(locate, 'Double Tap');
+                                return [
+                                    {
+                                        type: 'DoubleClick',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Type':
+                            {
+                                const typeAction = doAction;
+                                debug('Transform Type action:', typeAction);
+                                const { locate: latestLocate, source } = lastLocateRecorder.getLatestLocate();
+                                debug(`use latestLocate from ${source} as locate when Input`, latestLocate);
+                                return [
+                                    {
+                                        type: 'Input',
+                                        param: {
+                                            value: typeAction.text,
+                                            locate: latestLocate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Swipe':
+                            {
+                                const swipeAction = doAction;
+                                debug('Transform Swipe action:', swipeAction);
+                                const [x1, y1, x2, y2] = (0, external_util_js_namespaceObject.autoGLMCoordinateToBbox)(swipeAction.start[0], swipeAction.start[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                const deltaX = swipeAction.end[0] - swipeAction.start[0];
+                                const deltaY = swipeAction.end[1] - swipeAction.start[1];
+                                let direction;
+                                let distance;
+                                const absDeltaX = Math.abs(deltaX);
+                                const absDeltaY = Math.abs(deltaY);
+                                if (absDeltaY > absDeltaX) {
+                                    distance = Math.round(absDeltaY * size.height / external_util_js_namespaceObject.AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaY > 0 ? 'up' : 'down';
+                                } else {
+                                    distance = Math.round(absDeltaX * size.width / external_util_js_namespaceObject.AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaX > 0 ? 'left' : 'right';
+                                }
+                                debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
+                                return [
+                                    {
+                                        type: 'Scroll',
+                                        param: {
+                                            locate,
+                                            distance,
+                                            direction
+                                        },
+                                        thought: swipeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Long Press':
+                            {
+                                const longPressAction = doAction;
+                                debug('Transform Long Press action:', longPressAction);
+                                const [x1, y1, x2, y2] = (0, external_util_js_namespaceObject.autoGLMCoordinateToBbox)(longPressAction.element[0], longPressAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                lastLocateRecorder.recordLocate(locate, 'Long Press');
+                                return [
+                                    {
+                                        type: 'AndroidLongPress',
+                                        param: {
+                                            locate
+                                        },
+                                        thought: longPressAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Back':
+                            {
+                                const backAction = doAction;
+                                debug('Transform Back action:', backAction);
+                                return [
+                                    {
+                                        type: 'AndroidBackButton',
+                                        param: {},
+                                        thought: backAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Home':
+                            {
+                                const homeAction = doAction;
+                                debug('Transform Home action:', homeAction);
+                                return [
+                                    {
+                                        type: 'AndroidHomeButton',
+                                        param: {},
+                                        thought: homeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Wait':
+                            {
+                                const waitAction = doAction;
+                                debug('Transform Wait action:', waitAction);
+                                return [
+                                    {
+                                        type: 'Sleep',
+                                        param: {
+                                            timeMs: waitAction.durationMs
+                                        },
+                                        thought: waitAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Launch':
+                            throw new Error('Action "Launch" from auto-glm is not supported in the current implementation.');
+                        case 'Interact':
+                            throw new Error('Action "Interact" from auto-glm is not supported in the current implementation.');
+                        case 'Call_API':
+                            throw new Error('Action "Call_API" from auto-glm is not supported in the current implementation.');
+                        case 'Take_over':
+                            throw new Error('Action "Take_over" from auto-glm is not supported in the current implementation.');
+                        case 'Note':
+                            throw new Error('Action "Note" from auto-glm is not supported in the current implementation.');
+                        default:
+                            throw new Error(`Unknown do() action type: ${doAction.action}`);
+                    }
+                }
+            default:
+                throw new Error(`Unknown action metadata: ${action._metadata}`);
+        }
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        debug('Transform error:', errorMessage);
+        throw new Error(`Failed to transform action: ${errorMessage}`);
+    }
+}
+exports.transformAutoGLMAction = __webpack_exports__.transformAutoGLMAction;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "transformAutoGLMAction"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=actions.js.map

package/dist/lib/ai-model/auto-glm/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { LatestLocateRecorder } from '../latest-locate-recorder';\nimport { AUTO_GLM_COORDINATE_MAX, autoGLMCoordinateToBbox } from './util';\n\nconst debug = getDebug('auto-glm-actions');\nconst lastLocateRecorder = new LatestLocateRecorder();\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n            lastLocateRecorder.recordLocate(locate, 'Tap');\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n            lastLocateRecorder.recordLocate(locate, 'Double Tap');\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n            const { locate: latestLocate, source } =\n              lastLocateRecorder.getLatestLocate();\n            debug(\n              `use latestLocate from ${source} as locate when Input`,\n              latestLocate,\n            );\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                  locate: latestLocate,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n            lastLocateRecorder.recordLocate(locate, 'Long Press');\n\n            return [\n              {\n                type: 'AndroidLongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: 'AndroidBackButton',\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: 'AndroidHomeButton',\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            throw new Error(\n              `Action \"Launch\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","lastLocateRecorder","LatestLocateRecorder","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","autoGLMCoordinateToBbox","locate","doubleTapAction","typeAction","latestLocate","source","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","AUTO_GLM_COORDINATE_MAX","longPressAction","backAction","homeAction","waitAction","Error","error","errorMessage","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,qBAAqB,IAAIC,mDAAAA,oBAAoBA;AAwG5C,SAASC,uBACdC,MAAoB,EACpBC,IAAuC;IAEvC,IAAI;QACF,OAAQD,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBL,MAAM,4BAA4BO;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBR,MAAM,yBAAyBS;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,AAAAA,IAAAA,iCAAAA,uBAAAA,AAAAA,EACvBL,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCACAX,mBAAmB,YAAY,CAACa,QAAQ;gCAExC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLA;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBR;gCACxBR,MAAM,gCAAgCgB;gCACtC,MAAM,CAACN,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,AAAAA,IAAAA,iCAAAA,uBAAAA,AAAAA,EACvBE,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BV,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCACAX,mBAAmB,YAAY,CAACa,QAAQ;gCAExC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLA;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaT;gCACnBR,MAAM,0BAA0BiB;gCAChC,MAAM,EAAE,QAAQC,YAAY,EAAEC,MAAM,EAAE,GACpCjB,mBAAmB,eAAe;gCACpCF,MACE,CAAC,sBAAsB,EAAEmB,OAAO,qBAAqB,CAAC,EACtDD;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOD,WAAW,IAAI;4CACtB,QAAQC;wCACV;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAME,cAAcZ;gCACpBR,MAAM,2BAA2BoB;gCAGjC,MAAM,CAACV,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,AAAAA,IAAAA,iCAAAA,uBAAAA,AAAAA,EACvBM,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBd,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMQ,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYrB,KAAK,MAAM,GAAIsB,iCAAAA,uBAAuBA;oCAErDL,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYnB,KAAK,KAAK,GAAIsB,iCAAAA,uBAAuBA;oCAEpDL,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEArB,MACE,CAAC,2BAA2B,EAAEuB,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLT;4CAEAS;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMS,kBAAkBrB;gCACxBR,MAAM,gCAAgC6B;gCACtC,MAAM,CAACnB,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,AAAAA,IAAAA,iCAAAA,uBAAAA,AAAAA,EACvBe,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BvB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCACAX,mBAAmB,YAAY,CAACa,QAAQ;gCAExC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLA;wCACF;wCACA,SAASc,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAatB;gCACnBR,MAAM,0BAA0B8B;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO,CAAC;wCACR,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAavB;gCACnBR,MAAM,0BAA0B+B;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO,CAAC;wCACR,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaxB;gCACnBR,MAAM,0BAA0BgC;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGzB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIyB,MACR,CAAC,yBAAyB,EAAG5B,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO6B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrElC,MAAM,oBAAoBmC;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/lib/ai-model/auto-glm/index.js

@@ -0,0 +1,59 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    getAutoGLMLocatePrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMLocatePrompt,
+    autoGLMCoordinateToBbox: ()=>external_util_js_namespaceObject.autoGLMCoordinateToBbox,
+    parseAction: ()=>external_parser_js_namespaceObject.parseAction,
+    parseAutoGLMLocateResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMLocateResponse,
+    parseAutoGLMResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMResponse,
+    autoGLMPlanning: ()=>external_planning_js_namespaceObject.autoGLMPlanning,
+    getAutoGLMPlanPrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMPlanPrompt
+});
+const external_prompt_js_namespaceObject = require("./prompt.js");
+const external_parser_js_namespaceObject = require("./parser.js");
+const external_planning_js_namespaceObject = require("./planning.js");
+const external_util_js_namespaceObject = require("./util.js");
+exports.autoGLMCoordinateToBbox = __webpack_exports__.autoGLMCoordinateToBbox;
+exports.autoGLMPlanning = __webpack_exports__.autoGLMPlanning;
+exports.getAutoGLMLocatePrompt = __webpack_exports__.getAutoGLMLocatePrompt;
+exports.getAutoGLMPlanPrompt = __webpack_exports__.getAutoGLMPlanPrompt;
+exports.parseAction = __webpack_exports__.parseAction;
+exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
+exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "autoGLMCoordinateToBbox",
+    "autoGLMPlanning",
+    "getAutoGLMLocatePrompt",
+    "getAutoGLMPlanPrompt",
+    "parseAction",
+    "parseAutoGLMLocateResponse",
+    "parseAutoGLMResponse"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=index.js.map

package/dist/lib/ai-model/auto-glm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/ai-model/auto-glm/parser.js

@@ -0,0 +1,282 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    extractValueAfter: ()=>extractValueAfter,
+    parseAction: ()=>parseAction,
+    parseAutoGLMLocateResponse: ()=>parseAutoGLMLocateResponse,
+    parseAutoGLMResponse: ()=>parseAutoGLMResponse
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const debug = (0, logger_namespaceObject.getDebug)('auto-glm-parser');
+const extractValueAfter = (src, key)=>{
+    const idx = src.indexOf(key);
+    if (-1 === idx) throw new Error(`Missing key ${key} in action payload ${src}`);
+    let rest = src.slice(idx + key.length).trim();
+    if (rest.endsWith('")')) rest = rest.slice(0, -2);
+    return rest;
+};
+function parseAction(response) {
+    debug('Parsing action:', response);
+    let trimmedResponse = '';
+    try {
+        trimmedResponse = response.content.trim();
+        if (trimmedResponse.startsWith('do(action="Type"') || trimmedResponse.startsWith('do(action="Type_Name"')) {
+            const text = extractValueAfter(trimmedResponse, 'text="');
+            return {
+                _metadata: 'do',
+                action: 'Type',
+                text,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('finish(message=')) {
+            let message = extractValueAfter(trimmedResponse, 'finish(message="');
+            if (message.endsWith(')')) message = message.slice(0, -1);
+            return {
+                _metadata: 'finish',
+                message,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('do(')) {
+            const actionMatch = trimmedResponse.match(/do\(action="([^"]+)"/);
+            if (!actionMatch) throw new Error(`Failed to extract action type from do() call; raw="${trimmedResponse}"`);
+            const actionType = actionMatch[1];
+            const baseAction = {
+                _metadata: 'do',
+                think: response.think
+            };
+            switch(actionType){
+                case 'Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Double Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Double Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Double Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Swipe':
+                    {
+                        const startMatch = trimmedResponse.match(/start=\[(\d+),(\d+)\]/);
+                        const endMatch = trimmedResponse.match(/end=\[(\d+),(\d+)\]/);
+                        if (!startMatch || !endMatch) throw new Error(`Failed to extract start/end coordinates for Swipe; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Swipe',
+                            start: [
+                                Number(startMatch[1]),
+                                Number(startMatch[2])
+                            ],
+                            end: [
+                                Number(endMatch[1]),
+                                Number(endMatch[2])
+                            ]
+                        };
+                    }
+                case 'Long Press':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Long Press; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Long Press',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Launch':
+                    {
+                        const app = extractValueAfter(trimmedResponse, 'app="');
+                        return {
+                            ...baseAction,
+                            action: 'Launch',
+                            app
+                        };
+                    }
+                case 'Back':
+                    return {
+                        ...baseAction,
+                        action: 'Back'
+                    };
+                case 'Home':
+                    return {
+                        ...baseAction,
+                        action: 'Home'
+                    };
+                case 'Wait':
+                    {
+                        const durationMatch = trimmedResponse.match(/duration=(?:["\[])?(\d+)/);
+                        if (!durationMatch) throw new Error(`Failed to extract duration for Wait; raw="${trimmedResponse}"`);
+                        const seconds = Number.parseInt(durationMatch[1], 10);
+                        const durationMs = 1000 * seconds;
+                        return {
+                            ...baseAction,
+                            action: 'Wait',
+                            durationMs
+                        };
+                    }
+                case 'Interact':
+                    return {
+                        ...baseAction,
+                        action: 'Interact'
+                    };
+                case 'Call_API':
+                    {
+                        const instruction = extractValueAfter(trimmedResponse, 'instruction="');
+                        return {
+                            ...baseAction,
+                            action: 'Call_API',
+                            instruction
+                        };
+                    }
+                case 'Take_over':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Take_over',
+                            message
+                        };
+                    }
+                case 'Note':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Note',
+                            message
+                        };
+                    }
+                default:
+                    throw new Error(`Unknown action type: ${actionType}; raw="${trimmedResponse}"`);
+            }
+        }
+        throw new Error(`Failed to parse action: ${trimmedResponse}`);
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to parse action: ${errorMessage}; raw="${trimmedResponse}"`);
+    }
+}
+function parseAutoGLMResponse(content) {
+    if (content.includes('finish(message=')) {
+        const parts = content.split('finish(message=');
+        const think = parts[0].trim();
+        const actionContent = `finish(message=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('do(action=')) {
+        const parts = content.split('do(action=');
+        const think = parts[0].trim();
+        const actionContent = `do(action=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('<answer>')) {
+        const parts = content.split('<answer>');
+        const think = parts[0].replace(/<think>/g, '').replace(/<\/think>/g, '').trim();
+        const actionContent = parts[1].replace(/<\/answer>/g, '').trim();
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    return {
+        think: '',
+        content
+    };
+}
+function parseAutoGLMLocateResponse(rawResponse) {
+    const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);
+    if (!actionContent.startsWith('do(action="Tap"')) return {
+        think,
+        coordinates: null,
+        error: `Unexpected action type in auto-glm locate response: ${actionContent}`
+    };
+    try {
+        const elementMatch = actionContent.match(/element=\[(\d+),(\d+)\]/);
+        if (!elementMatch) return {
+            think,
+            coordinates: null,
+            error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`
+        };
+        const x = Number(elementMatch[1]);
+        const y = Number(elementMatch[2]);
+        return {
+            think,
+            coordinates: {
+                x,
+                y
+            }
+        };
+    } catch (e) {
+        const errorMessage = e instanceof Error ? e.message : String(e);
+        return {
+            think,
+            coordinates: null,
+            error: `Failed to parse coordinates "${actionContent}" with errorMessage: ${errorMessage}`
+        };
+    }
+}
+exports.extractValueAfter = __webpack_exports__.extractValueAfter;
+exports.parseAction = __webpack_exports__.parseAction;
+exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
+exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "extractValueAfter",
+    "parseAction",
+    "parseAutoGLMLocateResponse",
+    "parseAutoGLMResponse"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=parser.js.map