@midscene/core 1.2.2-beta-20260116064919.0 → 1.2.2-beta-20260116071350.0

package/dist/lib/agent/utils.js

@@ -150,7 +150,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.2.2-beta-20260116064919.0";
+const getMidsceneVersion = ()=>"1.2.2-beta-20260116071350.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/auto-glm/actions.js

@@ -0,0 +1,258 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    transformAutoGLMAction: ()=>transformAutoGLMAction
+});
+const external_common_js_namespaceObject = require("../../common.js");
+const logger_namespaceObject = require("@midscene/shared/logger");
+const debug = (0, logger_namespaceObject.getDebug)('auto-glm-actions');
+const AUTO_GLM_COORDINATE_MAX = 1000;
+function autoGLMCoordinateToBbox(x, y, width, height) {
+    const bbox = (0, external_common_js_namespaceObject.pointToBbox)(x, y, 10);
+    return (0, external_common_js_namespaceObject.adaptBbox)(bbox, width, height, width, height, 'auto-glm');
+}
+function transformAutoGLMAction(action, size) {
+    try {
+        switch(action._metadata){
+            case 'finish':
+                {
+                    const finishAction = action;
+                    debug('Transform finish action:', finishAction);
+                    return [
+                        {
+                            type: 'Finished',
+                            param: {},
+                            thought: finishAction.message
+                        }
+                    ];
+                }
+            case 'do':
+                {
+                    const doAction = action;
+                    switch(doAction.action){
+                        case 'Tap':
+                            {
+                                const tapAction = doAction;
+                                debug('Transform Tap action:', tapAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(tapAction.element[0], tapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                return [
+                                    {
+                                        type: 'Tap',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Double Tap':
+                            {
+                                const doubleTapAction = doAction;
+                                debug('Transform Double Tap action:', doubleTapAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(doubleTapAction.element[0], doubleTapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                return [
+                                    {
+                                        type: 'DoubleClick',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Type':
+                            {
+                                const typeAction = doAction;
+                                debug('Transform Type action:', typeAction);
+                                return [
+                                    {
+                                        type: 'Input',
+                                        param: {
+                                            value: typeAction.text
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Swipe':
+                            {
+                                const swipeAction = doAction;
+                                debug('Transform Swipe action:', swipeAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(swipeAction.start[0], swipeAction.start[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                const deltaX = swipeAction.end[0] - swipeAction.start[0];
+                                const deltaY = swipeAction.end[1] - swipeAction.start[1];
+                                let direction;
+                                let distance;
+                                const absDeltaX = Math.abs(deltaX);
+                                const absDeltaY = Math.abs(deltaY);
+                                if (absDeltaY > absDeltaX) {
+                                    distance = Math.round(absDeltaY * size.height / AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaY > 0 ? 'up' : 'down';
+                                } else {
+                                    distance = Math.round(absDeltaX * size.width / AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaX > 0 ? 'left' : 'right';
+                                }
+                                debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
+                                return [
+                                    {
+                                        type: 'Scroll',
+                                        param: {
+                                            locate,
+                                            distance,
+                                            direction
+                                        },
+                                        thought: swipeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Long Press':
+                            {
+                                const longPressAction = doAction;
+                                debug('Transform Long Press action:', longPressAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(longPressAction.element[0], longPressAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                return [
+                                    {
+                                        type: 'AndroidLongPress',
+                                        param: {
+                                            locate
+                                        },
+                                        thought: longPressAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Back':
+                            {
+                                const backAction = doAction;
+                                debug('Transform Back action:', backAction);
+                                return [
+                                    {
+                                        type: 'AndroidBackButton',
+                                        param: {},
+                                        thought: backAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Home':
+                            {
+                                const homeAction = doAction;
+                                debug('Transform Home action:', homeAction);
+                                return [
+                                    {
+                                        type: 'AndroidHomeButton',
+                                        param: {},
+                                        thought: homeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Wait':
+                            {
+                                const waitAction = doAction;
+                                debug('Transform Wait action:', waitAction);
+                                return [
+                                    {
+                                        type: 'Sleep',
+                                        param: {
+                                            timeMs: waitAction.durationMs
+                                        },
+                                        thought: waitAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Launch':
+                            {
+                                const launchAction = doAction;
+                                debug('Transform Launch action:', launchAction);
+                                return [
+                                    {
+                                        type: 'Launch',
+                                        param: launchAction.app,
+                                        thought: launchAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Interact':
+                            throw new Error('Action "Interact" from auto-glm is not supported in the current implementation.');
+                        case 'Call_API':
+                            throw new Error('Action "Call_API" from auto-glm is not supported in the current implementation.');
+                        case 'Take_over':
+                            throw new Error('Action "Take_over" from auto-glm is not supported in the current implementation.');
+                        case 'Note':
+                            throw new Error('Action "Note" from auto-glm is not supported in the current implementation.');
+                        default:
+                            throw new Error(`Unknown do() action type: ${doAction.action}`);
+                    }
+                }
+            default:
+                throw new Error(`Unknown action metadata: ${action._metadata}`);
+        }
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        debug('Transform error:', errorMessage);
+        throw new Error(`Failed to transform action: ${errorMessage}`);
+    }
+}
+exports.transformAutoGLMAction = __webpack_exports__.transformAutoGLMAction;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "transformAutoGLMAction"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=actions.js.map

package/dist/lib/ai-model/auto-glm/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { adaptBbox, pointToBbox } from '@/common';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\n/**\n * Convert auto-glm coordinate [0,1000] to bbox in pixel coordinates\n */\nfunction autoGLMCoordinateToBbox(\n  x: number,\n  y: number,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const bbox = pointToBbox(x, y, 10);\n  return adaptBbox(bbox, width, height, width, height, 'auto-glm');\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'AndroidLongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: 'AndroidBackButton',\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: 'AndroidHomeButton',\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: launchAction.app,\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToBbox","x","y","width","height","bbox","pointToBbox","adaptBbox","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACFA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,MAAMC,0BAA0B;AAKhC,SAASC,wBACPC,CAAS,EACTC,CAAS,EACTC,KAAa,EACbC,MAAc;IAEd,MAAMC,OAAOC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYL,GAAGC,GAAG;IAC/B,OAAOK,AAAAA,IAAAA,mCAAAA,SAAAA,AAAAA,EAAUF,MAAMF,OAAOC,QAAQD,OAAOC,QAAQ;AACvD;AAwGO,SAASI,uBACdC,MAAoB,EACpBC,IAAuC;IAEvC,IAAI;QACF,OAAQD,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBZ,MAAM,4BAA4Bc;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBf,MAAM,yBAAyBgB;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGjB,wBACvBa,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBP;gCACxBf,MAAM,gCAAgCsB;gCACtC,MAAM,CAACL,IAAIC,IAAIC,IAAIC,GAAG,GAAGjB,wBACvBmB,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BT,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaR;gCACnBf,MAAM,0BAA0BuB;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcT;gCACpBf,MAAM,2BAA2BwB;gCAGjC,MAAM,CAACP,IAAIC,IAAIC,IAAIC,GAAG,GAAGjB,wBACvBqB,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBX,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMK,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYlB,KAAK,MAAM,GAAIX;oCAE9ByB,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYhB,KAAK,KAAK,GAAIX;oCAE7ByB,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAzB,MACE,CAAC,2BAA2B,EAAE2B,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBjB;gCACxBf,MAAM,gCAAgCgC;gCACtC,MAAM,CAACf,IAAIC,IAAIC,IAAIC,GAAG,GAAGjB,wBACvB6B,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BnB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAalB;gCACnBf,MAAM,0BAA0BiC;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO,CAAC;wCACR,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAanB;gCACnBf,MAAM,0BAA0BkC;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO,CAAC;wCACR,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAapB;gCACnBf,MAAM,0BAA0BmC;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAerB;gCACrBf,MAAM,4BAA4BoC;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAOA,aAAa,GAAG;wCACvB,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGtB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIsB,MACR,CAAC,yBAAyB,EAAGzB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO0B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrEtC,MAAM,oBAAoBuC;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/lib/ai-model/auto-glm/index.js

@@ -0,0 +1,63 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    parseAction: ()=>external_parser_js_namespaceObject.parseAction,
+    isAutoGLM: ()=>external_util_js_namespaceObject.isAutoGLM,
+    autoGLMPlanning: ()=>external_planning_js_namespaceObject.autoGLMPlanning,
+    getAutoGLMPlanPrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMPlanPrompt,
+    getAutoGLMLocatePrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMLocatePrompt,
+    transformAutoGLMAction: ()=>external_actions_js_namespaceObject.transformAutoGLMAction,
+    parseAutoGLMLocateResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMLocateResponse,
+    parseAutoGLMResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMResponse
+});
+const external_prompt_js_namespaceObject = require("./prompt.js");
+const external_parser_js_namespaceObject = require("./parser.js");
+const external_planning_js_namespaceObject = require("./planning.js");
+const external_actions_js_namespaceObject = require("./actions.js");
+const external_util_js_namespaceObject = require("./util.js");
+exports.autoGLMPlanning = __webpack_exports__.autoGLMPlanning;
+exports.getAutoGLMLocatePrompt = __webpack_exports__.getAutoGLMLocatePrompt;
+exports.getAutoGLMPlanPrompt = __webpack_exports__.getAutoGLMPlanPrompt;
+exports.isAutoGLM = __webpack_exports__.isAutoGLM;
+exports.parseAction = __webpack_exports__.parseAction;
+exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
+exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
+exports.transformAutoGLMAction = __webpack_exports__.transformAutoGLMAction;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "autoGLMPlanning",
+    "getAutoGLMLocatePrompt",
+    "getAutoGLMPlanPrompt",
+    "isAutoGLM",
+    "parseAction",
+    "parseAutoGLMLocateResponse",
+    "parseAutoGLMResponse",
+    "transformAutoGLMAction"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=index.js.map

package/dist/lib/ai-model/auto-glm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/ai-model/auto-glm/parser.js

@@ -0,0 +1,282 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    extractValueAfter: ()=>extractValueAfter,
+    parseAction: ()=>parseAction,
+    parseAutoGLMLocateResponse: ()=>parseAutoGLMLocateResponse,
+    parseAutoGLMResponse: ()=>parseAutoGLMResponse
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const debug = (0, logger_namespaceObject.getDebug)('auto-glm-parser');
+const extractValueAfter = (src, key)=>{
+    const idx = src.indexOf(key);
+    if (-1 === idx) throw new Error(`Missing key ${key} in action payload ${src}`);
+    let rest = src.slice(idx + key.length).trim();
+    if (rest.endsWith('")')) rest = rest.slice(0, -2);
+    return rest;
+};
+function parseAction(response) {
+    debug('Parsing action:', response);
+    let trimmedResponse = '';
+    try {
+        trimmedResponse = response.content.trim();
+        if (trimmedResponse.startsWith('do(action="Type"') || trimmedResponse.startsWith('do(action="Type_Name"')) {
+            const text = extractValueAfter(trimmedResponse, 'text="');
+            return {
+                _metadata: 'do',
+                action: 'Type',
+                text,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('finish(message=')) {
+            let message = extractValueAfter(trimmedResponse, 'finish(message="');
+            if (message.endsWith(')')) message = message.slice(0, -1);
+            return {
+                _metadata: 'finish',
+                message,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('do(')) {
+            const actionMatch = trimmedResponse.match(/do\(action="([^"]+)"/);
+            if (!actionMatch) throw new Error(`Failed to extract action type from do() call; raw="${trimmedResponse}"`);
+            const actionType = actionMatch[1];
+            const baseAction = {
+                _metadata: 'do',
+                think: response.think
+            };
+            switch(actionType){
+                case 'Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Double Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Double Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Double Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Swipe':
+                    {
+                        const startMatch = trimmedResponse.match(/start=\[(\d+),(\d+)\]/);
+                        const endMatch = trimmedResponse.match(/end=\[(\d+),(\d+)\]/);
+                        if (!startMatch || !endMatch) throw new Error(`Failed to extract start/end coordinates for Swipe; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Swipe',
+                            start: [
+                                Number(startMatch[1]),
+                                Number(startMatch[2])
+                            ],
+                            end: [
+                                Number(endMatch[1]),
+                                Number(endMatch[2])
+                            ]
+                        };
+                    }
+                case 'Long Press':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Long Press; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Long Press',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Launch':
+                    {
+                        const app = extractValueAfter(trimmedResponse, 'app="');
+                        return {
+                            ...baseAction,
+                            action: 'Launch',
+                            app
+                        };
+                    }
+                case 'Back':
+                    return {
+                        ...baseAction,
+                        action: 'Back'
+                    };
+                case 'Home':
+                    return {
+                        ...baseAction,
+                        action: 'Home'
+                    };
+                case 'Wait':
+                    {
+                        const durationMatch = trimmedResponse.match(/duration=(?:["\[])?(\d+)/);
+                        if (!durationMatch) throw new Error(`Failed to extract duration for Wait; raw="${trimmedResponse}"`);
+                        const seconds = Number.parseInt(durationMatch[1], 10);
+                        const durationMs = 1000 * seconds;
+                        return {
+                            ...baseAction,
+                            action: 'Wait',
+                            durationMs
+                        };
+                    }
+                case 'Interact':
+                    return {
+                        ...baseAction,
+                        action: 'Interact'
+                    };
+                case 'Call_API':
+                    {
+                        const instruction = extractValueAfter(trimmedResponse, 'instruction="');
+                        return {
+                            ...baseAction,
+                            action: 'Call_API',
+                            instruction
+                        };
+                    }
+                case 'Take_over':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Take_over',
+                            message
+                        };
+                    }
+                case 'Note':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Note',
+                            message
+                        };
+                    }
+                default:
+                    throw new Error(`Unknown action type: ${actionType}; raw="${trimmedResponse}"`);
+            }
+        }
+        throw new Error(`Failed to parse action: ${trimmedResponse}`);
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to parse action: ${errorMessage}; raw="${trimmedResponse}"`);
+    }
+}
+function parseAutoGLMResponse(content) {
+    if (content.includes('finish(message=')) {
+        const parts = content.split('finish(message=');
+        const think = parts[0].trim();
+        const actionContent = `finish(message=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('do(action=')) {
+        const parts = content.split('do(action=');
+        const think = parts[0].trim();
+        const actionContent = `do(action=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('<answer>')) {
+        const parts = content.split('<answer>');
+        const think = parts[0].replace(/<think>/g, '').replace(/<\/think>/g, '').trim();
+        const actionContent = parts[1].replace(/<\/answer>/g, '').trim();
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    return {
+        think: '',
+        content
+    };
+}
+function parseAutoGLMLocateResponse(rawResponse) {
+    const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);
+    if (!actionContent.startsWith('do(action="Tap"')) return {
+        think,
+        coordinates: null,
+        error: `Unexpected action type in auto-glm locate response: ${actionContent}`
+    };
+    try {
+        const elementMatch = actionContent.match(/element=\[(\d+),(\d+)\]/);
+        if (!elementMatch) return {
+            think,
+            coordinates: null,
+            error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`
+        };
+        const x = Number(elementMatch[1]);
+        const y = Number(elementMatch[2]);
+        return {
+            think,
+            coordinates: {
+                x,
+                y
+            }
+        };
+    } catch (e) {
+        const errorMessage = e instanceof Error ? e.message : String(e);
+        return {
+            think,
+            coordinates: null,
+            error: `Failed to parse coordinates "${actionContent}" with errorMessage: ${errorMessage}`
+        };
+    }
+}
+exports.extractValueAfter = __webpack_exports__.extractValueAfter;
+exports.parseAction = __webpack_exports__.parseAction;
+exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
+exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "extractValueAfter",
+    "parseAction",
+    "parseAutoGLMLocateResponse",
+    "parseAutoGLMResponse"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=parser.js.map