@midscene/core 1.9.8 → 1.10.0

package/dist/lib/ai-model/models/auto-glm/actions.js

@@ -27,27 +27,7 @@ __webpack_require__.d(__webpack_exports__, {
     transformAutoGLMAction: ()=>transformAutoGLMAction
 });
 const logger_namespaceObject = require("@midscene/shared/logger");
-const bbox_js_namespaceObject = require("../../shared/model-locate-result/bbox.js");
-const pixel_bbox_mapper_js_namespaceObject = require("../../shared/model-locate-result/pixel-bbox-mapper.js");
 const debug = (0, logger_namespaceObject.getDebug)('auto-glm-actions');
-const AUTO_GLM_COORDINATE_MAX = 1000;
-function autoGLMCoordinateToLocateParam(coordinate, size) {
-    const ctx = {
-        preparedSize: size
-    };
-    const pixelBbox = (0, pixel_bbox_mapper_js_namespaceObject.mapLocateResultToPixelBboxByCoordinates)({
-        type: 'point',
-        coordinates: coordinate
-    }, ctx, {
-        shape: 'point',
-        order: 'xy',
-        normalizedBy: AUTO_GLM_COORDINATE_MAX
-    });
-    return {
-        prompt: '',
-        locatedPixelBbox: (0, bbox_js_namespaceObject.finalizePixelBbox)(pixelBbox, coordinate, ctx)
-    };
-}
 const BACK_BUTTON_NAMES = [
     'AndroidBackButton',
     'HarmonyBackButton'
@@ -61,7 +41,7 @@ function findActionName(actionSpace, knownNames, defaultName) {
     const match = actionSpace.find((a)=>knownNames.includes(a.name));
     return match ? match.name : defaultName;
 }
-function transformAutoGLMAction(action, size, actionSpace) {
+function transformAutoGLMAction(action, { actionSpace, coordinateDistanceToPixels }) {
     try {
         switch(action._metadata){
             case 'finish':
@@ -84,12 +64,14 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const tapAction = doAction;
                                 debug('Transform Tap action:', tapAction);
-                                const locate = autoGLMCoordinateToLocateParam(tapAction.element, size);
                                 return [
                                     {
                                         type: 'Tap',
                                         param: {
-                                            locate
+                                            locate: {
+                                                point: tapAction.element,
+                                                prompt: ''
+                                            }
                                         }
                                     }
                                 ];
@@ -98,12 +80,14 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const doubleTapAction = doAction;
                                 debug('Transform Double Tap action:', doubleTapAction);
-                                const locate = autoGLMCoordinateToLocateParam(doubleTapAction.element, size);
                                 return [
                                     {
                                         type: 'DoubleClick',
                                         param: {
-                                            locate
+                                            locate: {
+                                                point: doubleTapAction.element,
+                                                prompt: ''
+                                            }
                                         }
                                     }
                                 ];
@@ -125,7 +109,6 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const swipeAction = doAction;
                                 debug('Transform Swipe action:', swipeAction);
-                                const locate = autoGLMCoordinateToLocateParam(swipeAction.start, size);
                                 const deltaX = swipeAction.end[0] - swipeAction.start[0];
                                 const deltaY = swipeAction.end[1] - swipeAction.start[1];
                                 let direction;
@@ -133,10 +116,10 @@ function transformAutoGLMAction(action, size, actionSpace) {
                                 const absDeltaX = Math.abs(deltaX);
                                 const absDeltaY = Math.abs(deltaY);
                                 if (absDeltaY > absDeltaX) {
-                                    distance = Math.round(absDeltaY * size.height / AUTO_GLM_COORDINATE_MAX);
+                                    distance = coordinateDistanceToPixels(deltaY, 'y');
                                     direction = deltaY > 0 ? 'up' : 'down';
                                 } else {
-                                    distance = Math.round(absDeltaX * size.width / AUTO_GLM_COORDINATE_MAX);
+                                    distance = coordinateDistanceToPixels(deltaX, 'x');
                                     direction = deltaX > 0 ? 'left' : 'right';
                                 }
                                 debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
@@ -144,7 +127,10 @@ function transformAutoGLMAction(action, size, actionSpace) {
                                     {
                                         type: 'Scroll',
                                         param: {
-                                            locate,
+                                            locate: {
+                                                point: swipeAction.start,
+                                                prompt: ''
+                                            },
                                             distance,
                                             direction
                                         },
@@ -156,12 +142,14 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const longPressAction = doAction;
                                 debug('Transform Long Press action:', longPressAction);
-                                const locate = autoGLMCoordinateToLocateParam(longPressAction.element, size);
                                 return [
                                     {
                                         type: 'LongPress',
                                         param: {
-                                            locate
+                                            locate: {
+                                                point: longPressAction.element,
+                                                prompt: ''
+                                            }
                                         },
                                         thought: longPressAction.think || ''
                                     }

package/dist/lib/ai-model/models/auto-glm/actions.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/auto-glm/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/auto-glm/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/device';\nimport type {\n  PlanningAction,\n  PlanningLocateParamWithLocatedPixelBbox,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { finalizePixelBbox } from '../../shared/model-locate-result/bbox';\nimport { mapLocateResultToPixelBboxByCoordinates } from '../../shared/model-locate-result/pixel-bbox-mapper';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\nfunction autoGLMCoordinateToLocateParam(\n  coordinate: [number, number],\n  size: { width: number; height: number },\n): PlanningLocateParamWithLocatedPixelBbox {\n  const ctx = { preparedSize: size };\n  const pixelBbox = mapLocateResultToPixelBboxByCoordinates(\n    { type: 'point', coordinates: coordinate },\n    ctx,\n    { shape: 'point', order: 'xy', normalizedBy: AUTO_GLM_COORDINATE_MAX },\n  );\n\n  return {\n    prompt: '',\n    locatedPixelBbox: finalizePixelBbox(pixelBbox, coordinate, ctx),\n  };\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n  actionSpace?: DeviceAction[],\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const locate = autoGLMCoordinateToLocateParam(\n              tapAction.element,\n              size,\n            );\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const locate = autoGLMCoordinateToLocateParam(\n              doubleTapAction.element,\n              size,\n            );\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            const locate = autoGLMCoordinateToLocateParam(\n              swipeAction.start,\n              size,\n            );\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const locate = autoGLMCoordinateToLocateParam(\n              longPressAction.element,\n              size,\n            );\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: { uri: launchAction.app },\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToLocateParam","coordinate","size","ctx","pixelBbox","mapLocateResultToPixelBboxByCoordinates","finalizePixelBbox","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","finishAction","doAction","tapAction","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACGA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,MAAMC,0BAA0B;AAEhC,SAASC,+BACPC,UAA4B,EAC5BC,IAAuC;IAEvC,MAAMC,MAAM;QAAE,cAAcD;IAAK;IACjC,MAAME,YAAYC,AAAAA,IAAAA,qCAAAA,uCAAAA,AAAAA,EAChB;QAAE,MAAM;QAAS,aAAaJ;IAAW,GACzCE,KACA;QAAE,OAAO;QAAS,OAAO;QAAM,cAAcJ;IAAwB;IAGvE,OAAO;QACL,QAAQ;QACR,kBAAkBO,AAAAA,IAAAA,wBAAAA,iBAAAA,AAAAA,EAAkBF,WAAWH,YAAYE;IAC7D;AACF;AAwGA,MAAMI,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAAoB,EACpBd,IAAuC,EACvCQ,WAA4B;IAE5B,IAAI;QACF,OAAQM,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAMC,eAAeD;oBACrBnB,MAAM,4BAA4BoB;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWF;oBAejB,OAASE,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBrB,MAAM,yBAAyBsB;gCAC/B,MAAMC,SAASpB,+BACbmB,UAAU,OAAO,EACjBjB;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLkB;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBH;gCACxBrB,MAAM,gCAAgCwB;gCACtC,MAAMD,SAASpB,+BACbqB,gBAAgB,OAAO,EACvBnB;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLkB;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaJ;gCACnBrB,MAAM,0BAA0ByB;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcL;gCACpBrB,MAAM,2BAA2B0B;gCAEjC,MAAMH,SAASpB,+BACbuB,YAAY,KAAK,EACjBrB;gCAIF,MAAMsB,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAY5B,KAAK,MAAM,GAAIH;oCAE9B2B,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAY1B,KAAK,KAAK,GAAIH;oCAE7B2B,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEA3B,MACE,CAAC,2BAA2B,EAAE6B,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBb;gCACxBrB,MAAM,gCAAgCkC;gCACtC,MAAMX,SAASpB,+BACb+B,gBAAgB,OAAO,EACvB7B;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLkB;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAad;gCACnBrB,MAAM,0BAA0BmC;gCAChC,OAAO;oCACL;wCACE,MAAMvB,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaf;gCACnBrB,MAAM,0BAA0BoC;gCAChC,OAAO;oCACL;wCACE,MAAMxB,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAahB;gCACnBrB,MAAM,0BAA0BqC;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAejB;gCACrBrB,MAAM,4BAA4BsC;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CAAE,KAAKA,aAAa,GAAG;wCAAC;wCAC/B,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGlB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIkB,MACR,CAAC,yBAAyB,EAAGpB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAOqB,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrExC,MAAM,oBAAoByC;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}
+{"version":3,"file":"ai-model/models/auto-glm/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/auto-glm/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { CoordinateDistanceAxis } from '../../shared/model-locate-result';\nimport type {\n  LocatePlanningAction,\n  ScrollPlanningAction,\n} from '../../shared/planning-action';\n\nconst debug = getDebug('auto-glm-actions');\n\ntype CoordinateDistanceToPixels = (\n  delta: number,\n  axis: CoordinateDistanceAxis,\n) => number;\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type AutoGLMParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: AutoGLMParsedAction,\n  {\n    actionSpace,\n    coordinateDistanceToPixels,\n  }: {\n    actionSpace?: DeviceAction[];\n    coordinateDistanceToPixels: CoordinateDistanceToPixels;\n  },\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate: {\n                    point: tapAction.element,\n                    prompt: '',\n                  },\n                },\n              } satisfies LocatePlanningAction<'Tap'>,\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate: {\n                    point: doubleTapAction.element,\n                    prompt: '',\n                  },\n                },\n              } satisfies LocatePlanningAction<'DoubleClick'>,\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate horizontal and vertical delta in the model coordinate system.\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = coordinateDistanceToPixels(deltaY, 'y');\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = coordinateDistanceToPixels(deltaX, 'x');\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate: {\n                    point: swipeAction.start,\n                    prompt: '',\n                  },\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              } satisfies ScrollPlanningAction,\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate: {\n                    point: longPressAction.element,\n                    prompt: '',\n                  },\n                },\n                thought: longPressAction.think || '',\n              } satisfies LocatePlanningAction<'LongPress'>,\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: { uri: launchAction.app },\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","coordinateDistanceToPixels","finishAction","doAction","tapAction","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACGA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AA6GvB,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAA2B,EAC3B,EACEN,WAAW,EACXO,0BAA0B,EAI3B;IAED,IAAI;QACF,OAAQD,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBX,MAAM,4BAA4Ba;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBd,MAAM,yBAAyBe;gCAE/B,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOA,UAAU,OAAO;gDACxB,QAAQ;4CACV;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBF;gCACxBd,MAAM,gCAAgCgB;gCAEtC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOA,gBAAgB,OAAO;gDAC9B,QAAQ;4CACV;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaH;gCACnBd,MAAM,0BAA0BiB;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcJ;gCACpBd,MAAM,2BAA2BkB;gCAGjC,MAAMC,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWV,2BAA2BQ,QAAQ;oCAC9CC,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWV,2BAA2BO,QAAQ;oCAC9CE,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAnB,MACE,CAAC,2BAA2B,EAAEqB,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOJ,YAAY,KAAK;gDACxB,QAAQ;4CACV;4CAEAI;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBZ;gCACxBd,MAAM,gCAAgC0B;gCAEtC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOA,gBAAgB,OAAO;gDAC9B,QAAQ;4CACV;wCACF;wCACA,SAASA,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAab;gCACnBd,MAAM,0BAA0B2B;gCAChC,OAAO;oCACL;wCACE,MAAMvB,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAad;gCACnBd,MAAM,0BAA0B4B;gCAChC,OAAO;oCACL;wCACE,MAAMxB,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaf;gCACnBd,MAAM,0BAA0B6B;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAehB;gCACrBd,MAAM,4BAA4B8B;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CAAE,KAAKA,aAAa,GAAG;wCAAC;wCAC/B,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGjB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIiB,MACR,CAAC,yBAAyB,EAAGpB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAOqB,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrEhC,MAAM,oBAAoBiC;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/lib/ai-model/models/auto-glm/adapter.js

@@ -28,9 +28,7 @@ __webpack_require__.d(__webpack_exports__, {
 });
 const external_locate_js_namespaceObject = require("./locate.js");
 const external_planning_js_namespaceObject = require("./planning.js");
-const external_prompt_js_namespaceObject = require("./prompt.js");
-const defaultAutoGlmReplanningCycleLimit = 100;
-function createAutoGlmAdapter({ getPlanPrompt, getLocatePrompt }) {
+function createAutoGlmAdapter(isMultilingual) {
     return {
         chatCompletion: {
             unsupportedUserConfig: [
@@ -57,24 +55,18 @@ function createAutoGlmAdapter({ getPlanPrompt, getLocatePrompt }) {
         planning: {
             kind: 'custom',
             cacheEnabled: false,
-            defaultReplanningCycleLimit: defaultAutoGlmReplanningCycleLimit,
-            planFn: (userInstruction, options)=>(0, external_planning_js_namespaceObject.autoGlmPlanning)(userInstruction, options, getPlanPrompt)
+            defaultReplanningCycleLimit: 100,
+            planner: (0, external_planning_js_namespaceObject.createAutoGlmPlanner)(isMultilingual)
         },
         locate: {
             kind: 'custom',
-            locateFn: (elementDescription, options)=>(0, external_locate_js_namespaceObject.autoGlmLocate)(elementDescription, options, getLocatePrompt)
+            planningTapLocator: (0, external_locate_js_namespaceObject.createAutoGlmPlanningTapLocator)(isMultilingual)
         }
     };
 }
 const autoGlmAdapters = {
-    'auto-glm': createAutoGlmAdapter({
-        getPlanPrompt: external_prompt_js_namespaceObject.getAutoGLMChinesePlanPrompt,
-        getLocatePrompt: external_prompt_js_namespaceObject.getAutoGLMChineseLocatePrompt
-    }),
-    'auto-glm-multilingual': createAutoGlmAdapter({
-        getPlanPrompt: external_prompt_js_namespaceObject.getAutoGLMMultilingualPlanPrompt,
-        getLocatePrompt: external_prompt_js_namespaceObject.getAutoGLMMultilingualLocatePrompt
-    })
+    'auto-glm': createAutoGlmAdapter(false),
+    'auto-glm-multilingual': createAutoGlmAdapter(true)
 };
 exports.autoGlmAdapters = __webpack_exports__.autoGlmAdapters;
 for(var __rspack_i in __webpack_exports__)if (-1 === [

package/dist/lib/ai-model/models/auto-glm/adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/auto-glm/adapter.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/auto-glm/adapter.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../types';\nimport { autoGlmLocate } from './locate';\nimport { autoGlmPlanning } from './planning';\nimport {\n  getAutoGLMChineseLocatePrompt,\n  getAutoGLMChinesePlanPrompt,\n  getAutoGLMMultilingualLocatePrompt,\n  getAutoGLMMultilingualPlanPrompt,\n} from './prompt';\n\nconst defaultAutoGlmReplanningCycleLimit = 100;\n\nfunction createAutoGlmAdapter({\n  getPlanPrompt,\n  getLocatePrompt,\n}: {\n  getPlanPrompt: () => string;\n  getLocatePrompt: () => string;\n}): ModelAdapterDefinition {\n  return {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        const modelSpecificConfig = {\n          top_p: 0.85,\n          frequency_penalty: 0.2,\n        };\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n            ...modelSpecificConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultAutoGlmReplanningCycleLimit,\n      planFn: (userInstruction, options) =>\n        autoGlmPlanning(userInstruction, options, getPlanPrompt),\n    },\n    locate: {\n      kind: 'custom',\n      locateFn: (elementDescription, options) =>\n        autoGlmLocate(elementDescription, options, getLocatePrompt),\n    },\n  };\n}\n\nexport const autoGlmAdapters = {\n  'auto-glm': createAutoGlmAdapter({\n    getPlanPrompt: getAutoGLMChinesePlanPrompt,\n    getLocatePrompt: getAutoGLMChineseLocatePrompt,\n  }),\n  'auto-glm-multilingual': createAutoGlmAdapter({\n    getPlanPrompt: getAutoGLMMultilingualPlanPrompt,\n    getLocatePrompt: getAutoGLMMultilingualLocatePrompt,\n  }),\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'auto-glm' | 'auto-glm-multilingual'\n>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultAutoGlmReplanningCycleLimit","createAutoGlmAdapter","getPlanPrompt","getLocatePrompt","midsceneDefaults","userConfig","commonOverrideConfig","undefined","modelSpecificConfig","userInstruction","options","autoGlmPlanning","elementDescription","autoGlmLocate","autoGlmAdapters","getAutoGLMChinesePlanPrompt","getAutoGLMChineseLocatePrompt","getAutoGLMMultilingualPlanPrompt","getAutoGLMMultilingualLocatePrompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACKA,MAAMI,qCAAqC;AAE3C,SAASC,qBAAqB,EAC5BC,aAAa,EACbC,eAAe,EAIhB;IACC,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEC,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,MAAMG,sBAAsB;oBAC1B,OAAO;oBACP,mBAAmB;gBACrB;gBAEA,OAAO;oBACL,QAAQ;wBACN,GAAGJ,gBAAgB;wBACnB,GAAGE,oBAAoB;wBACvB,GAAGE,mBAAmB;oBACxB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BR;YAC7B,QAAQ,CAACS,iBAAiBC,UACxBC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBF,iBAAiBC,SAASR;QAC9C;QACA,QAAQ;YACN,MAAM;YACN,UAAU,CAACU,oBAAoBF,UAC7BG,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAcD,oBAAoBF,SAASP;QAC/C;IACF;AACF;AAEO,MAAMW,kBAAkB;IAC7B,YAAYb,qBAAqB;QAC/B,eAAec,mCAAAA,2BAA2BA;QAC1C,iBAAiBC,mCAAAA,6BAA6BA;IAChD;IACA,yBAAyBf,qBAAqB;QAC5C,eAAegB,mCAAAA,gCAAgCA;QAC/C,iBAAiBC,mCAAAA,kCAAkCA;IACrD;AACF"}
+{"version":3,"file":"ai-model/models/auto-glm/adapter.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/auto-glm/adapter.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { createAutoGlmPlanningTapLocator } from './locate';\nimport { createAutoGlmPlanner } from './planning';\n\nfunction createAutoGlmAdapter(isMultilingual: boolean): ModelAdapterDefinition {\n  return {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        const modelSpecificConfig = {\n          top_p: 0.85,\n          frequency_penalty: 0.2,\n        };\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n            ...modelSpecificConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: 100,\n      planner: createAutoGlmPlanner(isMultilingual),\n    },\n    locate: {\n      kind: 'custom',\n      planningTapLocator: createAutoGlmPlanningTapLocator(isMultilingual),\n    },\n  };\n}\n\nexport const autoGlmAdapters = {\n  'auto-glm': createAutoGlmAdapter(false),\n  'auto-glm-multilingual': createAutoGlmAdapter(true),\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'auto-glm' | 'auto-glm-multilingual'\n>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createAutoGlmAdapter","isMultilingual","midsceneDefaults","userConfig","commonOverrideConfig","undefined","modelSpecificConfig","createAutoGlmPlanner","createAutoGlmPlanningTapLocator","autoGlmAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACDA,SAASI,qBAAqBC,cAAuB;IACnD,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEC,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,MAAMG,sBAAsB;oBAC1B,OAAO;oBACP,mBAAmB;gBACrB;gBAEA,OAAO;oBACL,QAAQ;wBACN,GAAGJ,gBAAgB;wBACnB,GAAGE,oBAAoB;wBACvB,GAAGE,mBAAmB;oBACxB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6B;YAC7B,SAASC,AAAAA,IAAAA,qCAAAA,oBAAAA,AAAAA,EAAqBN;QAChC;QACA,QAAQ;YACN,MAAM;YACN,oBAAoBO,AAAAA,IAAAA,mCAAAA,+BAAAA,AAAAA,EAAgCP;QACtD;IACF;AACF;AAEO,MAAMQ,kBAAkB;IAC7B,YAAYT,qBAAqB;IACjC,yBAAyBA,qBAAqB;AAChD"}

package/dist/lib/ai-model/models/auto-glm/locate.js

@@ -24,121 +24,19 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    autoGlmLocate: ()=>autoGlmLocate
+    createAutoGlmPlanningTapLocator: ()=>createAutoGlmPlanningTapLocator
 });
-const extractor_namespaceObject = require("@midscene/shared/extractor");
-const logger_namespaceObject = require("@midscene/shared/logger");
-const utils_namespaceObject = require("@midscene/shared/utils");
-const external_inspect_js_namespaceObject = require("../../inspect.js");
-const llm_locator_js_namespaceObject = require("../../prompt/llm-locator.js");
-const index_js_namespaceObject = require("../../service-caller/index.js");
-const bbox_js_namespaceObject = require("../../shared/model-locate-result/bbox.js");
-const pixel_bbox_mapper_js_namespaceObject = require("../../shared/model-locate-result/pixel-bbox-mapper.js");
-const locate_result_rect_js_namespaceObject = require("../../workflows/inspect/locate-result-rect.js");
-const search_area_mapping_js_namespaceObject = require("../../workflows/inspect/search-area-mapping.js");
-const external_parser_js_namespaceObject = require("./parser.js");
-const debugInspect = (0, logger_namespaceObject.getDebug)('ai:inspect');
-async function autoGlmLocate(elementDescription, options, getSystemPrompt) {
-    const { context, modelRuntime } = options;
-    const screenshotBase64 = context.screenshot.base64;
-    (0, utils_namespaceObject.assert)(elementDescription, "cannot find the target element description");
-    const elementDescriptionText = (0, external_inspect_js_namespaceObject.extraTextFromUserPrompt)(elementDescription);
-    const userInstructionPrompt = (0, llm_locator_js_namespaceObject.findElementPrompt)(elementDescriptionText);
-    const locateImage = options.searchConfig?.image ?? {
-        imageBase64: screenshotBase64,
-        width: context.shotSize.width,
-        height: context.shotSize.height
-    };
-    const imagePayload = locateImage.imageBase64;
-    const imageWidth = locateImage.width;
-    const imageHeight = locateImage.height;
-    const msgs = [
-        {
-            role: 'system',
-            content: getSystemPrompt()
-        },
-        {
-            role: 'user',
-            content: [
-                {
-                    type: 'image_url',
-                    image_url: {
-                        url: imagePayload,
-                        detail: 'high'
-                    }
-                },
-                {
-                    type: 'text',
-                    text: `Tap: ${userInstructionPrompt}`
-                }
-            ]
-        }
-    ];
-    if ('string' != typeof elementDescription) {
-        const addOns = await (0, external_inspect_js_namespaceObject.promptsToChatParam)({
-            images: elementDescription.images,
-            convertHttpImage2Base64: elementDescription.convertHttpImage2Base64
-        });
-        msgs.push(...addOns);
-    }
-    const { content: rawResponseContent, usage, rawChoiceMessage } = await (0, index_js_namespaceObject.callAIWithStringResponse)(msgs, modelRuntime, {
-        abortSignal: options.abortSignal
-    });
-    debugInspect('auto-glm rawResponse:', rawResponseContent);
-    const parsed = (0, external_parser_js_namespaceObject.parseAutoGLMLocateResponse)(rawResponseContent);
-    debugInspect('auto-glm thinking:', parsed.think);
-    debugInspect('auto-glm coordinates:', parsed.coordinates);
-    let resRect;
-    let matchedElement;
-    let errors = [];
-    if (parsed.error || !parsed.coordinates) {
-        errors = [
-            parsed.error || 'Failed to parse auto-glm response'
-        ];
-        debugInspect('auto-glm parse error:', errors[0]);
-    } else {
-        const { x, y } = parsed.coordinates;
-        debugInspect('auto-glm coordinates [0-999]:', {
-            x,
-            y
-        });
-        const ctx = {
-            preparedSize: {
-                width: imageWidth,
-                height: imageHeight
-            }
-        };
-        const targetPixelBbox = (0, bbox_js_namespaceObject.finalizePixelBbox)((0, pixel_bbox_mapper_js_namespaceObject.mapLocateResultToPixelBboxByCoordinates)({
-            type: 'point',
-            coordinates: [
-                x,
-                y
-            ]
-        }, ctx, {
-            shape: 'point',
-            order: 'xy',
-            normalizedBy: 1000
-        }), parsed.coordinates, ctx);
-        resRect = (0, locate_result_rect_js_namespaceObject.pixelBboxToRect)((0, search_area_mapping_js_namespaceObject.mapSearchAreaPixelBboxToOriginalPixelBbox)(targetPixelBbox, options.searchConfig?.mapping));
-        debugInspect('auto-glm resRect:', resRect);
-        const element = (0, extractor_namespaceObject.generateElementByRect)(resRect, elementDescriptionText);
-        if (element) matchedElement = element;
-    }
+const planning_action_js_namespaceObject = require("../../shared/planning-action.js");
+const external_prompt_js_namespaceObject = require("./prompt.js");
+function createAutoGlmPlanningTapLocator(isMultilingual) {
     return {
-        rect: resRect,
-        parseResult: {
-            element: matchedElement,
-            errors
-        },
-        rawResponse: rawResponseContent,
-        rawChoiceMessage,
-        usage,
-        reasoning_content: parsed.think
+        buildSystemPrompt: ()=>isMultilingual ? (0, external_prompt_js_namespaceObject.getAutoGLMMultilingualLocatePrompt)() : (0, external_prompt_js_namespaceObject.getAutoGLMChineseLocatePrompt)(),
+        getLocatedPixelBbox: planning_action_js_namespaceObject.getTapLocatedPixelBbox
     };
 }
-exports.autoGlmLocate = __webpack_exports__.autoGlmLocate;
+exports.createAutoGlmPlanningTapLocator = __webpack_exports__.createAutoGlmPlanningTapLocator;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
-    "autoGlmLocate"
+    "createAutoGlmPlanningTapLocator"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true

package/dist/lib/ai-model/models/auto-glm/locate.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/auto-glm/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/auto-glm/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Rect } from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport {\n  type InspectAIArgs,\n  extraTextFromUserPrompt,\n  promptsToChatParam,\n} from '../../inspect';\nimport { findElementPrompt } from '../../prompt/llm-locator';\nimport { callAIWithStringResponse } from '../../service-caller/index';\nimport { finalizePixelBbox } from '../../shared/model-locate-result/bbox';\nimport { mapLocateResultToPixelBboxByCoordinates } from '../../shared/model-locate-result/pixel-bbox-mapper';\nimport { pixelBboxToRect } from '../../workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from '../../workflows/inspect/search-area-mapping';\nimport type {\n  LocateOptions,\n  LocateResult,\n} from '../../workflows/inspect/types';\nimport { parseAutoGLMLocateResponse } from './parser';\n\nconst debugInspect = getDebug('ai:inspect');\n\nexport async function autoGlmLocate(\n  elementDescription: TUserPrompt,\n  options: LocateOptions,\n  getSystemPrompt: () => string,\n): Promise<LocateResult> {\n  const { context, modelRuntime } = options;\n  const screenshotBase64 = context.screenshot.base64;\n\n  assert(elementDescription, 'cannot find the target element description');\n  const elementDescriptionText = extraTextFromUserPrompt(elementDescription);\n  const userInstructionPrompt = findElementPrompt(elementDescriptionText);\n\n  const locateImage = options.searchConfig?.image ?? {\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const imagePayload = locateImage.imageBase64;\n  const imageWidth = locateImage.width;\n  const imageHeight = locateImage.height;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: getSystemPrompt() },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: `Tap: ${userInstructionPrompt}`,\n        },\n      ],\n    },\n  ];\n\n  if (typeof elementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: elementDescription.images,\n      convertHttpImage2Base64: elementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponseContent,\n    usage,\n    rawChoiceMessage,\n  } = await callAIWithStringResponse(msgs, modelRuntime, {\n    abortSignal: options.abortSignal,\n  });\n\n  debugInspect('auto-glm rawResponse:', rawResponseContent);\n\n  const parsed = parseAutoGLMLocateResponse(rawResponseContent);\n\n  debugInspect('auto-glm thinking:', parsed.think);\n  debugInspect('auto-glm coordinates:', parsed.coordinates);\n\n  let resRect: Rect | undefined;\n  let matchedElement: LocateResultElement | undefined;\n  let errors: string[] = [];\n\n  if (parsed.error || !parsed.coordinates) {\n    errors = [parsed.error || 'Failed to parse auto-glm response'];\n    debugInspect('auto-glm parse error:', errors[0]);\n  } else {\n    const { x, y } = parsed.coordinates;\n\n    debugInspect('auto-glm coordinates [0-999]:', { x, y });\n\n    const ctx = { preparedSize: { width: imageWidth, height: imageHeight } };\n    const targetPixelBbox = finalizePixelBbox(\n      mapLocateResultToPixelBboxByCoordinates(\n        { type: 'point', coordinates: [x, y] },\n        ctx,\n        { shape: 'point', order: 'xy', normalizedBy: 1000 },\n      ),\n      parsed.coordinates,\n      ctx,\n    );\n    resRect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        targetPixelBbox,\n        options.searchConfig?.mapping,\n      ),\n    );\n\n    debugInspect('auto-glm resRect:', resRect);\n\n    const element: LocateResultElement = generateElementByRect(\n      resRect,\n      elementDescriptionText as string,\n    );\n\n    if (element) {\n      matchedElement = element;\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      element: matchedElement,\n      errors,\n    },\n    rawResponse: rawResponseContent,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: parsed.think,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInspect","getDebug","autoGlmLocate","elementDescription","options","getSystemPrompt","context","modelRuntime","screenshotBase64","assert","elementDescriptionText","extraTextFromUserPrompt","userInstructionPrompt","findElementPrompt","locateImage","imagePayload","imageWidth","imageHeight","msgs","addOns","promptsToChatParam","rawResponseContent","usage","rawChoiceMessage","callAIWithStringResponse","parsed","parseAutoGLMLocateResponse","resRect","matchedElement","errors","x","y","ctx","targetPixelBbox","finalizePixelBbox","mapLocateResultToPixelBboxByCoordinates","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","element","generateElementByRect"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ACiBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,eAAeC,cACpBC,kBAA+B,EAC/BC,OAAsB,EACtBC,eAA6B;IAE7B,MAAM,EAAEC,OAAO,EAAEC,YAAY,EAAE,GAAGH;IAClC,MAAMI,mBAAmBF,QAAQ,UAAU,CAAC,MAAM;IAElDG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAON,oBAAoB;IAC3B,MAAMO,yBAAyBC,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,EAAwBR;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,+BAAAA,iBAAAA,AAAAA,EAAkBH;IAEhD,MAAMI,cAAcV,QAAQ,YAAY,EAAE,SAAS;QACjD,aAAaI;QACb,OAAOF,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMS,eAAeD,YAAY,WAAW;IAC5C,MAAME,aAAaF,YAAY,KAAK;IACpC,MAAMG,cAAcH,YAAY,MAAM;IAEtC,MAAMI,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASb;QAAkB;QAC7C;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKU;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM,CAAC,KAAK,EAAEH,uBAAuB;gBACvC;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOT,oBAAiC;QAC1C,MAAMgB,SAAS,MAAMC,AAAAA,IAAAA,oCAAAA,kBAAAA,AAAAA,EAAmB;YACtC,QAAQjB,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAe,KAAK,IAAI,IAAIC;IACf;IAEA,MAAM,EACJ,SAASE,kBAAkB,EAC3BC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyBN,MAAMX,cAAc;QACrD,aAAaH,QAAQ,WAAW;IAClC;IAEAJ,aAAa,yBAAyBqB;IAEtC,MAAMI,SAASC,AAAAA,IAAAA,mCAAAA,0BAAAA,AAAAA,EAA2BL;IAE1CrB,aAAa,sBAAsByB,OAAO,KAAK;IAC/CzB,aAAa,yBAAyByB,OAAO,WAAW;IAExD,IAAIE;IACJ,IAAIC;IACJ,IAAIC,SAAmB,EAAE;IAEzB,IAAIJ,OAAO,KAAK,IAAI,CAACA,OAAO,WAAW,EAAE;QACvCI,SAAS;YAACJ,OAAO,KAAK,IAAI;SAAoC;QAC9DzB,aAAa,yBAAyB6B,MAAM,CAAC,EAAE;IACjD,OAAO;QACL,MAAM,EAAEC,CAAC,EAAEC,CAAC,EAAE,GAAGN,OAAO,WAAW;QAEnCzB,aAAa,iCAAiC;YAAE8B;YAAGC;QAAE;QAErD,MAAMC,MAAM;YAAE,cAAc;gBAAE,OAAOhB;gBAAY,QAAQC;YAAY;QAAE;QACvE,MAAMgB,kBAAkBC,AAAAA,IAAAA,wBAAAA,iBAAAA,AAAAA,EACtBC,AAAAA,IAAAA,qCAAAA,uCAAAA,AAAAA,EACE;YAAE,MAAM;YAAS,aAAa;gBAACL;gBAAGC;aAAE;QAAC,GACrCC,KACA;YAAE,OAAO;YAAS,OAAO;YAAM,cAAc;QAAK,IAEpDP,OAAO,WAAW,EAClBO;QAEFL,UAAUS,AAAAA,IAAAA,sCAAAA,eAAAA,AAAAA,EACRC,AAAAA,IAAAA,uCAAAA,yCAAAA,AAAAA,EACEJ,iBACA7B,QAAQ,YAAY,EAAE;QAI1BJ,aAAa,qBAAqB2B;QAElC,MAAMW,UAA+BC,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACnCZ,SACAjB;QAGF,IAAI4B,SACFV,iBAAiBU;IAErB;IAEA,OAAO;QACL,MAAMX;QACN,aAAa;YACX,SAASC;YACTC;QACF;QACA,aAAaR;QACbE;QACAD;QACA,mBAAmBG,OAAO,KAAK;IACjC;AACF"}
+{"version":3,"file":"ai-model/models/auto-glm/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/auto-glm/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { getTapLocatedPixelBbox } from '../../shared/planning-action';\nimport {\n  getAutoGLMChineseLocatePrompt,\n  getAutoGLMMultilingualLocatePrompt,\n} from './prompt';\n\nexport function createAutoGlmPlanningTapLocator(\n  isMultilingual: boolean,\n): PlanningTapLocatorDefinition {\n  return {\n    buildSystemPrompt: () =>\n      isMultilingual\n        ? getAutoGLMMultilingualLocatePrompt()\n        : getAutoGLMChineseLocatePrompt(),\n    getLocatedPixelBbox: getTapLocatedPixelBbox,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createAutoGlmPlanningTapLocator","isMultilingual","getAutoGLMMultilingualLocatePrompt","getAutoGLMChineseLocatePrompt","getTapLocatedPixelBbox"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACCO,SAASI,gCACdC,cAAuB;IAEvB,OAAO;QACL,mBAAmB,IACjBA,iBACIC,AAAAA,IAAAA,mCAAAA,kCAAAA,AAAAA,MACAC,AAAAA,IAAAA,mCAAAA,6BAAAA,AAAAA;QACN,qBAAqBC,mCAAAA,sBAAsBA;IAC7C;AACF"}

package/dist/lib/ai-model/models/auto-glm/parser.js

@@ -25,8 +25,8 @@ var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     extractValueAfter: ()=>extractValueAfter,
-    parseAction: ()=>parseAction,
-    parseAutoGLMLocateResponse: ()=>parseAutoGLMLocateResponse,
+    parseAutoGLMPlanningResponse: ()=>parseAutoGLMPlanningResponse,
+    parseAutoGLMPlanningAction: ()=>parseAutoGLMPlanningAction,
     parseAutoGLMResponse: ()=>parseAutoGLMResponse
 });
 const logger_namespaceObject = require("@midscene/shared/logger");
@@ -38,7 +38,7 @@ const extractValueAfter = (src, key)=>{
     if (rest.endsWith('")')) rest = rest.slice(0, -2);
     return rest;
 };
-function parseAction(response) {
+function parseAutoGLMPlanningAction(response) {
     debug('Parsing action:', response);
     let trimmedResponse = '';
     try {
@@ -201,78 +201,56 @@ function parseAction(response) {
     }
 }
 function parseAutoGLMResponse(content) {
+    let parsedResponse;
     if (content.includes('finish(message=')) {
         const parts = content.split('finish(message=');
         const think = parts[0].trim();
         const actionContent = `finish(message=${parts[1]}`;
-        return {
+        parsedResponse = {
             think,
             content: actionContent
         };
-    }
-    if (content.includes('do(action=')) {
+    } else if (content.includes('do(action=')) {
         const parts = content.split('do(action=');
         const think = parts[0].trim();
         const actionContent = `do(action=${parts[1]}`;
-        return {
+        parsedResponse = {
             think,
             content: actionContent
         };
-    }
-    if (content.includes('<answer>')) {
+    } else if (content.includes('<answer>')) {
         const parts = content.split('<answer>');
         const think = parts[0].replace(/<think>/g, '').replace(/<\/think>/g, '').trim();
         const actionContent = parts[1].replace(/<\/answer>/g, '').trim();
-        return {
+        parsedResponse = {
             think,
             content: actionContent
         };
-    }
-    return {
+    } else parsedResponse = {
         think: '',
         content
     };
+    debug('autoGLM rawResponse:', content);
+    debug('thinking in response:', parsedResponse.think);
+    debug('action in response:', parsedResponse.content);
+    return parsedResponse;
 }
-function parseAutoGLMLocateResponse(rawResponse) {
-    const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);
-    if (!actionContent.startsWith('do(action="Tap"')) return {
-        think,
-        coordinates: null,
-        error: `Unexpected action type in auto-glm locate response: ${actionContent}`
+function parseAutoGLMPlanningResponse(content) {
+    const response = parseAutoGLMResponse(content);
+    const action = parseAutoGLMPlanningAction(response);
+    return {
+        response,
+        action
     };
-    try {
-        const elementMatch = actionContent.match(/element=\[(\d+),(\d+)\]/);
-        if (!elementMatch) return {
-            think,
-            coordinates: null,
-            error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`
-        };
-        const x = Number(elementMatch[1]);
-        const y = Number(elementMatch[2]);
-        return {
-            think,
-            coordinates: {
-                x,
-                y
-            }
-        };
-    } catch (e) {
-        const errorMessage = e instanceof Error ? e.message : String(e);
-        return {
-            think,
-            coordinates: null,
-            error: `Failed to parse coordinates "${actionContent}" with errorMessage: ${errorMessage}`
-        };
-    }
 }
 exports.extractValueAfter = __webpack_exports__.extractValueAfter;
-exports.parseAction = __webpack_exports__.parseAction;
-exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
+exports.parseAutoGLMPlanningAction = __webpack_exports__.parseAutoGLMPlanningAction;
+exports.parseAutoGLMPlanningResponse = __webpack_exports__.parseAutoGLMPlanningResponse;
 exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "extractValueAfter",
-    "parseAction",
-    "parseAutoGLMLocateResponse",
+    "parseAutoGLMPlanningAction",
+    "parseAutoGLMPlanningResponse",
     "parseAutoGLMResponse"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {