@midscene/core 1.2.2-beta-20260115065918.0 → 1.2.2-beta-20260115090041.0

package/dist/es/ai-model/auto-glm/actions.mjs

@@ -0,0 +1,227 @@
+import { getDebug } from "@midscene/shared/logger";
+import { LatestLocateRecorder } from "../latest-locate-recorder.mjs";
+import { AUTO_GLM_COORDINATE_MAX, autoGLMCoordinateToBbox } from "./util.mjs";
+const debug = getDebug('auto-glm-actions');
+const lastLocateRecorder = new LatestLocateRecorder();
+function transformAutoGLMAction(action, size) {
+    try {
+        switch(action._metadata){
+            case 'finish':
+                {
+                    const finishAction = action;
+                    debug('Transform finish action:', finishAction);
+                    return [
+                        {
+                            type: 'Finished',
+                            param: {},
+                            thought: finishAction.message
+                        }
+                    ];
+                }
+            case 'do':
+                {
+                    const doAction = action;
+                    switch(doAction.action){
+                        case 'Tap':
+                            {
+                                const tapAction = doAction;
+                                debug('Transform Tap action:', tapAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(tapAction.element[0], tapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                lastLocateRecorder.recordLocate(locate, 'Tap');
+                                return [
+                                    {
+                                        type: 'Tap',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Double Tap':
+                            {
+                                const doubleTapAction = doAction;
+                                debug('Transform Double Tap action:', doubleTapAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(doubleTapAction.element[0], doubleTapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                lastLocateRecorder.recordLocate(locate, 'Double Tap');
+                                return [
+                                    {
+                                        type: 'DoubleClick',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Type':
+                            {
+                                const typeAction = doAction;
+                                debug('Transform Type action:', typeAction);
+                                const { locate: latestLocate, source } = lastLocateRecorder.getLatestLocate();
+                                debug(`use latestLocate from ${source} as locate when Input`, latestLocate);
+                                return [
+                                    {
+                                        type: 'Input',
+                                        param: {
+                                            value: typeAction.text,
+                                            locate: latestLocate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Swipe':
+                            {
+                                const swipeAction = doAction;
+                                debug('Transform Swipe action:', swipeAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(swipeAction.start[0], swipeAction.start[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                const deltaX = swipeAction.end[0] - swipeAction.start[0];
+                                const deltaY = swipeAction.end[1] - swipeAction.start[1];
+                                let direction;
+                                let distance;
+                                const absDeltaX = Math.abs(deltaX);
+                                const absDeltaY = Math.abs(deltaY);
+                                if (absDeltaY > absDeltaX) {
+                                    distance = Math.round(absDeltaY * size.height / AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaY > 0 ? 'up' : 'down';
+                                } else {
+                                    distance = Math.round(absDeltaX * size.width / AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaX > 0 ? 'left' : 'right';
+                                }
+                                debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
+                                return [
+                                    {
+                                        type: 'Scroll',
+                                        param: {
+                                            locate,
+                                            distance,
+                                            direction
+                                        },
+                                        thought: swipeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Long Press':
+                            {
+                                const longPressAction = doAction;
+                                debug('Transform Long Press action:', longPressAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(longPressAction.element[0], longPressAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                lastLocateRecorder.recordLocate(locate, 'Long Press');
+                                return [
+                                    {
+                                        type: 'AndroidLongPress',
+                                        param: {
+                                            locate
+                                        },
+                                        thought: longPressAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Back':
+                            {
+                                const backAction = doAction;
+                                debug('Transform Back action:', backAction);
+                                return [
+                                    {
+                                        type: 'AndroidBackButton',
+                                        param: {},
+                                        thought: backAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Home':
+                            {
+                                const homeAction = doAction;
+                                debug('Transform Home action:', homeAction);
+                                return [
+                                    {
+                                        type: 'AndroidHomeButton',
+                                        param: {},
+                                        thought: homeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Wait':
+                            {
+                                const waitAction = doAction;
+                                debug('Transform Wait action:', waitAction);
+                                return [
+                                    {
+                                        type: 'Sleep',
+                                        param: {
+                                            timeMs: waitAction.durationMs
+                                        },
+                                        thought: waitAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Launch':
+                            {
+                                const launchAction = doAction;
+                                debug('Transform Launch action:', launchAction);
+                                return [
+                                    {
+                                        type: 'Launch',
+                                        param: launchAction.app,
+                                        thought: launchAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Interact':
+                            throw new Error('Action "Interact" from auto-glm is not supported in the current implementation.');
+                        case 'Call_API':
+                            throw new Error('Action "Call_API" from auto-glm is not supported in the current implementation.');
+                        case 'Take_over':
+                            throw new Error('Action "Take_over" from auto-glm is not supported in the current implementation.');
+                        case 'Note':
+                            throw new Error('Action "Note" from auto-glm is not supported in the current implementation.');
+                        default:
+                            throw new Error(`Unknown do() action type: ${doAction.action}`);
+                    }
+                }
+            default:
+                throw new Error(`Unknown action metadata: ${action._metadata}`);
+        }
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        debug('Transform error:', errorMessage);
+        throw new Error(`Failed to transform action: ${errorMessage}`);
+    }
+}
+export { transformAutoGLMAction };
+
+//# sourceMappingURL=actions.mjs.map

package/dist/es/ai-model/auto-glm/actions.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/actions.mjs","sources":["../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["import type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { LatestLocateRecorder } from '../latest-locate-recorder';\nimport { AUTO_GLM_COORDINATE_MAX, autoGLMCoordinateToBbox } from './util';\n\nconst debug = getDebug('auto-glm-actions');\nconst lastLocateRecorder = new LatestLocateRecorder();\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n            lastLocateRecorder.recordLocate(locate, 'Tap');\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n            lastLocateRecorder.recordLocate(locate, 'Double Tap');\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n            const { locate: latestLocate, source } =\n              lastLocateRecorder.getLatestLocate();\n            debug(\n              `use latestLocate from ${source} as locate when Input`,\n              latestLocate,\n            );\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                  locate: latestLocate,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n            lastLocateRecorder.recordLocate(locate, 'Long Press');\n\n            return [\n              {\n                type: 'AndroidLongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: 'AndroidBackButton',\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: 'AndroidHomeButton',\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: launchAction.app,\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["debug","getDebug","lastLocateRecorder","LatestLocateRecorder","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","autoGLMCoordinateToBbox","locate","doubleTapAction","typeAction","latestLocate","source","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","AUTO_GLM_COORDINATE_MAX","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;;AAKA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,qBAAqB,IAAIC;AAwGxB,SAASC,uBACdC,MAAoB,EACpBC,IAAuC;IAEvC,IAAI;QACF,OAAQD,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBL,MAAM,4BAA4BO;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBR,MAAM,yBAAyBS;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,wBACvBL,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCACAX,mBAAmB,YAAY,CAACa,QAAQ;gCAExC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLA;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBR;gCACxBR,MAAM,gCAAgCgB;gCACtC,MAAM,CAACN,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,wBACvBE,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BV,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCACAX,mBAAmB,YAAY,CAACa,QAAQ;gCAExC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLA;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaT;gCACnBR,MAAM,0BAA0BiB;gCAChC,MAAM,EAAE,QAAQC,YAAY,EAAEC,MAAM,EAAE,GACpCjB,mBAAmB,eAAe;gCACpCF,MACE,CAAC,sBAAsB,EAAEmB,OAAO,qBAAqB,CAAC,EACtDD;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOD,WAAW,IAAI;4CACtB,QAAQC;wCACV;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAME,cAAcZ;gCACpBR,MAAM,2BAA2BoB;gCAGjC,MAAM,CAACV,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,wBACvBM,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBd,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMQ,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYrB,KAAK,MAAM,GAAIsB;oCAE9BL,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYnB,KAAK,KAAK,GAAIsB;oCAE7BL,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEArB,MACE,CAAC,2BAA2B,EAAEuB,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLT;4CAEAS;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMS,kBAAkBrB;gCACxBR,MAAM,gCAAgC6B;gCACtC,MAAM,CAACnB,IAAIC,IAAIC,IAAIC,GAAG,GAAGC,wBACvBe,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BvB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMS,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACL;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCACAX,mBAAmB,YAAY,CAACa,QAAQ;gCAExC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLA;wCACF;wCACA,SAASc,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAatB;gCACnBR,MAAM,0BAA0B8B;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO,CAAC;wCACR,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAavB;gCACnBR,MAAM,0BAA0B+B;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO,CAAC;wCACR,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaxB;gCACnBR,MAAM,0BAA0BgC;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAezB;gCACrBR,MAAM,4BAA4BiC;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAOA,aAAa,GAAG;wCACvB,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAG1B,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAI0B,MACR,CAAC,yBAAyB,EAAG7B,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO8B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrEnC,MAAM,oBAAoBoC;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/es/ai-model/auto-glm/index.mjs

@@ -0,0 +1,6 @@
+import { getAutoGLMLocatePrompt, getAutoGLMPlanPrompt } from "./prompt.mjs";
+import { parseAction, parseAutoGLMLocateResponse, parseAutoGLMResponse } from "./parser.mjs";
+import { autoGLMPlanning } from "./planning.mjs";
+import { transformAutoGLMAction } from "./actions.mjs";
+import { autoGLMCoordinateToBbox, isAutoGLM } from "./util.mjs";
+export { autoGLMCoordinateToBbox, autoGLMPlanning, getAutoGLMLocatePrompt, getAutoGLMPlanPrompt, isAutoGLM, parseAction, parseAutoGLMLocateResponse, parseAutoGLMResponse, transformAutoGLMAction };

package/dist/es/ai-model/auto-glm/parser.mjs

@@ -0,0 +1,239 @@
+import { getDebug } from "@midscene/shared/logger";
+const debug = getDebug('auto-glm-parser');
+const extractValueAfter = (src, key)=>{
+    const idx = src.indexOf(key);
+    if (-1 === idx) throw new Error(`Missing key ${key} in action payload ${src}`);
+    let rest = src.slice(idx + key.length).trim();
+    if (rest.endsWith('")')) rest = rest.slice(0, -2);
+    return rest;
+};
+function parseAction(response) {
+    debug('Parsing action:', response);
+    let trimmedResponse = '';
+    try {
+        trimmedResponse = response.content.trim();
+        if (trimmedResponse.startsWith('do(action="Type"') || trimmedResponse.startsWith('do(action="Type_Name"')) {
+            const text = extractValueAfter(trimmedResponse, 'text="');
+            return {
+                _metadata: 'do',
+                action: 'Type',
+                text,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('finish(message=')) {
+            let message = extractValueAfter(trimmedResponse, 'finish(message="');
+            if (message.endsWith(')')) message = message.slice(0, -1);
+            return {
+                _metadata: 'finish',
+                message,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('do(')) {
+            const actionMatch = trimmedResponse.match(/do\(action="([^"]+)"/);
+            if (!actionMatch) throw new Error(`Failed to extract action type from do() call; raw="${trimmedResponse}"`);
+            const actionType = actionMatch[1];
+            const baseAction = {
+                _metadata: 'do',
+                think: response.think
+            };
+            switch(actionType){
+                case 'Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Double Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Double Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Double Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Swipe':
+                    {
+                        const startMatch = trimmedResponse.match(/start=\[(\d+),(\d+)\]/);
+                        const endMatch = trimmedResponse.match(/end=\[(\d+),(\d+)\]/);
+                        if (!startMatch || !endMatch) throw new Error(`Failed to extract start/end coordinates for Swipe; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Swipe',
+                            start: [
+                                Number(startMatch[1]),
+                                Number(startMatch[2])
+                            ],
+                            end: [
+                                Number(endMatch[1]),
+                                Number(endMatch[2])
+                            ]
+                        };
+                    }
+                case 'Long Press':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Long Press; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Long Press',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Launch':
+                    {
+                        const app = extractValueAfter(trimmedResponse, 'app="');
+                        return {
+                            ...baseAction,
+                            action: 'Launch',
+                            app
+                        };
+                    }
+                case 'Back':
+                    return {
+                        ...baseAction,
+                        action: 'Back'
+                    };
+                case 'Home':
+                    return {
+                        ...baseAction,
+                        action: 'Home'
+                    };
+                case 'Wait':
+                    {
+                        const durationMatch = trimmedResponse.match(/duration=(?:["\[])?(\d+)/);
+                        if (!durationMatch) throw new Error(`Failed to extract duration for Wait; raw="${trimmedResponse}"`);
+                        const seconds = Number.parseInt(durationMatch[1], 10);
+                        const durationMs = 1000 * seconds;
+                        return {
+                            ...baseAction,
+                            action: 'Wait',
+                            durationMs
+                        };
+                    }
+                case 'Interact':
+                    return {
+                        ...baseAction,
+                        action: 'Interact'
+                    };
+                case 'Call_API':
+                    {
+                        const instruction = extractValueAfter(trimmedResponse, 'instruction="');
+                        return {
+                            ...baseAction,
+                            action: 'Call_API',
+                            instruction
+                        };
+                    }
+                case 'Take_over':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Take_over',
+                            message
+                        };
+                    }
+                case 'Note':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Note',
+                            message
+                        };
+                    }
+                default:
+                    throw new Error(`Unknown action type: ${actionType}; raw="${trimmedResponse}"`);
+            }
+        }
+        throw new Error(`Failed to parse action: ${trimmedResponse}`);
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to parse action: ${errorMessage}; raw="${trimmedResponse}"`);
+    }
+}
+function parseAutoGLMResponse(content) {
+    if (content.includes('finish(message=')) {
+        const parts = content.split('finish(message=');
+        const think = parts[0].trim();
+        const actionContent = `finish(message=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('do(action=')) {
+        const parts = content.split('do(action=');
+        const think = parts[0].trim();
+        const actionContent = `do(action=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('<answer>')) {
+        const parts = content.split('<answer>');
+        const think = parts[0].replace(/<think>/g, '').replace(/<\/think>/g, '').trim();
+        const actionContent = parts[1].replace(/<\/answer>/g, '').trim();
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    return {
+        think: '',
+        content
+    };
+}
+function parseAutoGLMLocateResponse(rawResponse) {
+    const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);
+    if (!actionContent.startsWith('do(action="Tap"')) return {
+        think,
+        coordinates: null,
+        error: `Unexpected action type in auto-glm locate response: ${actionContent}`
+    };
+    try {
+        const elementMatch = actionContent.match(/element=\[(\d+),(\d+)\]/);
+        if (!elementMatch) return {
+            think,
+            coordinates: null,
+            error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`
+        };
+        const x = Number(elementMatch[1]);
+        const y = Number(elementMatch[2]);
+        return {
+            think,
+            coordinates: {
+                x,
+                y
+            }
+        };
+    } catch (e) {
+        const errorMessage = e instanceof Error ? e.message : String(e);
+        return {
+            think,
+            coordinates: null,
+            error: `Failed to parse coordinates "${actionContent}" with errorMessage: ${errorMessage}`
+        };
+    }
+}
+export { extractValueAfter, parseAction, parseAutoGLMLocateResponse, parseAutoGLMResponse };
+
+//# sourceMappingURL=parser.mjs.map

package/dist/es/ai-model/auto-glm/parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/parser.mjs","sources":["../../../../src/ai-model/auto-glm/parser.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport type { ParsedAction } from './actions';\n\nconst debug = getDebug('auto-glm-parser');\n\n// Do not rely on regex extraction here; regex can fail on malformed input.\n// Bad Case: finish(message=\"Finished! Now There is a contact whose name is \\\"Tom\\\" in the list.\")\nexport const extractValueAfter = (src: string, key: string): string => {\n  const idx = src.indexOf(key);\n  if (idx === -1) {\n    throw new Error(`Missing key ${key} in action payload ${src}`);\n  }\n  let rest = src.slice(idx + key.length).trim();\n  if (rest.endsWith('\")')) {\n    rest = rest.slice(0, -2);\n  }\n  return rest;\n};\n\nexport function parseAction(response: {\n  think: string;\n  content: string;\n}): ParsedAction {\n  debug('Parsing action:', response);\n  let trimmedResponse = '';\n  try {\n    trimmedResponse = response.content.trim();\n\n    if (\n      trimmedResponse.startsWith('do(action=\"Type\"') ||\n      trimmedResponse.startsWith('do(action=\"Type_Name\"')\n    ) {\n      const text = extractValueAfter(trimmedResponse, 'text=\"');\n      return {\n        _metadata: 'do',\n        action: 'Type',\n        text,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('finish(message=')) {\n      let message = extractValueAfter(trimmedResponse, 'finish(message=\"');\n      if (message.endsWith(')')) message = message.slice(0, -1);\n      return {\n        _metadata: 'finish',\n        message,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('do(')) {\n      const actionMatch = trimmedResponse.match(/do\\(action=\"([^\"]+)\"/);\n      if (!actionMatch)\n        throw new Error(\n          `Failed to extract action type from do() call; raw=\"${trimmedResponse}\"`,\n        );\n      const actionType = actionMatch[1];\n\n      const baseAction = { _metadata: 'do' as const, think: response.think };\n      switch (actionType) {\n        case 'Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Double Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Double Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Double Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Swipe': {\n          const startMatch = trimmedResponse.match(/start=\\[(\\d+),(\\d+)\\]/);\n          const endMatch = trimmedResponse.match(/end=\\[(\\d+),(\\d+)\\]/);\n          if (!startMatch || !endMatch)\n            throw new Error(\n              `Failed to extract start/end coordinates for Swipe; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Swipe',\n            start: [Number(startMatch[1]), Number(startMatch[2])],\n            end: [Number(endMatch[1]), Number(endMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Long Press': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Long Press; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Long Press',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Launch': {\n          const app = extractValueAfter(trimmedResponse, 'app=\"');\n          return { ...baseAction, action: 'Launch', app } as ParsedAction;\n        }\n        case 'Back': {\n          return { ...baseAction, action: 'Back' } as ParsedAction;\n        }\n        case 'Home': {\n          return { ...baseAction, action: 'Home' } as ParsedAction;\n        }\n        case 'Wait': {\n          const durationMatch = trimmedResponse.match(\n            /duration=(?:[\"\\[])?(\\d+)/,\n          );\n          if (!durationMatch) {\n            throw new Error(\n              `Failed to extract duration for Wait; raw=\"${trimmedResponse}\"`,\n            );\n          }\n          const seconds = Number.parseInt(durationMatch[1], 10);\n          const durationMs = seconds * 1000;\n          return {\n            ...baseAction,\n            action: 'Wait',\n            durationMs,\n          } as ParsedAction;\n        }\n        case 'Interact': {\n          return { ...baseAction, action: 'Interact' } as ParsedAction;\n        }\n        case 'Call_API': {\n          const instruction = extractValueAfter(\n            trimmedResponse,\n            'instruction=\"',\n          );\n          return {\n            ...baseAction,\n            action: 'Call_API',\n            instruction,\n          } as ParsedAction;\n        }\n        case 'Take_over': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Take_over',\n            message,\n          } as ParsedAction;\n        }\n        case 'Note': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Note',\n            message,\n          } as ParsedAction;\n        }\n        default:\n          throw new Error(\n            `Unknown action type: ${actionType}; raw=\"${trimmedResponse}\"`,\n          );\n      }\n    }\n    throw new Error(`Failed to parse action: ${trimmedResponse}`);\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    throw new Error(\n      `Failed to parse action: ${errorMessage}; raw=\"${trimmedResponse}\"`,\n    );\n  }\n}\n\nexport function parseAutoGLMResponse(content: string): {\n  think: string;\n  content: string;\n} {\n  if (content.includes('finish(message=')) {\n    const parts = content.split('finish(message=');\n    const think = parts[0].trim();\n    const actionContent = `finish(message=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('do(action=')) {\n    const parts = content.split('do(action=');\n    const think = parts[0].trim();\n    const actionContent = `do(action=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('<answer>')) {\n    const parts = content.split('<answer>');\n    const think = parts[0]\n      .replace(/<think>/g, '')\n      .replace(/<\\/think>/g, '')\n      .trim();\n    const actionContent = parts[1].replace(/<\\/answer>/g, '').trim();\n    return { think, content: actionContent };\n  }\n  return { think: '', content };\n}\n\nexport function parseAutoGLMLocateResponse(rawResponse: string): {\n  think: string;\n  coordinates: { x: number; y: number } | null;\n  error?: string;\n} {\n  const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);\n  if (!actionContent.startsWith('do(action=\"Tap\"')) {\n    return {\n      think,\n      coordinates: null,\n      error: `Unexpected action type in auto-glm locate response: ${actionContent}`,\n    };\n  }\n  try {\n    const elementMatch = actionContent.match(/element=\\[(\\d+),(\\d+)\\]/);\n    if (!elementMatch) {\n      return {\n        think,\n        coordinates: null,\n        error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`,\n      };\n    }\n    const x = Number(elementMatch[1]);\n    const y = Number(elementMatch[2]);\n    return { think, coordinates: { x, y } };\n  } catch (e) {\n    const errorMessage = e instanceof Error ? e.message : String(e);\n    return {\n      think,\n      coordinates: null,\n      error: `Failed to parse coordinates \"${actionContent}\" with errorMessage: ${errorMessage}`,\n    };\n  }\n}\n"],"names":["debug","getDebug","extractValueAfter","src","key","idx","Error","rest","parseAction","response","trimmedResponse","text","message","actionMatch","actionType","baseAction","elementMatch","Number","startMatch","endMatch","app","durationMatch","seconds","durationMs","instruction","error","errorMessage","String","parseAutoGLMResponse","content","parts","think","actionContent","parseAutoGLMLocateResponse","rawResponse","x","y","e"],"mappings":";AAGA,MAAMA,QAAQC,SAAS;AAIhB,MAAMC,oBAAoB,CAACC,KAAaC;IAC7C,MAAMC,MAAMF,IAAI,OAAO,CAACC;IACxB,IAAIC,AAAQ,OAARA,KACF,MAAM,IAAIC,MAAM,CAAC,YAAY,EAAEF,IAAI,mBAAmB,EAAED,KAAK;IAE/D,IAAII,OAAOJ,IAAI,KAAK,CAACE,MAAMD,IAAI,MAAM,EAAE,IAAI;IAC3C,IAAIG,KAAK,QAAQ,CAAC,OAChBA,OAAOA,KAAK,KAAK,CAAC,GAAG;IAEvB,OAAOA;AACT;AAEO,SAASC,YAAYC,QAG3B;IACCT,MAAM,mBAAmBS;IACzB,IAAIC,kBAAkB;IACtB,IAAI;QACFA,kBAAkBD,SAAS,OAAO,CAAC,IAAI;QAEvC,IACEC,gBAAgB,UAAU,CAAC,uBAC3BA,gBAAgB,UAAU,CAAC,0BAC3B;YACA,MAAMC,OAAOT,kBAAkBQ,iBAAiB;YAChD,OAAO;gBACL,WAAW;gBACX,QAAQ;gBACRC;gBACA,OAAOF,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,oBAAoB;YACjD,IAAIE,UAAUV,kBAAkBQ,iBAAiB;YACjD,IAAIE,QAAQ,QAAQ,CAAC,MAAMA,UAAUA,QAAQ,KAAK,CAAC,GAAG;YACtD,OAAO;gBACL,WAAW;gBACXA;gBACA,OAAOH,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,QAAQ;YACrC,MAAMG,cAAcH,gBAAgB,KAAK,CAAC;YAC1C,IAAI,CAACG,aACH,MAAM,IAAIP,MACR,CAAC,mDAAmD,EAAEI,gBAAgB,CAAC,CAAC;YAE5E,MAAMI,aAAaD,WAAW,CAAC,EAAE;YAEjC,MAAME,aAAa;gBAAE,WAAW;gBAAe,OAAON,SAAS,KAAK;YAAC;YACrE,OAAQK;gBACN,KAAK;oBAAO;wBACV,MAAME,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,oDAAoD,EAAEI,gBAAgB,CAAC,CAAC;wBAE7E,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMA,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAS;wBACZ,MAAME,aAAaR,gBAAgB,KAAK,CAAC;wBACzC,MAAMS,WAAWT,gBAAgB,KAAK,CAAC;wBACvC,IAAI,CAACQ,cAAc,CAACC,UAClB,MAAM,IAAIb,MACR,CAAC,wDAAwD,EAAEI,gBAAgB,CAAC,CAAC;wBAEjF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,OAAO;gCAACE,OAAOC,UAAU,CAAC,EAAE;gCAAGD,OAAOC,UAAU,CAAC,EAAE;6BAAE;4BACrD,KAAK;gCAACD,OAAOE,QAAQ,CAAC,EAAE;gCAAGF,OAAOE,QAAQ,CAAC,EAAE;6BAAE;wBACjD;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMH,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAU;wBACb,MAAMI,MAAMlB,kBAAkBQ,iBAAiB;wBAC/C,OAAO;4BAAE,GAAGK,UAAU;4BAAE,QAAQ;4BAAUK;wBAAI;oBAChD;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGL,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBACH,OAAO;wBAAE,GAAGA,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBAAQ;wBACX,MAAMM,gBAAgBX,gBAAgB,KAAK,CACzC;wBAEF,IAAI,CAACW,eACH,MAAM,IAAIf,MACR,CAAC,0CAA0C,EAAEI,gBAAgB,CAAC,CAAC;wBAGnE,MAAMY,UAAUL,OAAO,QAAQ,CAACI,aAAa,CAAC,EAAE,EAAE;wBAClD,MAAME,aAAaD,AAAU,OAAVA;wBACnB,OAAO;4BACL,GAAGP,UAAU;4BACb,QAAQ;4BACRQ;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGR,UAAU;wBAAE,QAAQ;oBAAW;gBAE7C,KAAK;oBAAY;wBACf,MAAMS,cAActB,kBAClBQ,iBACA;wBAEF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRS;wBACF;oBACF;gBACA,KAAK;oBAAa;wBAChB,MAAMZ,UAAUV,kBAAkBQ,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA,KAAK;oBAAQ;wBACX,MAAMA,UAAUV,kBAAkBQ,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA;oBACE,MAAM,IAAIN,MACR,CAAC,qBAAqB,EAAEQ,WAAW,OAAO,EAAEJ,gBAAgB,CAAC,CAAC;YAEpE;QACF;QACA,MAAM,IAAIJ,MAAM,CAAC,wBAAwB,EAAEI,iBAAiB;IAC9D,EAAE,OAAOe,OAAO;QACd,MAAMC,eAAeD,iBAAiBnB,QAAQmB,MAAM,OAAO,GAAGE,OAAOF;QACrE,MAAM,IAAInB,MACR,CAAC,wBAAwB,EAAEoB,aAAa,OAAO,EAAEhB,gBAAgB,CAAC,CAAC;IAEvE;AACF;AAEO,SAASkB,qBAAqBC,OAAe;IAIlD,IAAIA,QAAQ,QAAQ,CAAC,oBAAoB;QACvC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,eAAe,EAAEF,KAAK,CAAC,EAAE,EAAE;QAClD,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,eAAe;QAClC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,UAAU,EAAEF,KAAK,CAAC,EAAE,EAAE;QAC7C,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,aAAa;QAChC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CACnB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,cAAc,IACtB,IAAI;QACP,MAAME,gBAAgBF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI;QAC9D,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,OAAO;QAAE,OAAO;QAAIH;IAAQ;AAC9B;AAEO,SAASI,2BAA2BC,WAAmB;IAK5D,MAAM,EAAEH,KAAK,EAAE,SAASC,aAAa,EAAE,GAAGJ,qBAAqBM;IAC/D,IAAI,CAACF,cAAc,UAAU,CAAC,oBAC5B,OAAO;QACLD;QACA,aAAa;QACb,OAAO,CAAC,oDAAoD,EAAEC,eAAe;IAC/E;IAEF,IAAI;QACF,MAAMhB,eAAegB,cAAc,KAAK,CAAC;QACzC,IAAI,CAAChB,cACH,OAAO;YACLe;YACA,aAAa;YACb,OAAO,CAAC,8DAA8D,EAAEC,eAAe;QACzF;QAEF,MAAMG,IAAIlB,OAAOD,YAAY,CAAC,EAAE;QAChC,MAAMoB,IAAInB,OAAOD,YAAY,CAAC,EAAE;QAChC,OAAO;YAAEe;YAAO,aAAa;gBAAEI;gBAAGC;YAAE;QAAE;IACxC,EAAE,OAAOC,GAAG;QACV,MAAMX,eAAeW,aAAa/B,QAAQ+B,EAAE,OAAO,GAAGV,OAAOU;QAC7D,OAAO;YACLN;YACA,aAAa;YACb,OAAO,CAAC,6BAA6B,EAAEC,cAAc,qBAAqB,EAAEN,cAAc;QAC5F;IACF;AACF"}

package/dist/es/ai-model/auto-glm/planning.mjs

@@ -0,0 +1,63 @@
+import { getDebug } from "@midscene/shared/logger";
+import { callAIWithStringResponse } from "../service-caller/index.mjs";
+import { transformAutoGLMAction } from "./actions.mjs";
+import { parseAction, parseAutoGLMResponse } from "./parser.mjs";
+import { getAutoGLMPlanPrompt } from "./prompt.mjs";
+const debug = getDebug('auto-glm-planning');
+async function autoGLMPlanning(userInstruction, options) {
+    const { conversationHistory, context, modelConfig, actionContext } = options;
+    const systemPrompt = getAutoGLMPlanPrompt(modelConfig.vlMode) + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>` : '');
+    const imagePayload = context.screenshotBase64;
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'text',
+                text: userInstruction
+            }
+        ]
+    });
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload
+                }
+            }
+        ]
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        ...conversationHistory.snapshot(1)
+    ];
+    const { content: rawResponse, usage } = await callAIWithStringResponse(msgs, modelConfig);
+    debug('autoGLMPlanning rawResponse:', rawResponse);
+    const parsedResponse = parseAutoGLMResponse(rawResponse);
+    debug('thinking in response:', parsedResponse.think);
+    debug('action in response:', parsedResponse.content);
+    let transformedActions = [];
+    const parsedAction = parseAction(parsedResponse);
+    debug('Parsed action object:', parsedAction);
+    transformedActions = transformAutoGLMAction(parsedAction, context.size);
+    debug('Transformed actions:', transformedActions);
+    conversationHistory.append({
+        role: 'assistant',
+        content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`
+    });
+    const shouldContinue = !parsedResponse.content.startsWith('finish(');
+    return {
+        actions: transformedActions,
+        log: rawResponse,
+        usage,
+        more_actions_needed_by_instruction: shouldContinue,
+        rawResponse: JSON.stringify(rawResponse, void 0, 2)
+    };
+}
+export { autoGLMPlanning };
+
+//# sourceMappingURL=planning.mjs.map

package/dist/es/ai-model/auto-glm/planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/planning.mjs","sources":["../../../../src/ai-model/auto-glm/planning.ts"],"sourcesContent":["import type { PlanningAIResponse, UIContext } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ConversationHistory } from '../conversation-history';\nimport { callAIWithStringResponse } from '../service-caller/index';\nimport { transformAutoGLMAction } from './actions';\nimport { parseAction, parseAutoGLMResponse } from './parser';\nimport { getAutoGLMPlanPrompt } from './prompt';\n\nconst debug = getDebug('auto-glm-planning');\n\nexport async function autoGLMPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n\n  const systemPrompt =\n    getAutoGLMPlanPrompt(modelConfig.vlMode) +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>`\n      : '');\n\n  const imagePayload = context.screenshotBase64;\n\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'text', text: userInstruction }],\n  });\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'image_url', image_url: { url: imagePayload } }],\n  });\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...conversationHistory.snapshot(1),\n  ];\n\n  const { content: rawResponse, usage } = await callAIWithStringResponse(\n    msgs,\n    modelConfig,\n  );\n\n  debug('autoGLMPlanning rawResponse:', rawResponse);\n\n  const parsedResponse = parseAutoGLMResponse(rawResponse);\n  debug('thinking in response:', parsedResponse.think);\n  debug('action in response:', parsedResponse.content);\n\n  let transformedActions = [] as ReturnType<typeof transformAutoGLMAction>;\n\n  const parsedAction = parseAction(parsedResponse);\n  debug('Parsed action object:', parsedAction);\n  transformedActions = transformAutoGLMAction(parsedAction, context.size);\n  debug('Transformed actions:', transformedActions);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`,\n  });\n\n  const shouldContinue = !parsedResponse.content.startsWith('finish(');\n\n  return {\n    actions: transformedActions,\n    log: rawResponse,\n    usage,\n    more_actions_needed_by_instruction: shouldContinue,\n    rawResponse: JSON.stringify(rawResponse, undefined, 2),\n  };\n}\n"],"names":["debug","getDebug","autoGLMPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","systemPrompt","getAutoGLMPlanPrompt","imagePayload","msgs","rawResponse","usage","callAIWithStringResponse","parsedResponse","parseAutoGLMResponse","transformedActions","parsedAction","parseAction","transformAutoGLMAction","shouldContinue","JSON","undefined"],"mappings":";;;;;AAUA,MAAMA,QAAQC,SAAS;AAEhB,eAAeC,gBACpBC,eAAuB,EACvBC,OAKC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IAErE,MAAMK,eACJC,qBAAqBH,YAAY,MAAM,IACtCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,0BAA0B,CAAC,GACrE,EAAC;IAEP,MAAMG,eAAeL,QAAQ,gBAAgB;IAE7CD,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAQ,MAAMF;YAAgB;SAAE;IACpD;IACAE,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKM;gBAAa;YAAE;SAAE;IACpE;IAEA,MAAMC,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASH;QAAa;WACrCJ,oBAAoB,QAAQ,CAAC;KACjC;IAED,MAAM,EAAE,SAASQ,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,yBAC5CH,MACAL;IAGFP,MAAM,gCAAgCa;IAEtC,MAAMG,iBAAiBC,qBAAqBJ;IAC5Cb,MAAM,yBAAyBgB,eAAe,KAAK;IACnDhB,MAAM,uBAAuBgB,eAAe,OAAO;IAEnD,IAAIE,qBAAqB,EAAE;IAE3B,MAAMC,eAAeC,YAAYJ;IACjChB,MAAM,yBAAyBmB;IAC/BD,qBAAqBG,uBAAuBF,cAAcb,QAAQ,IAAI;IACtEN,MAAM,wBAAwBkB;IAE9Bb,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS,CAAC,OAAO,EAAEW,eAAe,KAAK,CAAC,gBAAgB,EAAEA,eAAe,OAAO,CAAC,SAAS,CAAC;IAC7F;IAEA,MAAMM,iBAAiB,CAACN,eAAe,OAAO,CAAC,UAAU,CAAC;IAE1D,OAAO;QACL,SAASE;QACT,KAAKL;QACLC;QACA,oCAAoCQ;QACpC,aAAaC,KAAK,SAAS,CAACV,aAAaW,QAAW;IACtD;AACF"}