@midscene/core 1.2.1-beta-20260113073450.0 → 1.2.1-beta-20260114072539.0

package/dist/lib/ai-model/auto-glm/parser.js

@@ -0,0 +1,282 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    extractValueAfter: ()=>extractValueAfter,
+    parseAction: ()=>parseAction,
+    parseAutoGLMLocateResponse: ()=>parseAutoGLMLocateResponse,
+    parseAutoGLMResponse: ()=>parseAutoGLMResponse
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const debug = (0, logger_namespaceObject.getDebug)('auto-glm-parser');
+const extractValueAfter = (src, key)=>{
+    const idx = src.indexOf(key);
+    if (-1 === idx) throw new Error(`Missing key ${key} in action payload ${src}`);
+    let rest = src.slice(idx + key.length).trim();
+    if (rest.endsWith('")')) rest = rest.slice(0, -2);
+    return rest;
+};
+function parseAction(response) {
+    debug('Parsing action:', response);
+    let trimmedResponse = '';
+    try {
+        trimmedResponse = response.content.trim();
+        if (trimmedResponse.startsWith('do(action="Type"') || trimmedResponse.startsWith('do(action="Type_Name"')) {
+            const text = extractValueAfter(trimmedResponse, 'text="');
+            return {
+                _metadata: 'do',
+                action: 'Type',
+                text,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('finish(message=')) {
+            let message = extractValueAfter(trimmedResponse, 'finish(message="');
+            if (message.endsWith(')')) message = message.slice(0, -1);
+            return {
+                _metadata: 'finish',
+                message,
+                think: response.think
+            };
+        }
+        if (trimmedResponse.startsWith('do(')) {
+            const actionMatch = trimmedResponse.match(/do\(action="([^"]+)"/);
+            if (!actionMatch) throw new Error(`Failed to extract action type from do() call; raw="${trimmedResponse}"`);
+            const actionType = actionMatch[1];
+            const baseAction = {
+                _metadata: 'do',
+                think: response.think
+            };
+            switch(actionType){
+                case 'Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Double Tap':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Double Tap; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Double Tap',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Swipe':
+                    {
+                        const startMatch = trimmedResponse.match(/start=\[(\d+),(\d+)\]/);
+                        const endMatch = trimmedResponse.match(/end=\[(\d+),(\d+)\]/);
+                        if (!startMatch || !endMatch) throw new Error(`Failed to extract start/end coordinates for Swipe; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Swipe',
+                            start: [
+                                Number(startMatch[1]),
+                                Number(startMatch[2])
+                            ],
+                            end: [
+                                Number(endMatch[1]),
+                                Number(endMatch[2])
+                            ]
+                        };
+                    }
+                case 'Long Press':
+                    {
+                        const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
+                        if (!elementMatch) throw new Error(`Failed to extract element coordinates for Long Press; raw="${trimmedResponse}"`);
+                        return {
+                            ...baseAction,
+                            action: 'Long Press',
+                            element: [
+                                Number(elementMatch[1]),
+                                Number(elementMatch[2])
+                            ]
+                        };
+                    }
+                case 'Launch':
+                    {
+                        const app = extractValueAfter(trimmedResponse, 'app="');
+                        return {
+                            ...baseAction,
+                            action: 'Launch',
+                            app
+                        };
+                    }
+                case 'Back':
+                    return {
+                        ...baseAction,
+                        action: 'Back'
+                    };
+                case 'Home':
+                    return {
+                        ...baseAction,
+                        action: 'Home'
+                    };
+                case 'Wait':
+                    {
+                        const durationMatch = trimmedResponse.match(/duration=(?:["\[])?(\d+)/);
+                        if (!durationMatch) throw new Error(`Failed to extract duration for Wait; raw="${trimmedResponse}"`);
+                        const seconds = Number.parseInt(durationMatch[1], 10);
+                        const durationMs = 1000 * seconds;
+                        return {
+                            ...baseAction,
+                            action: 'Wait',
+                            durationMs
+                        };
+                    }
+                case 'Interact':
+                    return {
+                        ...baseAction,
+                        action: 'Interact'
+                    };
+                case 'Call_API':
+                    {
+                        const instruction = extractValueAfter(trimmedResponse, 'instruction="');
+                        return {
+                            ...baseAction,
+                            action: 'Call_API',
+                            instruction
+                        };
+                    }
+                case 'Take_over':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Take_over',
+                            message
+                        };
+                    }
+                case 'Note':
+                    {
+                        const message = extractValueAfter(trimmedResponse, 'message="');
+                        return {
+                            ...baseAction,
+                            action: 'Note',
+                            message
+                        };
+                    }
+                default:
+                    throw new Error(`Unknown action type: ${actionType}; raw="${trimmedResponse}"`);
+            }
+        }
+        throw new Error(`Failed to parse action: ${trimmedResponse}`);
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to parse action: ${errorMessage}; raw="${trimmedResponse}"`);
+    }
+}
+function parseAutoGLMResponse(content) {
+    if (content.includes('finish(message=')) {
+        const parts = content.split('finish(message=');
+        const think = parts[0].trim();
+        const actionContent = `finish(message=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('do(action=')) {
+        const parts = content.split('do(action=');
+        const think = parts[0].trim();
+        const actionContent = `do(action=${parts[1]}`;
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    if (content.includes('<answer>')) {
+        const parts = content.split('<answer>');
+        const think = parts[0].replace(/<think>/g, '').replace(/<\/think>/g, '').trim();
+        const actionContent = parts[1].replace(/<\/answer>/g, '').trim();
+        return {
+            think,
+            content: actionContent
+        };
+    }
+    return {
+        think: '',
+        content
+    };
+}
+function parseAutoGLMLocateResponse(rawResponse) {
+    const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);
+    if (!actionContent.startsWith('do(action="Tap"')) return {
+        think,
+        coordinates: null,
+        error: `Unexpected action type in auto-glm locate response: ${actionContent}`
+    };
+    try {
+        const elementMatch = actionContent.match(/element=\[(\d+),(\d+)\]/);
+        if (!elementMatch) return {
+            think,
+            coordinates: null,
+            error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`
+        };
+        const x = Number(elementMatch[1]);
+        const y = Number(elementMatch[2]);
+        return {
+            think,
+            coordinates: {
+                x,
+                y
+            }
+        };
+    } catch (e) {
+        const errorMessage = e instanceof Error ? e.message : String(e);
+        return {
+            think,
+            coordinates: null,
+            error: `Failed to parse coordinates "${actionContent}" with errorMessage: ${errorMessage}`
+        };
+    }
+}
+exports.extractValueAfter = __webpack_exports__.extractValueAfter;
+exports.parseAction = __webpack_exports__.parseAction;
+exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
+exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "extractValueAfter",
+    "parseAction",
+    "parseAutoGLMLocateResponse",
+    "parseAutoGLMResponse"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=parser.js.map

package/dist/lib/ai-model/auto-glm/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/parser.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/parser.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@midscene/shared/logger';\nimport type { ParsedAction } from './actions';\n\nconst debug = getDebug('auto-glm-parser');\n\n// Do not rely on regex extraction here; regex can fail on malformed input.\n// Bad Case: finish(message=\"Finished! Now There is a contact whose name is \\\"Tom\\\" in the list.\")\nexport const extractValueAfter = (src: string, key: string): string => {\n  const idx = src.indexOf(key);\n  if (idx === -1) {\n    throw new Error(`Missing key ${key} in action payload ${src}`);\n  }\n  let rest = src.slice(idx + key.length).trim();\n  if (rest.endsWith('\")')) {\n    rest = rest.slice(0, -2);\n  }\n  return rest;\n};\n\nexport function parseAction(response: {\n  think: string;\n  content: string;\n}): ParsedAction {\n  debug('Parsing action:', response);\n  let trimmedResponse = '';\n  try {\n    trimmedResponse = response.content.trim();\n\n    if (\n      trimmedResponse.startsWith('do(action=\"Type\"') ||\n      trimmedResponse.startsWith('do(action=\"Type_Name\"')\n    ) {\n      const text = extractValueAfter(trimmedResponse, 'text=\"');\n      return {\n        _metadata: 'do',\n        action: 'Type',\n        text,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('finish(message=')) {\n      let message = extractValueAfter(trimmedResponse, 'finish(message=\"');\n      if (message.endsWith(')')) message = message.slice(0, -1);\n      return {\n        _metadata: 'finish',\n        message,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('do(')) {\n      const actionMatch = trimmedResponse.match(/do\\(action=\"([^\"]+)\"/);\n      if (!actionMatch)\n        throw new Error(\n          `Failed to extract action type from do() call; raw=\"${trimmedResponse}\"`,\n        );\n      const actionType = actionMatch[1];\n\n      const baseAction = { _metadata: 'do' as const, think: response.think };\n      switch (actionType) {\n        case 'Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Double Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Double Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Double Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Swipe': {\n          const startMatch = trimmedResponse.match(/start=\\[(\\d+),(\\d+)\\]/);\n          const endMatch = trimmedResponse.match(/end=\\[(\\d+),(\\d+)\\]/);\n          if (!startMatch || !endMatch)\n            throw new Error(\n              `Failed to extract start/end coordinates for Swipe; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Swipe',\n            start: [Number(startMatch[1]), Number(startMatch[2])],\n            end: [Number(endMatch[1]), Number(endMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Long Press': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Long Press; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Long Press',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Launch': {\n          const app = extractValueAfter(trimmedResponse, 'app=\"');\n          return { ...baseAction, action: 'Launch', app } as ParsedAction;\n        }\n        case 'Back': {\n          return { ...baseAction, action: 'Back' } as ParsedAction;\n        }\n        case 'Home': {\n          return { ...baseAction, action: 'Home' } as ParsedAction;\n        }\n        case 'Wait': {\n          const durationMatch = trimmedResponse.match(\n            /duration=(?:[\"\\[])?(\\d+)/,\n          );\n          if (!durationMatch) {\n            throw new Error(\n              `Failed to extract duration for Wait; raw=\"${trimmedResponse}\"`,\n            );\n          }\n          const seconds = Number.parseInt(durationMatch[1], 10);\n          const durationMs = seconds * 1000;\n          return {\n            ...baseAction,\n            action: 'Wait',\n            durationMs,\n          } as ParsedAction;\n        }\n        case 'Interact': {\n          return { ...baseAction, action: 'Interact' } as ParsedAction;\n        }\n        case 'Call_API': {\n          const instruction = extractValueAfter(\n            trimmedResponse,\n            'instruction=\"',\n          );\n          return {\n            ...baseAction,\n            action: 'Call_API',\n            instruction,\n          } as ParsedAction;\n        }\n        case 'Take_over': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Take_over',\n            message,\n          } as ParsedAction;\n        }\n        case 'Note': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Note',\n            message,\n          } as ParsedAction;\n        }\n        default:\n          throw new Error(\n            `Unknown action type: ${actionType}; raw=\"${trimmedResponse}\"`,\n          );\n      }\n    }\n    throw new Error(`Failed to parse action: ${trimmedResponse}`);\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    throw new Error(\n      `Failed to parse action: ${errorMessage}; raw=\"${trimmedResponse}\"`,\n    );\n  }\n}\n\nexport function parseAutoGLMResponse(content: string): {\n  think: string;\n  content: string;\n} {\n  if (content.includes('finish(message=')) {\n    const parts = content.split('finish(message=');\n    const think = parts[0].trim();\n    const actionContent = `finish(message=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('do(action=')) {\n    const parts = content.split('do(action=');\n    const think = parts[0].trim();\n    const actionContent = `do(action=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('<answer>')) {\n    const parts = content.split('<answer>');\n    const think = parts[0]\n      .replace(/<think>/g, '')\n      .replace(/<\\/think>/g, '')\n      .trim();\n    const actionContent = parts[1].replace(/<\\/answer>/g, '').trim();\n    return { think, content: actionContent };\n  }\n  return { think: '', content };\n}\n\nexport function parseAutoGLMLocateResponse(rawResponse: string): {\n  think: string;\n  coordinates: { x: number; y: number } | null;\n  error?: string;\n} {\n  const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);\n  if (!actionContent.startsWith('do(action=\"Tap\"')) {\n    return {\n      think,\n      coordinates: null,\n      error: `Unexpected action type in auto-glm locate response: ${actionContent}`,\n    };\n  }\n  try {\n    const elementMatch = actionContent.match(/element=\\[(\\d+),(\\d+)\\]/);\n    if (!elementMatch) {\n      return {\n        think,\n        coordinates: null,\n        error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`,\n      };\n    }\n    const x = Number(elementMatch[1]);\n    const y = Number(elementMatch[2]);\n    return { think, coordinates: { x, y } };\n  } catch (e) {\n    const errorMessage = e instanceof Error ? e.message : String(e);\n    return {\n      think,\n      coordinates: null,\n      error: `Failed to parse coordinates \"${actionContent}\" with errorMessage: ${errorMessage}`,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","extractValueAfter","src","idx","Error","rest","parseAction","response","trimmedResponse","text","message","actionMatch","actionType","baseAction","elementMatch","Number","startMatch","endMatch","app","durationMatch","seconds","durationMs","instruction","error","errorMessage","String","parseAutoGLMResponse","content","parts","think","actionContent","parseAutoGLMLocateResponse","rawResponse","x","y","e"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACHA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIhB,MAAMC,oBAAoB,CAACC,KAAaR;IAC7C,MAAMS,MAAMD,IAAI,OAAO,CAACR;IACxB,IAAIS,AAAQ,OAARA,KACF,MAAM,IAAIC,MAAM,CAAC,YAAY,EAAEV,IAAI,mBAAmB,EAAEQ,KAAK;IAE/D,IAAIG,OAAOH,IAAI,KAAK,CAACC,MAAMT,IAAI,MAAM,EAAE,IAAI;IAC3C,IAAIW,KAAK,QAAQ,CAAC,OAChBA,OAAOA,KAAK,KAAK,CAAC,GAAG;IAEvB,OAAOA;AACT;AAEO,SAASC,YAAYC,QAG3B;IACCR,MAAM,mBAAmBQ;IACzB,IAAIC,kBAAkB;IACtB,IAAI;QACFA,kBAAkBD,SAAS,OAAO,CAAC,IAAI;QAEvC,IACEC,gBAAgB,UAAU,CAAC,uBAC3BA,gBAAgB,UAAU,CAAC,0BAC3B;YACA,MAAMC,OAAOR,kBAAkBO,iBAAiB;YAChD,OAAO;gBACL,WAAW;gBACX,QAAQ;gBACRC;gBACA,OAAOF,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,oBAAoB;YACjD,IAAIE,UAAUT,kBAAkBO,iBAAiB;YACjD,IAAIE,QAAQ,QAAQ,CAAC,MAAMA,UAAUA,QAAQ,KAAK,CAAC,GAAG;YACtD,OAAO;gBACL,WAAW;gBACXA;gBACA,OAAOH,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,QAAQ;YACrC,MAAMG,cAAcH,gBAAgB,KAAK,CAAC;YAC1C,IAAI,CAACG,aACH,MAAM,IAAIP,MACR,CAAC,mDAAmD,EAAEI,gBAAgB,CAAC,CAAC;YAE5E,MAAMI,aAAaD,WAAW,CAAC,EAAE;YAEjC,MAAME,aAAa;gBAAE,WAAW;gBAAe,OAAON,SAAS,KAAK;YAAC;YACrE,OAAQK;gBACN,KAAK;oBAAO;wBACV,MAAME,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,oDAAoD,EAAEI,gBAAgB,CAAC,CAAC;wBAE7E,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMA,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAS;wBACZ,MAAME,aAAaR,gBAAgB,KAAK,CAAC;wBACzC,MAAMS,WAAWT,gBAAgB,KAAK,CAAC;wBACvC,IAAI,CAACQ,cAAc,CAACC,UAClB,MAAM,IAAIb,MACR,CAAC,wDAAwD,EAAEI,gBAAgB,CAAC,CAAC;wBAEjF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,OAAO;gCAACE,OAAOC,UAAU,CAAC,EAAE;gCAAGD,OAAOC,UAAU,CAAC,EAAE;6BAAE;4BACrD,KAAK;gCAACD,OAAOE,QAAQ,CAAC,EAAE;gCAAGF,OAAOE,QAAQ,CAAC,EAAE;6BAAE;wBACjD;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMH,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAU;wBACb,MAAMI,MAAMjB,kBAAkBO,iBAAiB;wBAC/C,OAAO;4BAAE,GAAGK,UAAU;4BAAE,QAAQ;4BAAUK;wBAAI;oBAChD;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGL,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBACH,OAAO;wBAAE,GAAGA,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBAAQ;wBACX,MAAMM,gBAAgBX,gBAAgB,KAAK,CACzC;wBAEF,IAAI,CAACW,eACH,MAAM,IAAIf,MACR,CAAC,0CAA0C,EAAEI,gBAAgB,CAAC,CAAC;wBAGnE,MAAMY,UAAUL,OAAO,QAAQ,CAACI,aAAa,CAAC,EAAE,EAAE;wBAClD,MAAME,aAAaD,AAAU,OAAVA;wBACnB,OAAO;4BACL,GAAGP,UAAU;4BACb,QAAQ;4BACRQ;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGR,UAAU;wBAAE,QAAQ;oBAAW;gBAE7C,KAAK;oBAAY;wBACf,MAAMS,cAAcrB,kBAClBO,iBACA;wBAEF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRS;wBACF;oBACF;gBACA,KAAK;oBAAa;wBAChB,MAAMZ,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA,KAAK;oBAAQ;wBACX,MAAMA,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA;oBACE,MAAM,IAAIN,MACR,CAAC,qBAAqB,EAAEQ,WAAW,OAAO,EAAEJ,gBAAgB,CAAC,CAAC;YAEpE;QACF;QACA,MAAM,IAAIJ,MAAM,CAAC,wBAAwB,EAAEI,iBAAiB;IAC9D,EAAE,OAAOe,OAAO;QACd,MAAMC,eAAeD,iBAAiBnB,QAAQmB,MAAM,OAAO,GAAGE,OAAOF;QACrE,MAAM,IAAInB,MACR,CAAC,wBAAwB,EAAEoB,aAAa,OAAO,EAAEhB,gBAAgB,CAAC,CAAC;IAEvE;AACF;AAEO,SAASkB,qBAAqBC,OAAe;IAIlD,IAAIA,QAAQ,QAAQ,CAAC,oBAAoB;QACvC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,eAAe,EAAEF,KAAK,CAAC,EAAE,EAAE;QAClD,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,eAAe;QAClC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,UAAU,EAAEF,KAAK,CAAC,EAAE,EAAE;QAC7C,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,aAAa;QAChC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CACnB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,cAAc,IACtB,IAAI;QACP,MAAME,gBAAgBF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI;QAC9D,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,OAAO;QAAE,OAAO;QAAIH;IAAQ;AAC9B;AAEO,SAASI,2BAA2BC,WAAmB;IAK5D,MAAM,EAAEH,KAAK,EAAE,SAASC,aAAa,EAAE,GAAGJ,qBAAqBM;IAC/D,IAAI,CAACF,cAAc,UAAU,CAAC,oBAC5B,OAAO;QACLD;QACA,aAAa;QACb,OAAO,CAAC,oDAAoD,EAAEC,eAAe;IAC/E;IAEF,IAAI;QACF,MAAMhB,eAAegB,cAAc,KAAK,CAAC;QACzC,IAAI,CAAChB,cACH,OAAO;YACLe;YACA,aAAa;YACb,OAAO,CAAC,8DAA8D,EAAEC,eAAe;QACzF;QAEF,MAAMG,IAAIlB,OAAOD,YAAY,CAAC,EAAE;QAChC,MAAMoB,IAAInB,OAAOD,YAAY,CAAC,EAAE;QAChC,OAAO;YAAEe;YAAO,aAAa;gBAAEI;gBAAGC;YAAE;QAAE;IACxC,EAAE,OAAOC,GAAG;QACV,MAAMX,eAAeW,aAAa/B,QAAQ+B,EAAE,OAAO,GAAGV,OAAOU;QAC7D,OAAO;YACLN;YACA,aAAa;YACb,OAAO,CAAC,6BAA6B,EAAEC,cAAc,qBAAqB,EAAEN,cAAc;QAC5F;IACF;AACF"}

package/dist/lib/ai-model/auto-glm/planning.js

@@ -0,0 +1,97 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    autoGLMPlanning: ()=>autoGLMPlanning
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const index_js_namespaceObject = require("../service-caller/index.js");
+const external_actions_js_namespaceObject = require("./actions.js");
+const external_parser_js_namespaceObject = require("./parser.js");
+const external_prompt_js_namespaceObject = require("./prompt.js");
+const debug = (0, logger_namespaceObject.getDebug)('auto-glm-planning');
+async function autoGLMPlanning(userInstruction, options) {
+    const { conversationHistory, context, modelConfig, actionContext } = options;
+    const systemPrompt = (0, external_prompt_js_namespaceObject.getAutoGLMPlanPrompt)(modelConfig.vlMode) + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>` : '');
+    const imagePayload = context.screenshotBase64;
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'text',
+                text: userInstruction
+            }
+        ]
+    });
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload
+                }
+            }
+        ]
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        ...conversationHistory.snapshot(1)
+    ];
+    const { content: rawResponse, usage } = await (0, index_js_namespaceObject.callAIWithStringResponse)(msgs, modelConfig);
+    debug('autoGLMPlanning rawResponse:', rawResponse);
+    const parsedResponse = (0, external_parser_js_namespaceObject.parseAutoGLMResponse)(rawResponse);
+    debug('thinking in response:', parsedResponse.think);
+    debug('action in response:', parsedResponse.content);
+    let transformedActions = [];
+    const parsedAction = (0, external_parser_js_namespaceObject.parseAction)(parsedResponse);
+    debug('Parsed action object:', parsedAction);
+    transformedActions = (0, external_actions_js_namespaceObject.transformAutoGLMAction)(parsedAction, context.size);
+    debug('Transformed actions:', transformedActions);
+    conversationHistory.append({
+        role: 'assistant',
+        content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`
+    });
+    const shouldContinue = !parsedResponse.content.startsWith('finish(');
+    return {
+        actions: transformedActions,
+        log: rawResponse,
+        usage,
+        more_actions_needed_by_instruction: shouldContinue,
+        rawResponse: JSON.stringify(rawResponse, void 0, 2)
+    };
+}
+exports.autoGLMPlanning = __webpack_exports__.autoGLMPlanning;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "autoGLMPlanning"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=planning.js.map

package/dist/lib/ai-model/auto-glm/planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAIResponse, UIContext } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ConversationHistory } from '../conversation-history';\nimport { callAIWithStringResponse } from '../service-caller/index';\nimport { transformAutoGLMAction } from './actions';\nimport { parseAction, parseAutoGLMResponse } from './parser';\nimport { getAutoGLMPlanPrompt } from './prompt';\n\nconst debug = getDebug('auto-glm-planning');\n\nexport async function autoGLMPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n\n  const systemPrompt =\n    getAutoGLMPlanPrompt(modelConfig.vlMode) +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>`\n      : '');\n\n  const imagePayload = context.screenshotBase64;\n\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'text', text: userInstruction }],\n  });\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'image_url', image_url: { url: imagePayload } }],\n  });\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...conversationHistory.snapshot(1),\n  ];\n\n  const { content: rawResponse, usage } = await callAIWithStringResponse(\n    msgs,\n    modelConfig,\n  );\n\n  debug('autoGLMPlanning rawResponse:', rawResponse);\n\n  const parsedResponse = parseAutoGLMResponse(rawResponse);\n  debug('thinking in response:', parsedResponse.think);\n  debug('action in response:', parsedResponse.content);\n\n  let transformedActions = [] as ReturnType<typeof transformAutoGLMAction>;\n\n  const parsedAction = parseAction(parsedResponse);\n  debug('Parsed action object:', parsedAction);\n  transformedActions = transformAutoGLMAction(parsedAction, context.size);\n  debug('Transformed actions:', transformedActions);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`,\n  });\n\n  const shouldContinue = !parsedResponse.content.startsWith('finish(');\n\n  return {\n    actions: transformedActions,\n    log: rawResponse,\n    usage,\n    more_actions_needed_by_instruction: shouldContinue,\n    rawResponse: JSON.stringify(rawResponse, undefined, 2),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","autoGLMPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","systemPrompt","getAutoGLMPlanPrompt","imagePayload","msgs","rawResponse","usage","callAIWithStringResponse","parsedResponse","parseAutoGLMResponse","transformedActions","parsedAction","parseAction","transformAutoGLMAction","shouldContinue","JSON","undefined"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACIA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,gBACpBC,eAAuB,EACvBC,OAKC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IAErE,MAAMK,eACJC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBH,YAAY,MAAM,IACtCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,0BAA0B,CAAC,GACrE,EAAC;IAEP,MAAMG,eAAeL,QAAQ,gBAAgB;IAE7CD,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAQ,MAAMF;YAAgB;SAAE;IACpD;IACAE,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKM;gBAAa;YAAE;SAAE;IACpE;IAEA,MAAMC,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASH;QAAa;WACrCJ,oBAAoB,QAAQ,CAAC;KACjC;IAED,MAAM,EAAE,SAASQ,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC5CH,MACAL;IAGFP,MAAM,gCAAgCa;IAEtC,MAAMG,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBJ;IAC5Cb,MAAM,yBAAyBgB,eAAe,KAAK;IACnDhB,MAAM,uBAAuBgB,eAAe,OAAO;IAEnD,IAAIE,qBAAqB,EAAE;IAE3B,MAAMC,eAAeC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYJ;IACjChB,MAAM,yBAAyBmB;IAC/BD,qBAAqBG,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EAAuBF,cAAcb,QAAQ,IAAI;IACtEN,MAAM,wBAAwBkB;IAE9Bb,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS,CAAC,OAAO,EAAEW,eAAe,KAAK,CAAC,gBAAgB,EAAEA,eAAe,OAAO,CAAC,SAAS,CAAC;IAC7F;IAEA,MAAMM,iBAAiB,CAACN,eAAe,OAAO,CAAC,UAAU,CAAC;IAE1D,OAAO;QACL,SAASE;QACT,KAAKL;QACLC;QACA,oCAAoCQ;QACpC,aAAaC,KAAK,SAAS,CAACV,aAAaW,QAAW;IACtD;AACF"}