@midscene/core 1.2.1-beta-20260114072539.0 → 1.2.1-beta-20260115021413.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent.mjs +2 -5
- package/dist/es/agent/agent.mjs.map +1 -1
- package/dist/es/agent/tasks.mjs +2 -4
- package/dist/es/agent/tasks.mjs.map +1 -1
- package/dist/es/agent/utils.mjs +1 -1
- package/dist/es/ai-model/index.mjs +1 -2
- package/dist/es/ai-model/inspect.mjs +3 -68
- package/dist/es/ai-model/inspect.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/index.mjs +8 -5
- package/dist/es/ai-model/service-caller/index.mjs.map +1 -1
- package/dist/es/ai-model/ui-tars-planning.mjs +29 -41
- package/dist/es/ai-model/ui-tars-planning.mjs.map +1 -1
- package/dist/es/common.mjs +1 -1
- package/dist/es/common.mjs.map +1 -1
- package/dist/es/service/index.mjs +0 -5
- package/dist/es/service/index.mjs.map +1 -1
- package/dist/es/utils.mjs +2 -2
- package/dist/lib/agent/agent.js +2 -5
- package/dist/lib/agent/agent.js.map +1 -1
- package/dist/lib/agent/tasks.js +1 -3
- package/dist/lib/agent/tasks.js.map +1 -1
- package/dist/lib/agent/utils.js +1 -1
- package/dist/lib/ai-model/index.js +11 -15
- package/dist/lib/ai-model/inspect.js +2 -67
- package/dist/lib/ai-model/inspect.js.map +1 -1
- package/dist/lib/ai-model/service-caller/index.js +8 -5
- package/dist/lib/ai-model/service-caller/index.js.map +1 -1
- package/dist/lib/ai-model/ui-tars-planning.js +29 -41
- package/dist/lib/ai-model/ui-tars-planning.js.map +1 -1
- package/dist/lib/common.js +1 -1
- package/dist/lib/common.js.map +1 -1
- package/dist/lib/service/index.js +0 -5
- package/dist/lib/service/index.js.map +1 -1
- package/dist/lib/utils.js +2 -2
- package/dist/types/ai-model/index.d.ts +0 -1
- package/package.json +2 -2
- package/dist/es/ai-model/auto-glm/actions.mjs +0 -217
- package/dist/es/ai-model/auto-glm/actions.mjs.map +0 -1
- package/dist/es/ai-model/auto-glm/index.mjs +0 -5
- package/dist/es/ai-model/auto-glm/parser.mjs +0 -239
- package/dist/es/ai-model/auto-glm/parser.mjs.map +0 -1
- package/dist/es/ai-model/auto-glm/planning.mjs +0 -63
- package/dist/es/ai-model/auto-glm/planning.mjs.map +0 -1
- package/dist/es/ai-model/auto-glm/prompt.mjs +0 -222
- package/dist/es/ai-model/auto-glm/prompt.mjs.map +0 -1
- package/dist/es/ai-model/auto-glm/util.mjs +0 -22
- package/dist/es/ai-model/auto-glm/util.mjs.map +0 -1
- package/dist/es/ai-model/latest-locate-recorder.mjs +0 -29
- package/dist/es/ai-model/latest-locate-recorder.mjs.map +0 -1
- package/dist/lib/ai-model/auto-glm/actions.js +0 -251
- package/dist/lib/ai-model/auto-glm/actions.js.map +0 -1
- package/dist/lib/ai-model/auto-glm/index.js +0 -59
- package/dist/lib/ai-model/auto-glm/index.js.map +0 -1
- package/dist/lib/ai-model/auto-glm/parser.js +0 -282
- package/dist/lib/ai-model/auto-glm/parser.js.map +0 -1
- package/dist/lib/ai-model/auto-glm/planning.js +0 -97
- package/dist/lib/ai-model/auto-glm/planning.js.map +0 -1
- package/dist/lib/ai-model/auto-glm/prompt.js +0 -259
- package/dist/lib/ai-model/auto-glm/prompt.js.map +0 -1
- package/dist/lib/ai-model/auto-glm/util.js +0 -62
- package/dist/lib/ai-model/auto-glm/util.js.map +0 -1
- package/dist/lib/ai-model/latest-locate-recorder.js +0 -63
- package/dist/lib/ai-model/latest-locate-recorder.js.map +0 -1
- package/dist/types/ai-model/auto-glm/actions.d.ts +0 -77
- package/dist/types/ai-model/auto-glm/index.d.ts +0 -5
- package/dist/types/ai-model/auto-glm/parser.d.ts +0 -18
- package/dist/types/ai-model/auto-glm/planning.d.ts +0 -9
- package/dist/types/ai-model/auto-glm/prompt.d.ts +0 -27
- package/dist/types/ai-model/auto-glm/util.d.ts +0 -16
- package/dist/types/ai-model/latest-locate-recorder.d.ts +0 -14
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __webpack_require__ = {};
|
|
3
|
-
(()=>{
|
|
4
|
-
__webpack_require__.d = (exports1, definition)=>{
|
|
5
|
-
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
|
|
6
|
-
enumerable: true,
|
|
7
|
-
get: definition[key]
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
})();
|
|
11
|
-
(()=>{
|
|
12
|
-
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
|
|
13
|
-
})();
|
|
14
|
-
(()=>{
|
|
15
|
-
__webpack_require__.r = (exports1)=>{
|
|
16
|
-
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
|
|
17
|
-
value: 'Module'
|
|
18
|
-
});
|
|
19
|
-
Object.defineProperty(exports1, '__esModule', {
|
|
20
|
-
value: true
|
|
21
|
-
});
|
|
22
|
-
};
|
|
23
|
-
})();
|
|
24
|
-
var __webpack_exports__ = {};
|
|
25
|
-
__webpack_require__.r(__webpack_exports__);
|
|
26
|
-
__webpack_require__.d(__webpack_exports__, {
|
|
27
|
-
getAutoGLMLocatePrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMLocatePrompt,
|
|
28
|
-
autoGLMCoordinateToBbox: ()=>external_util_js_namespaceObject.autoGLMCoordinateToBbox,
|
|
29
|
-
parseAction: ()=>external_parser_js_namespaceObject.parseAction,
|
|
30
|
-
parseAutoGLMLocateResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMLocateResponse,
|
|
31
|
-
parseAutoGLMResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMResponse,
|
|
32
|
-
autoGLMPlanning: ()=>external_planning_js_namespaceObject.autoGLMPlanning,
|
|
33
|
-
getAutoGLMPlanPrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMPlanPrompt
|
|
34
|
-
});
|
|
35
|
-
const external_prompt_js_namespaceObject = require("./prompt.js");
|
|
36
|
-
const external_parser_js_namespaceObject = require("./parser.js");
|
|
37
|
-
const external_planning_js_namespaceObject = require("./planning.js");
|
|
38
|
-
const external_util_js_namespaceObject = require("./util.js");
|
|
39
|
-
exports.autoGLMCoordinateToBbox = __webpack_exports__.autoGLMCoordinateToBbox;
|
|
40
|
-
exports.autoGLMPlanning = __webpack_exports__.autoGLMPlanning;
|
|
41
|
-
exports.getAutoGLMLocatePrompt = __webpack_exports__.getAutoGLMLocatePrompt;
|
|
42
|
-
exports.getAutoGLMPlanPrompt = __webpack_exports__.getAutoGLMPlanPrompt;
|
|
43
|
-
exports.parseAction = __webpack_exports__.parseAction;
|
|
44
|
-
exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
|
|
45
|
-
exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
|
|
46
|
-
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
47
|
-
"autoGLMCoordinateToBbox",
|
|
48
|
-
"autoGLMPlanning",
|
|
49
|
-
"getAutoGLMLocatePrompt",
|
|
50
|
-
"getAutoGLMPlanPrompt",
|
|
51
|
-
"parseAction",
|
|
52
|
-
"parseAutoGLMLocateResponse",
|
|
53
|
-
"parseAutoGLMResponse"
|
|
54
|
-
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
55
|
-
Object.defineProperty(exports, '__esModule', {
|
|
56
|
-
value: true
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/auto-glm/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}
|
|
@@ -1,282 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __webpack_require__ = {};
|
|
3
|
-
(()=>{
|
|
4
|
-
__webpack_require__.d = (exports1, definition)=>{
|
|
5
|
-
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
|
|
6
|
-
enumerable: true,
|
|
7
|
-
get: definition[key]
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
})();
|
|
11
|
-
(()=>{
|
|
12
|
-
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
|
|
13
|
-
})();
|
|
14
|
-
(()=>{
|
|
15
|
-
__webpack_require__.r = (exports1)=>{
|
|
16
|
-
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
|
|
17
|
-
value: 'Module'
|
|
18
|
-
});
|
|
19
|
-
Object.defineProperty(exports1, '__esModule', {
|
|
20
|
-
value: true
|
|
21
|
-
});
|
|
22
|
-
};
|
|
23
|
-
})();
|
|
24
|
-
var __webpack_exports__ = {};
|
|
25
|
-
__webpack_require__.r(__webpack_exports__);
|
|
26
|
-
__webpack_require__.d(__webpack_exports__, {
|
|
27
|
-
extractValueAfter: ()=>extractValueAfter,
|
|
28
|
-
parseAction: ()=>parseAction,
|
|
29
|
-
parseAutoGLMLocateResponse: ()=>parseAutoGLMLocateResponse,
|
|
30
|
-
parseAutoGLMResponse: ()=>parseAutoGLMResponse
|
|
31
|
-
});
|
|
32
|
-
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
33
|
-
const debug = (0, logger_namespaceObject.getDebug)('auto-glm-parser');
|
|
34
|
-
const extractValueAfter = (src, key)=>{
|
|
35
|
-
const idx = src.indexOf(key);
|
|
36
|
-
if (-1 === idx) throw new Error(`Missing key ${key} in action payload ${src}`);
|
|
37
|
-
let rest = src.slice(idx + key.length).trim();
|
|
38
|
-
if (rest.endsWith('")')) rest = rest.slice(0, -2);
|
|
39
|
-
return rest;
|
|
40
|
-
};
|
|
41
|
-
function parseAction(response) {
|
|
42
|
-
debug('Parsing action:', response);
|
|
43
|
-
let trimmedResponse = '';
|
|
44
|
-
try {
|
|
45
|
-
trimmedResponse = response.content.trim();
|
|
46
|
-
if (trimmedResponse.startsWith('do(action="Type"') || trimmedResponse.startsWith('do(action="Type_Name"')) {
|
|
47
|
-
const text = extractValueAfter(trimmedResponse, 'text="');
|
|
48
|
-
return {
|
|
49
|
-
_metadata: 'do',
|
|
50
|
-
action: 'Type',
|
|
51
|
-
text,
|
|
52
|
-
think: response.think
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
if (trimmedResponse.startsWith('finish(message=')) {
|
|
56
|
-
let message = extractValueAfter(trimmedResponse, 'finish(message="');
|
|
57
|
-
if (message.endsWith(')')) message = message.slice(0, -1);
|
|
58
|
-
return {
|
|
59
|
-
_metadata: 'finish',
|
|
60
|
-
message,
|
|
61
|
-
think: response.think
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
if (trimmedResponse.startsWith('do(')) {
|
|
65
|
-
const actionMatch = trimmedResponse.match(/do\(action="([^"]+)"/);
|
|
66
|
-
if (!actionMatch) throw new Error(`Failed to extract action type from do() call; raw="${trimmedResponse}"`);
|
|
67
|
-
const actionType = actionMatch[1];
|
|
68
|
-
const baseAction = {
|
|
69
|
-
_metadata: 'do',
|
|
70
|
-
think: response.think
|
|
71
|
-
};
|
|
72
|
-
switch(actionType){
|
|
73
|
-
case 'Tap':
|
|
74
|
-
{
|
|
75
|
-
const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
|
|
76
|
-
if (!elementMatch) throw new Error(`Failed to extract element coordinates for Tap; raw="${trimmedResponse}"`);
|
|
77
|
-
return {
|
|
78
|
-
...baseAction,
|
|
79
|
-
action: 'Tap',
|
|
80
|
-
element: [
|
|
81
|
-
Number(elementMatch[1]),
|
|
82
|
-
Number(elementMatch[2])
|
|
83
|
-
]
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
case 'Double Tap':
|
|
87
|
-
{
|
|
88
|
-
const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
|
|
89
|
-
if (!elementMatch) throw new Error(`Failed to extract element coordinates for Double Tap; raw="${trimmedResponse}"`);
|
|
90
|
-
return {
|
|
91
|
-
...baseAction,
|
|
92
|
-
action: 'Double Tap',
|
|
93
|
-
element: [
|
|
94
|
-
Number(elementMatch[1]),
|
|
95
|
-
Number(elementMatch[2])
|
|
96
|
-
]
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
case 'Swipe':
|
|
100
|
-
{
|
|
101
|
-
const startMatch = trimmedResponse.match(/start=\[(\d+),(\d+)\]/);
|
|
102
|
-
const endMatch = trimmedResponse.match(/end=\[(\d+),(\d+)\]/);
|
|
103
|
-
if (!startMatch || !endMatch) throw new Error(`Failed to extract start/end coordinates for Swipe; raw="${trimmedResponse}"`);
|
|
104
|
-
return {
|
|
105
|
-
...baseAction,
|
|
106
|
-
action: 'Swipe',
|
|
107
|
-
start: [
|
|
108
|
-
Number(startMatch[1]),
|
|
109
|
-
Number(startMatch[2])
|
|
110
|
-
],
|
|
111
|
-
end: [
|
|
112
|
-
Number(endMatch[1]),
|
|
113
|
-
Number(endMatch[2])
|
|
114
|
-
]
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
case 'Long Press':
|
|
118
|
-
{
|
|
119
|
-
const elementMatch = trimmedResponse.match(/element=\[(\d+),(\d+)\]/);
|
|
120
|
-
if (!elementMatch) throw new Error(`Failed to extract element coordinates for Long Press; raw="${trimmedResponse}"`);
|
|
121
|
-
return {
|
|
122
|
-
...baseAction,
|
|
123
|
-
action: 'Long Press',
|
|
124
|
-
element: [
|
|
125
|
-
Number(elementMatch[1]),
|
|
126
|
-
Number(elementMatch[2])
|
|
127
|
-
]
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
|
-
case 'Launch':
|
|
131
|
-
{
|
|
132
|
-
const app = extractValueAfter(trimmedResponse, 'app="');
|
|
133
|
-
return {
|
|
134
|
-
...baseAction,
|
|
135
|
-
action: 'Launch',
|
|
136
|
-
app
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
case 'Back':
|
|
140
|
-
return {
|
|
141
|
-
...baseAction,
|
|
142
|
-
action: 'Back'
|
|
143
|
-
};
|
|
144
|
-
case 'Home':
|
|
145
|
-
return {
|
|
146
|
-
...baseAction,
|
|
147
|
-
action: 'Home'
|
|
148
|
-
};
|
|
149
|
-
case 'Wait':
|
|
150
|
-
{
|
|
151
|
-
const durationMatch = trimmedResponse.match(/duration=(?:["\[])?(\d+)/);
|
|
152
|
-
if (!durationMatch) throw new Error(`Failed to extract duration for Wait; raw="${trimmedResponse}"`);
|
|
153
|
-
const seconds = Number.parseInt(durationMatch[1], 10);
|
|
154
|
-
const durationMs = 1000 * seconds;
|
|
155
|
-
return {
|
|
156
|
-
...baseAction,
|
|
157
|
-
action: 'Wait',
|
|
158
|
-
durationMs
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
|
-
case 'Interact':
|
|
162
|
-
return {
|
|
163
|
-
...baseAction,
|
|
164
|
-
action: 'Interact'
|
|
165
|
-
};
|
|
166
|
-
case 'Call_API':
|
|
167
|
-
{
|
|
168
|
-
const instruction = extractValueAfter(trimmedResponse, 'instruction="');
|
|
169
|
-
return {
|
|
170
|
-
...baseAction,
|
|
171
|
-
action: 'Call_API',
|
|
172
|
-
instruction
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
case 'Take_over':
|
|
176
|
-
{
|
|
177
|
-
const message = extractValueAfter(trimmedResponse, 'message="');
|
|
178
|
-
return {
|
|
179
|
-
...baseAction,
|
|
180
|
-
action: 'Take_over',
|
|
181
|
-
message
|
|
182
|
-
};
|
|
183
|
-
}
|
|
184
|
-
case 'Note':
|
|
185
|
-
{
|
|
186
|
-
const message = extractValueAfter(trimmedResponse, 'message="');
|
|
187
|
-
return {
|
|
188
|
-
...baseAction,
|
|
189
|
-
action: 'Note',
|
|
190
|
-
message
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
default:
|
|
194
|
-
throw new Error(`Unknown action type: ${actionType}; raw="${trimmedResponse}"`);
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
throw new Error(`Failed to parse action: ${trimmedResponse}`);
|
|
198
|
-
} catch (error) {
|
|
199
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
200
|
-
throw new Error(`Failed to parse action: ${errorMessage}; raw="${trimmedResponse}"`);
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
function parseAutoGLMResponse(content) {
|
|
204
|
-
if (content.includes('finish(message=')) {
|
|
205
|
-
const parts = content.split('finish(message=');
|
|
206
|
-
const think = parts[0].trim();
|
|
207
|
-
const actionContent = `finish(message=${parts[1]}`;
|
|
208
|
-
return {
|
|
209
|
-
think,
|
|
210
|
-
content: actionContent
|
|
211
|
-
};
|
|
212
|
-
}
|
|
213
|
-
if (content.includes('do(action=')) {
|
|
214
|
-
const parts = content.split('do(action=');
|
|
215
|
-
const think = parts[0].trim();
|
|
216
|
-
const actionContent = `do(action=${parts[1]}`;
|
|
217
|
-
return {
|
|
218
|
-
think,
|
|
219
|
-
content: actionContent
|
|
220
|
-
};
|
|
221
|
-
}
|
|
222
|
-
if (content.includes('<answer>')) {
|
|
223
|
-
const parts = content.split('<answer>');
|
|
224
|
-
const think = parts[0].replace(/<think>/g, '').replace(/<\/think>/g, '').trim();
|
|
225
|
-
const actionContent = parts[1].replace(/<\/answer>/g, '').trim();
|
|
226
|
-
return {
|
|
227
|
-
think,
|
|
228
|
-
content: actionContent
|
|
229
|
-
};
|
|
230
|
-
}
|
|
231
|
-
return {
|
|
232
|
-
think: '',
|
|
233
|
-
content
|
|
234
|
-
};
|
|
235
|
-
}
|
|
236
|
-
function parseAutoGLMLocateResponse(rawResponse) {
|
|
237
|
-
const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);
|
|
238
|
-
if (!actionContent.startsWith('do(action="Tap"')) return {
|
|
239
|
-
think,
|
|
240
|
-
coordinates: null,
|
|
241
|
-
error: `Unexpected action type in auto-glm locate response: ${actionContent}`
|
|
242
|
-
};
|
|
243
|
-
try {
|
|
244
|
-
const elementMatch = actionContent.match(/element=\[(\d+),(\d+)\]/);
|
|
245
|
-
if (!elementMatch) return {
|
|
246
|
-
think,
|
|
247
|
-
coordinates: null,
|
|
248
|
-
error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`
|
|
249
|
-
};
|
|
250
|
-
const x = Number(elementMatch[1]);
|
|
251
|
-
const y = Number(elementMatch[2]);
|
|
252
|
-
return {
|
|
253
|
-
think,
|
|
254
|
-
coordinates: {
|
|
255
|
-
x,
|
|
256
|
-
y
|
|
257
|
-
}
|
|
258
|
-
};
|
|
259
|
-
} catch (e) {
|
|
260
|
-
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
261
|
-
return {
|
|
262
|
-
think,
|
|
263
|
-
coordinates: null,
|
|
264
|
-
error: `Failed to parse coordinates "${actionContent}" with errorMessage: ${errorMessage}`
|
|
265
|
-
};
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
exports.extractValueAfter = __webpack_exports__.extractValueAfter;
|
|
269
|
-
exports.parseAction = __webpack_exports__.parseAction;
|
|
270
|
-
exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
|
|
271
|
-
exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
|
|
272
|
-
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
273
|
-
"extractValueAfter",
|
|
274
|
-
"parseAction",
|
|
275
|
-
"parseAutoGLMLocateResponse",
|
|
276
|
-
"parseAutoGLMResponse"
|
|
277
|
-
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
278
|
-
Object.defineProperty(exports, '__esModule', {
|
|
279
|
-
value: true
|
|
280
|
-
});
|
|
281
|
-
|
|
282
|
-
//# sourceMappingURL=parser.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/auto-glm/parser.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/parser.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@midscene/shared/logger';\nimport type { ParsedAction } from './actions';\n\nconst debug = getDebug('auto-glm-parser');\n\n// Do not rely on regex extraction here; regex can fail on malformed input.\n// Bad Case: finish(message=\"Finished! Now There is a contact whose name is \\\"Tom\\\" in the list.\")\nexport const extractValueAfter = (src: string, key: string): string => {\n const idx = src.indexOf(key);\n if (idx === -1) {\n throw new Error(`Missing key ${key} in action payload ${src}`);\n }\n let rest = src.slice(idx + key.length).trim();\n if (rest.endsWith('\")')) {\n rest = rest.slice(0, -2);\n }\n return rest;\n};\n\nexport function parseAction(response: {\n think: string;\n content: string;\n}): ParsedAction {\n debug('Parsing action:', response);\n let trimmedResponse = '';\n try {\n trimmedResponse = response.content.trim();\n\n if (\n trimmedResponse.startsWith('do(action=\"Type\"') ||\n trimmedResponse.startsWith('do(action=\"Type_Name\"')\n ) {\n const text = extractValueAfter(trimmedResponse, 'text=\"');\n return {\n _metadata: 'do',\n action: 'Type',\n text,\n think: response.think,\n } as ParsedAction;\n }\n\n if (trimmedResponse.startsWith('finish(message=')) {\n let message = extractValueAfter(trimmedResponse, 'finish(message=\"');\n if (message.endsWith(')')) message = message.slice(0, -1);\n return {\n _metadata: 'finish',\n message,\n think: response.think,\n } as ParsedAction;\n }\n\n if (trimmedResponse.startsWith('do(')) {\n const actionMatch = trimmedResponse.match(/do\\(action=\"([^\"]+)\"/);\n if (!actionMatch)\n throw new Error(\n `Failed to extract action type from do() call; raw=\"${trimmedResponse}\"`,\n );\n const actionType = actionMatch[1];\n\n const baseAction = { _metadata: 'do' as const, think: response.think };\n switch (actionType) {\n case 'Tap': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Tap; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Tap',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as ParsedAction;\n }\n case 'Double Tap': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Double Tap; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Double Tap',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as ParsedAction;\n }\n case 'Swipe': {\n const startMatch = trimmedResponse.match(/start=\\[(\\d+),(\\d+)\\]/);\n const endMatch = trimmedResponse.match(/end=\\[(\\d+),(\\d+)\\]/);\n if (!startMatch || !endMatch)\n throw new Error(\n `Failed to extract start/end coordinates for Swipe; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Swipe',\n start: [Number(startMatch[1]), Number(startMatch[2])],\n end: [Number(endMatch[1]), Number(endMatch[2])],\n } as ParsedAction;\n }\n case 'Long Press': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Long Press; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Long Press',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as ParsedAction;\n }\n case 'Launch': {\n const app = extractValueAfter(trimmedResponse, 'app=\"');\n return { ...baseAction, action: 'Launch', app } as ParsedAction;\n }\n case 'Back': {\n return { ...baseAction, action: 'Back' } as ParsedAction;\n }\n case 'Home': {\n return { ...baseAction, action: 'Home' } as ParsedAction;\n }\n case 'Wait': {\n const durationMatch = trimmedResponse.match(\n /duration=(?:[\"\\[])?(\\d+)/,\n );\n if (!durationMatch) {\n throw new Error(\n `Failed to extract duration for Wait; raw=\"${trimmedResponse}\"`,\n );\n }\n const seconds = Number.parseInt(durationMatch[1], 10);\n const durationMs = seconds * 1000;\n return {\n ...baseAction,\n action: 'Wait',\n durationMs,\n } as ParsedAction;\n }\n case 'Interact': {\n return { ...baseAction, action: 'Interact' } as ParsedAction;\n }\n case 'Call_API': {\n const instruction = extractValueAfter(\n trimmedResponse,\n 'instruction=\"',\n );\n return {\n ...baseAction,\n action: 'Call_API',\n instruction,\n } as ParsedAction;\n }\n case 'Take_over': {\n const message = extractValueAfter(trimmedResponse, 'message=\"');\n return {\n ...baseAction,\n action: 'Take_over',\n message,\n } as ParsedAction;\n }\n case 'Note': {\n const message = extractValueAfter(trimmedResponse, 'message=\"');\n return {\n ...baseAction,\n action: 'Note',\n message,\n } as ParsedAction;\n }\n default:\n throw new Error(\n `Unknown action type: ${actionType}; raw=\"${trimmedResponse}\"`,\n );\n }\n }\n throw new Error(`Failed to parse action: ${trimmedResponse}`);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n throw new Error(\n `Failed to parse action: ${errorMessage}; raw=\"${trimmedResponse}\"`,\n );\n }\n}\n\nexport function parseAutoGLMResponse(content: string): {\n think: string;\n content: string;\n} {\n if (content.includes('finish(message=')) {\n const parts = content.split('finish(message=');\n const think = parts[0].trim();\n const actionContent = `finish(message=${parts[1]}`;\n return { think, content: actionContent };\n }\n if (content.includes('do(action=')) {\n const parts = content.split('do(action=');\n const think = parts[0].trim();\n const actionContent = `do(action=${parts[1]}`;\n return { think, content: actionContent };\n }\n if (content.includes('<answer>')) {\n const parts = content.split('<answer>');\n const think = parts[0]\n .replace(/<think>/g, '')\n .replace(/<\\/think>/g, '')\n .trim();\n const actionContent = parts[1].replace(/<\\/answer>/g, '').trim();\n return { think, content: actionContent };\n }\n return { think: '', content };\n}\n\nexport function parseAutoGLMLocateResponse(rawResponse: string): {\n think: string;\n coordinates: { x: number; y: number } | null;\n error?: string;\n} {\n const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);\n if (!actionContent.startsWith('do(action=\"Tap\"')) {\n return {\n think,\n coordinates: null,\n error: `Unexpected action type in auto-glm locate response: ${actionContent}`,\n };\n }\n try {\n const elementMatch = actionContent.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch) {\n return {\n think,\n coordinates: null,\n error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`,\n };\n }\n const x = Number(elementMatch[1]);\n const y = Number(elementMatch[2]);\n return { think, coordinates: { x, y } };\n } catch (e) {\n const errorMessage = e instanceof Error ? e.message : String(e);\n return {\n think,\n coordinates: null,\n error: `Failed to parse coordinates \"${actionContent}\" with errorMessage: ${errorMessage}`,\n };\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","extractValueAfter","src","idx","Error","rest","parseAction","response","trimmedResponse","text","message","actionMatch","actionType","baseAction","elementMatch","Number","startMatch","endMatch","app","durationMatch","seconds","durationMs","instruction","error","errorMessage","String","parseAutoGLMResponse","content","parts","think","actionContent","parseAutoGLMLocateResponse","rawResponse","x","y","e"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACHA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIhB,MAAMC,oBAAoB,CAACC,KAAaR;IAC7C,MAAMS,MAAMD,IAAI,OAAO,CAACR;IACxB,IAAIS,AAAQ,OAARA,KACF,MAAM,IAAIC,MAAM,CAAC,YAAY,EAAEV,IAAI,mBAAmB,EAAEQ,KAAK;IAE/D,IAAIG,OAAOH,IAAI,KAAK,CAACC,MAAMT,IAAI,MAAM,EAAE,IAAI;IAC3C,IAAIW,KAAK,QAAQ,CAAC,OAChBA,OAAOA,KAAK,KAAK,CAAC,GAAG;IAEvB,OAAOA;AACT;AAEO,SAASC,YAAYC,QAG3B;IACCR,MAAM,mBAAmBQ;IACzB,IAAIC,kBAAkB;IACtB,IAAI;QACFA,kBAAkBD,SAAS,OAAO,CAAC,IAAI;QAEvC,IACEC,gBAAgB,UAAU,CAAC,uBAC3BA,gBAAgB,UAAU,CAAC,0BAC3B;YACA,MAAMC,OAAOR,kBAAkBO,iBAAiB;YAChD,OAAO;gBACL,WAAW;gBACX,QAAQ;gBACRC;gBACA,OAAOF,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,oBAAoB;YACjD,IAAIE,UAAUT,kBAAkBO,iBAAiB;YACjD,IAAIE,QAAQ,QAAQ,CAAC,MAAMA,UAAUA,QAAQ,KAAK,CAAC,GAAG;YACtD,OAAO;gBACL,WAAW;gBACXA;gBACA,OAAOH,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,QAAQ;YACrC,MAAMG,cAAcH,gBAAgB,KAAK,CAAC;YAC1C,IAAI,CAACG,aACH,MAAM,IAAIP,MACR,CAAC,mDAAmD,EAAEI,gBAAgB,CAAC,CAAC;YAE5E,MAAMI,aAAaD,WAAW,CAAC,EAAE;YAEjC,MAAME,aAAa;gBAAE,WAAW;gBAAe,OAAON,SAAS,KAAK;YAAC;YACrE,OAAQK;gBACN,KAAK;oBAAO;wBACV,MAAME,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,oDAAoD,EAAEI,gBAAgB,CAAC,CAAC;wBAE7E,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMA,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAS;wBACZ,MAAME,aAAaR,gBAAgB,KAAK,CAAC;wBACzC,MAAMS,WAAWT,gBAAgB,KAAK,CAAC;wBACvC,IAAI,CAACQ,cAAc,CAACC,UAClB,MAAM,IAAIb,MACR,CAAC,wDAAwD,EAAEI,gBAAgB,CAAC,CAAC;wBAEjF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,OAAO;gCAACE,OAAOC,UAAU,CAAC,EAAE;gCAAGD,OAAOC,UAAU,CAAC,EAAE;6BAAE;4BACrD,KAAK;gCAACD,OAAOE,QAAQ,CAAC,EAAE;gCAAGF,OAAOE,QAAQ,CAAC,EAAE;6BAAE;wBACjD;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMH,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAU;wBACb,MAAMI,MAAMjB,kBAAkBO,iBAAiB;wBAC/C,OAAO;4BAAE,GAAGK,UAAU;4BAAE,QAAQ;4BAAUK;wBAAI;oBAChD;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGL,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBACH,OAAO;wBAAE,GAAGA,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBAAQ;wBACX,MAAMM,gBAAgBX,gBAAgB,KAAK,CACzC;wBAEF,IAAI,CAACW,eACH,MAAM,IAAIf,MACR,CAAC,0CAA0C,EAAEI,gBAAgB,CAAC,CAAC;wBAGnE,MAAMY,UAAUL,OAAO,QAAQ,CAACI,aAAa,CAAC,EAAE,EAAE;wBAClD,MAAME,aAAaD,AAAU,OAAVA;wBACnB,OAAO;4BACL,GAAGP,UAAU;4BACb,QAAQ;4BACRQ;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGR,UAAU;wBAAE,QAAQ;oBAAW;gBAE7C,KAAK;oBAAY;wBACf,MAAMS,cAAcrB,kBAClBO,iBACA;wBAEF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRS;wBACF;oBACF;gBACA,KAAK;oBAAa;wBAChB,MAAMZ,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA,KAAK;oBAAQ;wBACX,MAAMA,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA;oBACE,MAAM,IAAIN,MACR,CAAC,qBAAqB,EAAEQ,WAAW,OAAO,EAAEJ,gBAAgB,CAAC,CAAC;YAEpE;QACF;QACA,MAAM,IAAIJ,MAAM,CAAC,wBAAwB,EAAEI,iBAAiB;IAC9D,EAAE,OAAOe,OAAO;QACd,MAAMC,eAAeD,iBAAiBnB,QAAQmB,MAAM,OAAO,GAAGE,OAAOF;QACrE,MAAM,IAAInB,MACR,CAAC,wBAAwB,EAAEoB,aAAa,OAAO,EAAEhB,gBAAgB,CAAC,CAAC;IAEvE;AACF;AAEO,SAASkB,qBAAqBC,OAAe;IAIlD,IAAIA,QAAQ,QAAQ,CAAC,oBAAoB;QACvC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,eAAe,EAAEF,KAAK,CAAC,EAAE,EAAE;QAClD,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,eAAe;QAClC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,UAAU,EAAEF,KAAK,CAAC,EAAE,EAAE;QAC7C,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,aAAa;QAChC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CACnB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,cAAc,IACtB,IAAI;QACP,MAAME,gBAAgBF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI;QAC9D,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,OAAO;QAAE,OAAO;QAAIH;IAAQ;AAC9B;AAEO,SAASI,2BAA2BC,WAAmB;IAK5D,MAAM,EAAEH,KAAK,EAAE,SAASC,aAAa,EAAE,GAAGJ,qBAAqBM;IAC/D,IAAI,CAACF,cAAc,UAAU,CAAC,oBAC5B,OAAO;QACLD;QACA,aAAa;QACb,OAAO,CAAC,oDAAoD,EAAEC,eAAe;IAC/E;IAEF,IAAI;QACF,MAAMhB,eAAegB,cAAc,KAAK,CAAC;QACzC,IAAI,CAAChB,cACH,OAAO;YACLe;YACA,aAAa;YACb,OAAO,CAAC,8DAA8D,EAAEC,eAAe;QACzF;QAEF,MAAMG,IAAIlB,OAAOD,YAAY,CAAC,EAAE;QAChC,MAAMoB,IAAInB,OAAOD,YAAY,CAAC,EAAE;QAChC,OAAO;YAAEe;YAAO,aAAa;gBAAEI;gBAAGC;YAAE;QAAE;IACxC,EAAE,OAAOC,GAAG;QACV,MAAMX,eAAeW,aAAa/B,QAAQ+B,EAAE,OAAO,GAAGV,OAAOU;QAC7D,OAAO;YACLN;YACA,aAAa;YACb,OAAO,CAAC,6BAA6B,EAAEC,cAAc,qBAAqB,EAAEN,cAAc;QAC5F;IACF;AACF"}
|
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __webpack_require__ = {};
|
|
3
|
-
(()=>{
|
|
4
|
-
__webpack_require__.d = (exports1, definition)=>{
|
|
5
|
-
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
|
|
6
|
-
enumerable: true,
|
|
7
|
-
get: definition[key]
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
})();
|
|
11
|
-
(()=>{
|
|
12
|
-
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
|
|
13
|
-
})();
|
|
14
|
-
(()=>{
|
|
15
|
-
__webpack_require__.r = (exports1)=>{
|
|
16
|
-
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
|
|
17
|
-
value: 'Module'
|
|
18
|
-
});
|
|
19
|
-
Object.defineProperty(exports1, '__esModule', {
|
|
20
|
-
value: true
|
|
21
|
-
});
|
|
22
|
-
};
|
|
23
|
-
})();
|
|
24
|
-
var __webpack_exports__ = {};
|
|
25
|
-
__webpack_require__.r(__webpack_exports__);
|
|
26
|
-
__webpack_require__.d(__webpack_exports__, {
|
|
27
|
-
autoGLMPlanning: ()=>autoGLMPlanning
|
|
28
|
-
});
|
|
29
|
-
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
30
|
-
const index_js_namespaceObject = require("../service-caller/index.js");
|
|
31
|
-
const external_actions_js_namespaceObject = require("./actions.js");
|
|
32
|
-
const external_parser_js_namespaceObject = require("./parser.js");
|
|
33
|
-
const external_prompt_js_namespaceObject = require("./prompt.js");
|
|
34
|
-
const debug = (0, logger_namespaceObject.getDebug)('auto-glm-planning');
|
|
35
|
-
async function autoGLMPlanning(userInstruction, options) {
|
|
36
|
-
const { conversationHistory, context, modelConfig, actionContext } = options;
|
|
37
|
-
const systemPrompt = (0, external_prompt_js_namespaceObject.getAutoGLMPlanPrompt)(modelConfig.vlMode) + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>` : '');
|
|
38
|
-
const imagePayload = context.screenshotBase64;
|
|
39
|
-
conversationHistory.append({
|
|
40
|
-
role: 'user',
|
|
41
|
-
content: [
|
|
42
|
-
{
|
|
43
|
-
type: 'text',
|
|
44
|
-
text: userInstruction
|
|
45
|
-
}
|
|
46
|
-
]
|
|
47
|
-
});
|
|
48
|
-
conversationHistory.append({
|
|
49
|
-
role: 'user',
|
|
50
|
-
content: [
|
|
51
|
-
{
|
|
52
|
-
type: 'image_url',
|
|
53
|
-
image_url: {
|
|
54
|
-
url: imagePayload
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
]
|
|
58
|
-
});
|
|
59
|
-
const msgs = [
|
|
60
|
-
{
|
|
61
|
-
role: 'system',
|
|
62
|
-
content: systemPrompt
|
|
63
|
-
},
|
|
64
|
-
...conversationHistory.snapshot(1)
|
|
65
|
-
];
|
|
66
|
-
const { content: rawResponse, usage } = await (0, index_js_namespaceObject.callAIWithStringResponse)(msgs, modelConfig);
|
|
67
|
-
debug('autoGLMPlanning rawResponse:', rawResponse);
|
|
68
|
-
const parsedResponse = (0, external_parser_js_namespaceObject.parseAutoGLMResponse)(rawResponse);
|
|
69
|
-
debug('thinking in response:', parsedResponse.think);
|
|
70
|
-
debug('action in response:', parsedResponse.content);
|
|
71
|
-
let transformedActions = [];
|
|
72
|
-
const parsedAction = (0, external_parser_js_namespaceObject.parseAction)(parsedResponse);
|
|
73
|
-
debug('Parsed action object:', parsedAction);
|
|
74
|
-
transformedActions = (0, external_actions_js_namespaceObject.transformAutoGLMAction)(parsedAction, context.size);
|
|
75
|
-
debug('Transformed actions:', transformedActions);
|
|
76
|
-
conversationHistory.append({
|
|
77
|
-
role: 'assistant',
|
|
78
|
-
content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`
|
|
79
|
-
});
|
|
80
|
-
const shouldContinue = !parsedResponse.content.startsWith('finish(');
|
|
81
|
-
return {
|
|
82
|
-
actions: transformedActions,
|
|
83
|
-
log: rawResponse,
|
|
84
|
-
usage,
|
|
85
|
-
more_actions_needed_by_instruction: shouldContinue,
|
|
86
|
-
rawResponse: JSON.stringify(rawResponse, void 0, 2)
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
exports.autoGLMPlanning = __webpack_exports__.autoGLMPlanning;
|
|
90
|
-
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
91
|
-
"autoGLMPlanning"
|
|
92
|
-
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
93
|
-
Object.defineProperty(exports, '__esModule', {
|
|
94
|
-
value: true
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
//# sourceMappingURL=planning.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/auto-glm/planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { PlanningAIResponse, UIContext } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ConversationHistory } from '../conversation-history';\nimport { callAIWithStringResponse } from '../service-caller/index';\nimport { transformAutoGLMAction } from './actions';\nimport { parseAction, parseAutoGLMResponse } from './parser';\nimport { getAutoGLMPlanPrompt } from './prompt';\n\nconst debug = getDebug('auto-glm-planning');\n\nexport async function autoGLMPlanning(\n userInstruction: string,\n options: {\n conversationHistory: ConversationHistory;\n context: UIContext;\n modelConfig: IModelConfig;\n actionContext?: string;\n },\n): Promise<PlanningAIResponse> {\n const { conversationHistory, context, modelConfig, actionContext } = options;\n\n const systemPrompt =\n getAutoGLMPlanPrompt(modelConfig.vlMode) +\n (actionContext\n ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>`\n : '');\n\n const imagePayload = context.screenshotBase64;\n\n conversationHistory.append({\n role: 'user',\n content: [{ type: 'text', text: userInstruction }],\n });\n conversationHistory.append({\n role: 'user',\n content: [{ type: 'image_url', image_url: { url: imagePayload } }],\n });\n\n const msgs: ChatCompletionMessageParam[] = [\n { role: 'system', content: systemPrompt },\n ...conversationHistory.snapshot(1),\n ];\n\n const { content: rawResponse, usage } = await callAIWithStringResponse(\n msgs,\n modelConfig,\n );\n\n debug('autoGLMPlanning rawResponse:', rawResponse);\n\n const parsedResponse = parseAutoGLMResponse(rawResponse);\n debug('thinking in response:', parsedResponse.think);\n debug('action in response:', parsedResponse.content);\n\n let transformedActions = [] as ReturnType<typeof transformAutoGLMAction>;\n\n const parsedAction = parseAction(parsedResponse);\n debug('Parsed action object:', parsedAction);\n transformedActions = transformAutoGLMAction(parsedAction, context.size);\n debug('Transformed actions:', transformedActions);\n\n conversationHistory.append({\n role: 'assistant',\n content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`,\n });\n\n const shouldContinue = !parsedResponse.content.startsWith('finish(');\n\n return {\n actions: transformedActions,\n log: rawResponse,\n usage,\n more_actions_needed_by_instruction: shouldContinue,\n rawResponse: JSON.stringify(rawResponse, undefined, 2),\n };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","autoGLMPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","systemPrompt","getAutoGLMPlanPrompt","imagePayload","msgs","rawResponse","usage","callAIWithStringResponse","parsedResponse","parseAutoGLMResponse","transformedActions","parsedAction","parseAction","transformAutoGLMAction","shouldContinue","JSON","undefined"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACIA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,gBACpBC,eAAuB,EACvBC,OAKC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IAErE,MAAMK,eACJC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBH,YAAY,MAAM,IACtCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,0BAA0B,CAAC,GACrE,EAAC;IAEP,MAAMG,eAAeL,QAAQ,gBAAgB;IAE7CD,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAQ,MAAMF;YAAgB;SAAE;IACpD;IACAE,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKM;gBAAa;YAAE;SAAE;IACpE;IAEA,MAAMC,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASH;QAAa;WACrCJ,oBAAoB,QAAQ,CAAC;KACjC;IAED,MAAM,EAAE,SAASQ,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC5CH,MACAL;IAGFP,MAAM,gCAAgCa;IAEtC,MAAMG,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBJ;IAC5Cb,MAAM,yBAAyBgB,eAAe,KAAK;IACnDhB,MAAM,uBAAuBgB,eAAe,OAAO;IAEnD,IAAIE,qBAAqB,EAAE;IAE3B,MAAMC,eAAeC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYJ;IACjChB,MAAM,yBAAyBmB;IAC/BD,qBAAqBG,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EAAuBF,cAAcb,QAAQ,IAAI;IACtEN,MAAM,wBAAwBkB;IAE9Bb,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS,CAAC,OAAO,EAAEW,eAAe,KAAK,CAAC,gBAAgB,EAAEA,eAAe,OAAO,CAAC,SAAS,CAAC;IAC7F;IAEA,MAAMM,iBAAiB,CAACN,eAAe,OAAO,CAAC,UAAU,CAAC;IAE1D,OAAO;QACL,SAASE;QACT,KAAKL;QACLC;QACA,oCAAoCQ;QACpC,aAAaC,KAAK,SAAS,CAACV,aAAaW,QAAW;IACtD;AACF"}
|