@midscene/core 0.27.1-beta-20250822053848.0 → 0.27.1-beta-20250822103738.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent.mjs +518 -0
- package/dist/es/agent/agent.mjs.map +1 -0
- package/dist/es/agent/index.mjs +6 -0
- package/dist/es/agent/task-cache.mjs +149 -0
- package/dist/es/agent/task-cache.mjs.map +1 -0
- package/dist/es/agent/tasks.mjs +767 -0
- package/dist/es/agent/tasks.mjs.map +1 -0
- package/dist/es/agent/ui-utils.mjs +89 -0
- package/dist/es/agent/ui-utils.mjs.map +1 -0
- package/dist/es/agent/utils.mjs +303 -0
- package/dist/es/agent/utils.mjs.map +1 -0
- package/dist/es/device/index.mjs +20 -0
- package/dist/es/device/index.mjs.map +1 -0
- package/dist/es/index.mjs +2 -1
- package/dist/es/index.mjs.map +1 -1
- package/dist/es/types.mjs.map +1 -1
- package/dist/es/utils.mjs +2 -2
- package/dist/es/yaml/builder.mjs +13 -0
- package/dist/es/yaml/builder.mjs.map +1 -0
- package/dist/es/yaml/index.mjs +3 -0
- package/dist/es/yaml/player.mjs +375 -0
- package/dist/es/yaml/player.mjs.map +1 -0
- package/dist/es/yaml/utils.mjs +75 -0
- package/dist/es/yaml/utils.mjs.map +1 -0
- package/dist/lib/agent/agent.js +562 -0
- package/dist/lib/agent/agent.js.map +1 -0
- package/dist/lib/agent/index.js +84 -0
- package/dist/lib/agent/index.js.map +1 -0
- package/dist/lib/agent/task-cache.js +201 -0
- package/dist/lib/agent/task-cache.js.map +1 -0
- package/dist/lib/agent/tasks.js +804 -0
- package/dist/lib/agent/tasks.js.map +1 -0
- package/dist/lib/agent/ui-utils.js +141 -0
- package/dist/lib/agent/ui-utils.js.map +1 -0
- package/dist/lib/agent/utils.js +377 -0
- package/dist/lib/agent/utils.js.map +1 -0
- package/dist/lib/device/index.js +54 -0
- package/dist/lib/device/index.js.map +1 -0
- package/dist/lib/index.js +5 -1
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/types.js.map +1 -1
- package/dist/lib/utils.js +2 -2
- package/dist/lib/yaml/builder.js +57 -0
- package/dist/lib/yaml/builder.js.map +1 -0
- package/dist/lib/yaml/index.js +80 -0
- package/dist/lib/yaml/index.js.map +1 -0
- package/dist/lib/yaml/player.js +409 -0
- package/dist/lib/yaml/player.js.map +1 -0
- package/dist/lib/yaml/utils.js +128 -0
- package/dist/lib/yaml/utils.js.map +1 -0
- package/dist/types/agent/agent.d.ts +123 -0
- package/dist/types/agent/index.d.ts +9 -0
- package/dist/types/agent/task-cache.d.ts +38 -0
- package/dist/types/agent/tasks.d.ts +56 -0
- package/dist/types/agent/ui-utils.d.ts +11 -0
- package/dist/types/agent/utils.d.ts +49 -0
- package/dist/types/device/index.d.ts +55 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/types.d.ts +1 -0
- package/dist/types/yaml/builder.d.ts +2 -0
- package/dist/types/yaml/index.d.ts +3 -0
- package/dist/types/yaml/player.d.ts +34 -0
- package/dist/types/yaml/utils.d.ts +8 -0
- package/package.json +27 -3
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __webpack_require__ = {};
|
|
3
|
+
(()=>{
|
|
4
|
+
__webpack_require__.n = (module)=>{
|
|
5
|
+
var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
|
|
6
|
+
__webpack_require__.d(getter, {
|
|
7
|
+
a: getter
|
|
8
|
+
});
|
|
9
|
+
return getter;
|
|
10
|
+
};
|
|
11
|
+
})();
|
|
12
|
+
(()=>{
|
|
13
|
+
__webpack_require__.d = (exports1, definition)=>{
|
|
14
|
+
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
get: definition[key]
|
|
17
|
+
});
|
|
18
|
+
};
|
|
19
|
+
})();
|
|
20
|
+
(()=>{
|
|
21
|
+
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
|
|
22
|
+
})();
|
|
23
|
+
(()=>{
|
|
24
|
+
__webpack_require__.r = (exports1)=>{
|
|
25
|
+
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
|
|
26
|
+
value: 'Module'
|
|
27
|
+
});
|
|
28
|
+
Object.defineProperty(exports1, '__esModule', {
|
|
29
|
+
value: true
|
|
30
|
+
});
|
|
31
|
+
};
|
|
32
|
+
})();
|
|
33
|
+
var __webpack_exports__ = {};
|
|
34
|
+
__webpack_require__.r(__webpack_exports__);
|
|
35
|
+
__webpack_require__.d(__webpack_exports__, {
|
|
36
|
+
Agent: ()=>Agent
|
|
37
|
+
});
|
|
38
|
+
const external_index_js_namespaceObject = require("../index.js");
|
|
39
|
+
const external_js_yaml_namespaceObject = require("js-yaml");
|
|
40
|
+
var external_js_yaml_default = /*#__PURE__*/ __webpack_require__.n(external_js_yaml_namespaceObject);
|
|
41
|
+
const external_utils_js_namespaceObject = require("../utils.js");
|
|
42
|
+
const index_js_namespaceObject = require("../yaml/index.js");
|
|
43
|
+
const env_namespaceObject = require("@midscene/shared/env");
|
|
44
|
+
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
45
|
+
const utils_namespaceObject = require("@midscene/shared/utils");
|
|
46
|
+
const external_task_cache_js_namespaceObject = require("./task-cache.js");
|
|
47
|
+
const external_tasks_js_namespaceObject = require("./tasks.js");
|
|
48
|
+
const external_ui_utils_js_namespaceObject = require("./ui-utils.js");
|
|
49
|
+
const external_utils_js_namespaceObject_1 = require("./utils.js");
|
|
50
|
+
function _define_property(obj, key, value) {
|
|
51
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
52
|
+
value: value,
|
|
53
|
+
enumerable: true,
|
|
54
|
+
configurable: true,
|
|
55
|
+
writable: true
|
|
56
|
+
});
|
|
57
|
+
else obj[key] = value;
|
|
58
|
+
return obj;
|
|
59
|
+
}
|
|
60
|
+
const debug = (0, logger_namespaceObject.getDebug)('agent');
|
|
61
|
+
const distanceOfTwoPoints = (p1, p2)=>{
|
|
62
|
+
const [x1, y1] = p1;
|
|
63
|
+
const [x2, y2] = p2;
|
|
64
|
+
return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));
|
|
65
|
+
};
|
|
66
|
+
const includedInRect = (point, rect)=>{
|
|
67
|
+
const [x, y] = point;
|
|
68
|
+
const { left, top, width, height } = rect;
|
|
69
|
+
return x >= left && x <= left + width && y >= top && y <= top + height;
|
|
70
|
+
};
|
|
71
|
+
const defaultInsightExtractOption = {
|
|
72
|
+
domIncluded: false,
|
|
73
|
+
screenshotIncluded: true
|
|
74
|
+
};
|
|
75
|
+
class Agent {
|
|
76
|
+
async getActionSpace() {
|
|
77
|
+
return this.page.actionSpace();
|
|
78
|
+
}
|
|
79
|
+
async getUIContext(action) {
|
|
80
|
+
if (this.frozenPageContext) {
|
|
81
|
+
debug('Using frozen page context for action:', action);
|
|
82
|
+
return this.frozenPageContext;
|
|
83
|
+
}
|
|
84
|
+
if (this.page.getContext) {
|
|
85
|
+
debug('Using page.getContext for action:', action);
|
|
86
|
+
return await this.page.getContext();
|
|
87
|
+
}
|
|
88
|
+
debug('Using commonContextParser for action:', action);
|
|
89
|
+
return await (0, external_utils_js_namespaceObject_1.commonContextParser)(this.page);
|
|
90
|
+
}
|
|
91
|
+
async _snapshotContext() {
|
|
92
|
+
return await this.getUIContext('locate');
|
|
93
|
+
}
|
|
94
|
+
async setAIActionContext(prompt) {
|
|
95
|
+
this.opts.aiActionContext = prompt;
|
|
96
|
+
}
|
|
97
|
+
resetDump() {
|
|
98
|
+
this.dump = {
|
|
99
|
+
groupName: this.opts.groupName,
|
|
100
|
+
groupDescription: this.opts.groupDescription,
|
|
101
|
+
executions: [],
|
|
102
|
+
modelBriefs: []
|
|
103
|
+
};
|
|
104
|
+
return this.dump;
|
|
105
|
+
}
|
|
106
|
+
appendExecutionDump(execution) {
|
|
107
|
+
const trimmedExecution = (0, external_utils_js_namespaceObject_1.trimContextByViewport)(execution);
|
|
108
|
+
const currentDump = this.dump;
|
|
109
|
+
currentDump.executions.push(trimmedExecution);
|
|
110
|
+
}
|
|
111
|
+
dumpDataString() {
|
|
112
|
+
this.dump.groupName = this.opts.groupName;
|
|
113
|
+
this.dump.groupDescription = this.opts.groupDescription;
|
|
114
|
+
return (0, external_utils_js_namespaceObject.stringifyDumpData)(this.dump);
|
|
115
|
+
}
|
|
116
|
+
reportHTMLString() {
|
|
117
|
+
return (0, external_utils_js_namespaceObject.reportHTMLContent)(this.dumpDataString());
|
|
118
|
+
}
|
|
119
|
+
writeOutActionDumps() {
|
|
120
|
+
if (this.destroyed) throw new Error('PageAgent has been destroyed. Cannot update report file.');
|
|
121
|
+
const { generateReport, autoPrintReportMsg } = this.opts;
|
|
122
|
+
this.reportFile = (0, external_utils_js_namespaceObject.writeLogFile)({
|
|
123
|
+
fileName: this.reportFileName,
|
|
124
|
+
fileExt: external_utils_js_namespaceObject.groupedActionDumpFileExt,
|
|
125
|
+
fileContent: this.dumpDataString(),
|
|
126
|
+
type: 'dump',
|
|
127
|
+
generateReport
|
|
128
|
+
});
|
|
129
|
+
debug('writeOutActionDumps', this.reportFile);
|
|
130
|
+
if (generateReport && autoPrintReportMsg && this.reportFile) (0, external_utils_js_namespaceObject_1.printReportMsg)(this.reportFile);
|
|
131
|
+
}
|
|
132
|
+
async callbackOnTaskStartTip(task) {
|
|
133
|
+
const param = (0, external_ui_utils_js_namespaceObject.paramStr)(task);
|
|
134
|
+
const tip = param ? `${(0, external_ui_utils_js_namespaceObject.typeStr)(task)} - ${param}` : (0, external_ui_utils_js_namespaceObject.typeStr)(task);
|
|
135
|
+
if (this.onTaskStartTip) await this.onTaskStartTip(tip);
|
|
136
|
+
}
|
|
137
|
+
async afterTaskRunning(executor, doNotThrowError = false) {
|
|
138
|
+
this.appendExecutionDump(executor.dump());
|
|
139
|
+
try {
|
|
140
|
+
var _this_onDumpUpdate, _this;
|
|
141
|
+
await (null == (_this_onDumpUpdate = (_this = this).onDumpUpdate) ? void 0 : _this_onDumpUpdate.call(_this, this.dumpDataString()));
|
|
142
|
+
} catch (error) {
|
|
143
|
+
console.error('Error in onDumpUpdate', error);
|
|
144
|
+
}
|
|
145
|
+
this.writeOutActionDumps();
|
|
146
|
+
if (executor.isInErrorState() && !doNotThrowError) {
|
|
147
|
+
const errorTask = executor.latestErrorTask();
|
|
148
|
+
throw new Error(`${null == errorTask ? void 0 : errorTask.errorMessage}\n${null == errorTask ? void 0 : errorTask.errorStack}`, {
|
|
149
|
+
cause: null == errorTask ? void 0 : errorTask.error
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async callActionInActionSpace(type, opt) {
|
|
154
|
+
debug('callActionInActionSpace', type, ',', opt, ',', opt);
|
|
155
|
+
const actionPlan = {
|
|
156
|
+
type: type,
|
|
157
|
+
param: opt || {},
|
|
158
|
+
thought: ''
|
|
159
|
+
};
|
|
160
|
+
debug('actionPlan', actionPlan);
|
|
161
|
+
const plans = [
|
|
162
|
+
actionPlan
|
|
163
|
+
].filter(Boolean);
|
|
164
|
+
const title = (0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, (0, external_ui_utils_js_namespaceObject.locateParamStr)((null == opt ? void 0 : opt.locate) || {}));
|
|
165
|
+
const { output, executor } = await this.taskExecutor.runPlans(title, plans);
|
|
166
|
+
await this.afterTaskRunning(executor);
|
|
167
|
+
return output;
|
|
168
|
+
}
|
|
169
|
+
async aiTap(locatePrompt, opt) {
|
|
170
|
+
(0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for tap');
|
|
171
|
+
const detailedLocateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
|
|
172
|
+
return this.callActionInActionSpace('Tap', {
|
|
173
|
+
locate: detailedLocateParam
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
async aiRightClick(locatePrompt, opt) {
|
|
177
|
+
(0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for right click');
|
|
178
|
+
const detailedLocateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
|
|
179
|
+
return this.callActionInActionSpace('RightClick', {
|
|
180
|
+
locate: detailedLocateParam
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
async aiHover(locatePrompt, opt) {
|
|
184
|
+
(0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for hover');
|
|
185
|
+
const detailedLocateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
|
|
186
|
+
return this.callActionInActionSpace('Hover', {
|
|
187
|
+
locate: detailedLocateParam
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
async aiInput(locatePromptOrValue, locatePromptOrOpt, optOrUndefined) {
|
|
191
|
+
let value;
|
|
192
|
+
let locatePrompt;
|
|
193
|
+
let opt;
|
|
194
|
+
if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'value' in locatePromptOrOpt) {
|
|
195
|
+
locatePrompt = locatePromptOrValue;
|
|
196
|
+
const optWithValue = locatePromptOrOpt;
|
|
197
|
+
value = optWithValue.value;
|
|
198
|
+
opt = optWithValue;
|
|
199
|
+
} else {
|
|
200
|
+
value = locatePromptOrValue;
|
|
201
|
+
locatePrompt = locatePromptOrOpt;
|
|
202
|
+
opt = {
|
|
203
|
+
...optOrUndefined,
|
|
204
|
+
value
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
(0, utils_namespaceObject.assert)('string' == typeof value, 'input value must be a string, use empty string if you want to clear the input');
|
|
208
|
+
(0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for input');
|
|
209
|
+
const detailedLocateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
|
|
210
|
+
return this.callActionInActionSpace('Input', {
|
|
211
|
+
...opt || {},
|
|
212
|
+
locate: detailedLocateParam
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
async aiKeyboardPress(locatePromptOrKeyName, locatePromptOrOpt, optOrUndefined) {
|
|
216
|
+
let keyName;
|
|
217
|
+
let locatePrompt;
|
|
218
|
+
let opt;
|
|
219
|
+
if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'keyName' in locatePromptOrOpt) {
|
|
220
|
+
locatePrompt = locatePromptOrKeyName;
|
|
221
|
+
opt = locatePromptOrOpt;
|
|
222
|
+
} else {
|
|
223
|
+
keyName = locatePromptOrKeyName;
|
|
224
|
+
locatePrompt = locatePromptOrOpt;
|
|
225
|
+
opt = {
|
|
226
|
+
...optOrUndefined || {},
|
|
227
|
+
keyName
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
(0, utils_namespaceObject.assert)(null == opt ? void 0 : opt.keyName, 'missing keyName for keyboard press');
|
|
231
|
+
const detailedLocateParam = locatePrompt ? (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt) : void 0;
|
|
232
|
+
return this.callActionInActionSpace('KeyboardPress', {
|
|
233
|
+
...opt || {},
|
|
234
|
+
locate: detailedLocateParam
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
async aiScroll(locatePromptOrScrollParam, locatePromptOrOpt, optOrUndefined) {
|
|
238
|
+
let scrollParam;
|
|
239
|
+
let locatePrompt;
|
|
240
|
+
let opt;
|
|
241
|
+
if ('object' == typeof locatePromptOrOpt && ('direction' in locatePromptOrOpt || 'scrollType' in locatePromptOrOpt || 'distance' in locatePromptOrOpt)) {
|
|
242
|
+
locatePrompt = locatePromptOrScrollParam;
|
|
243
|
+
opt = locatePromptOrOpt;
|
|
244
|
+
} else {
|
|
245
|
+
scrollParam = locatePromptOrScrollParam;
|
|
246
|
+
locatePrompt = locatePromptOrOpt;
|
|
247
|
+
opt = {
|
|
248
|
+
...optOrUndefined || {},
|
|
249
|
+
...scrollParam || {}
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
const detailedLocateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt || '', opt);
|
|
253
|
+
return this.callActionInActionSpace('Scroll', {
|
|
254
|
+
...opt || {},
|
|
255
|
+
locate: detailedLocateParam
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
async aiAction(taskPrompt, opt) {
|
|
259
|
+
var _this_taskCache, _this_taskCache1;
|
|
260
|
+
const modelPreferences = {
|
|
261
|
+
intent: 'planning'
|
|
262
|
+
};
|
|
263
|
+
const cacheable = null == opt ? void 0 : opt.cacheable;
|
|
264
|
+
const isVlmUiTars = 'vlm-ui-tars' === (0, env_namespaceObject.vlLocateMode)(modelPreferences);
|
|
265
|
+
const matchedCache = isVlmUiTars || false === cacheable ? void 0 : null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchPlanCache(taskPrompt);
|
|
266
|
+
if (matchedCache && (null == (_this_taskCache1 = this.taskCache) ? void 0 : _this_taskCache1.isCacheResultUsed)) {
|
|
267
|
+
var _matchedCache_cacheContent, _matchedCache_cacheContent1;
|
|
268
|
+
const { executor } = await this.taskExecutor.loadYamlFlowAsPlanning(taskPrompt, null == (_matchedCache_cacheContent = matchedCache.cacheContent) ? void 0 : _matchedCache_cacheContent.yamlWorkflow);
|
|
269
|
+
await await this.afterTaskRunning(executor);
|
|
270
|
+
debug('matched cache, will call .runYaml to run the action');
|
|
271
|
+
const yaml = null == (_matchedCache_cacheContent1 = matchedCache.cacheContent) ? void 0 : _matchedCache_cacheContent1.yamlWorkflow;
|
|
272
|
+
return this.runYaml(yaml);
|
|
273
|
+
}
|
|
274
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext));
|
|
275
|
+
if (this.taskCache && (null == output ? void 0 : output.yamlFlow) && false !== cacheable) {
|
|
276
|
+
const yamlContent = {
|
|
277
|
+
tasks: [
|
|
278
|
+
{
|
|
279
|
+
name: taskPrompt,
|
|
280
|
+
flow: output.yamlFlow
|
|
281
|
+
}
|
|
282
|
+
]
|
|
283
|
+
};
|
|
284
|
+
const yamlFlowStr = external_js_yaml_default().dump(yamlContent);
|
|
285
|
+
this.taskCache.updateOrAppendCacheRecord({
|
|
286
|
+
type: 'plan',
|
|
287
|
+
prompt: taskPrompt,
|
|
288
|
+
yamlWorkflow: yamlFlowStr
|
|
289
|
+
}, matchedCache);
|
|
290
|
+
}
|
|
291
|
+
await this.afterTaskRunning(executor);
|
|
292
|
+
return output;
|
|
293
|
+
}
|
|
294
|
+
async aiQuery(demand, opt = defaultInsightExtractOption) {
|
|
295
|
+
const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Query', demand, opt);
|
|
296
|
+
await this.afterTaskRunning(executor);
|
|
297
|
+
return output;
|
|
298
|
+
}
|
|
299
|
+
async aiBoolean(prompt, opt = defaultInsightExtractOption) {
|
|
300
|
+
const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
|
|
301
|
+
const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Boolean', textPrompt, opt, multimodalPrompt);
|
|
302
|
+
await this.afterTaskRunning(executor);
|
|
303
|
+
return output;
|
|
304
|
+
}
|
|
305
|
+
async aiNumber(prompt, opt = defaultInsightExtractOption) {
|
|
306
|
+
const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
|
|
307
|
+
const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Number', textPrompt, opt, multimodalPrompt);
|
|
308
|
+
await this.afterTaskRunning(executor);
|
|
309
|
+
return output;
|
|
310
|
+
}
|
|
311
|
+
async aiString(prompt, opt = defaultInsightExtractOption) {
|
|
312
|
+
const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
|
|
313
|
+
const { output, executor } = await this.taskExecutor.createTypeQueryExecution('String', textPrompt, opt, multimodalPrompt);
|
|
314
|
+
await this.afterTaskRunning(executor);
|
|
315
|
+
return output;
|
|
316
|
+
}
|
|
317
|
+
async aiAsk(prompt, opt = defaultInsightExtractOption) {
|
|
318
|
+
return this.aiString(prompt, opt);
|
|
319
|
+
}
|
|
320
|
+
async describeElementAtPoint(center, opt) {
|
|
321
|
+
const { verifyPrompt = true, retryLimit = 3 } = opt || {};
|
|
322
|
+
let success = false;
|
|
323
|
+
let retryCount = 0;
|
|
324
|
+
let resultPrompt = '';
|
|
325
|
+
let deepThink = (null == opt ? void 0 : opt.deepThink) || false;
|
|
326
|
+
let verifyResult;
|
|
327
|
+
while(!success && retryCount < retryLimit){
|
|
328
|
+
if (retryCount >= 2) deepThink = true;
|
|
329
|
+
debug('aiDescribe', center, 'verifyPrompt', verifyPrompt, 'retryCount', retryCount, 'deepThink', deepThink);
|
|
330
|
+
const text = await this.insight.describe(center, {
|
|
331
|
+
deepThink
|
|
332
|
+
});
|
|
333
|
+
debug('aiDescribe text', text);
|
|
334
|
+
(0, utils_namespaceObject.assert)(text.description, `failed to describe element at [${center}]`);
|
|
335
|
+
resultPrompt = text.description;
|
|
336
|
+
verifyResult = await this.verifyLocator(resultPrompt, deepThink ? {
|
|
337
|
+
deepThink: true
|
|
338
|
+
} : void 0, center, opt);
|
|
339
|
+
if (verifyResult.pass) success = true;
|
|
340
|
+
else retryCount++;
|
|
341
|
+
}
|
|
342
|
+
return {
|
|
343
|
+
prompt: resultPrompt,
|
|
344
|
+
deepThink,
|
|
345
|
+
verifyResult
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
async verifyLocator(prompt, locateOpt, expectCenter, verifyLocateOption) {
|
|
349
|
+
debug('verifyLocator', prompt, locateOpt, expectCenter, verifyLocateOption);
|
|
350
|
+
const { center: verifyCenter, rect: verifyRect } = await this.aiLocate(prompt, locateOpt);
|
|
351
|
+
const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
|
|
352
|
+
const included = includedInRect(expectCenter, verifyRect);
|
|
353
|
+
const pass = distance <= ((null == verifyLocateOption ? void 0 : verifyLocateOption.centerDistanceThreshold) || 20) || included;
|
|
354
|
+
const verifyResult = {
|
|
355
|
+
pass,
|
|
356
|
+
rect: verifyRect,
|
|
357
|
+
center: verifyCenter,
|
|
358
|
+
centerDistance: distance
|
|
359
|
+
};
|
|
360
|
+
debug('aiDescribe verifyResult', verifyResult);
|
|
361
|
+
return verifyResult;
|
|
362
|
+
}
|
|
363
|
+
async aiLocate(prompt, opt) {
|
|
364
|
+
const locateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(prompt, opt);
|
|
365
|
+
(0, utils_namespaceObject.assert)(locateParam, 'cannot get locate param for aiLocate');
|
|
366
|
+
const locatePlan = (0, external_tasks_js_namespaceObject.locatePlanForLocate)(locateParam);
|
|
367
|
+
const plans = [
|
|
368
|
+
locatePlan
|
|
369
|
+
];
|
|
370
|
+
const { executor, output } = await this.taskExecutor.runPlans((0, external_ui_utils_js_namespaceObject.taskTitleStr)('Locate', (0, external_ui_utils_js_namespaceObject.locateParamStr)(locateParam)), plans);
|
|
371
|
+
await this.afterTaskRunning(executor);
|
|
372
|
+
const { element } = output;
|
|
373
|
+
return {
|
|
374
|
+
rect: null == element ? void 0 : element.rect,
|
|
375
|
+
center: null == element ? void 0 : element.center,
|
|
376
|
+
scale: (await this.page.size()).dpr
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
async aiAssert(assertion, msg, opt) {
|
|
380
|
+
var _executor_latestErrorTask;
|
|
381
|
+
const insightOpt = {
|
|
382
|
+
domIncluded: (null == opt ? void 0 : opt.domIncluded) ?? defaultInsightExtractOption.domIncluded,
|
|
383
|
+
screenshotIncluded: (null == opt ? void 0 : opt.screenshotIncluded) ?? defaultInsightExtractOption.screenshotIncluded,
|
|
384
|
+
returnThought: (null == opt ? void 0 : opt.returnThought) ?? true,
|
|
385
|
+
isWaitForAssert: null == opt ? void 0 : opt.isWaitForAssert,
|
|
386
|
+
doNotThrowError: null == opt ? void 0 : opt.doNotThrowError
|
|
387
|
+
};
|
|
388
|
+
const { output, executor, thought } = await this.taskExecutor.assert(assertion, insightOpt);
|
|
389
|
+
await this.afterTaskRunning(executor, true);
|
|
390
|
+
const message = output ? void 0 : `Assertion failed: ${msg || assertion}\nReason: ${thought || (null == (_executor_latestErrorTask = executor.latestErrorTask()) ? void 0 : _executor_latestErrorTask.error) || '(no_reason)'}`;
|
|
391
|
+
if (null == opt ? void 0 : opt.keepRawResponse) return {
|
|
392
|
+
pass: output,
|
|
393
|
+
thought,
|
|
394
|
+
message
|
|
395
|
+
};
|
|
396
|
+
if (!output) throw new Error(message);
|
|
397
|
+
}
|
|
398
|
+
async aiWaitFor(assertion, opt) {
|
|
399
|
+
const { executor } = await this.taskExecutor.waitFor(assertion, {
|
|
400
|
+
timeoutMs: (null == opt ? void 0 : opt.timeoutMs) || 15000,
|
|
401
|
+
checkIntervalMs: (null == opt ? void 0 : opt.checkIntervalMs) || 3000
|
|
402
|
+
});
|
|
403
|
+
await this.afterTaskRunning(executor, true);
|
|
404
|
+
if (executor.isInErrorState()) {
|
|
405
|
+
const errorTask = executor.latestErrorTask();
|
|
406
|
+
throw new Error(`${null == errorTask ? void 0 : errorTask.error}\n${null == errorTask ? void 0 : errorTask.errorStack}`);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
async ai(taskPrompt, type = 'action') {
|
|
410
|
+
if ('action' === type) return this.aiAction(taskPrompt);
|
|
411
|
+
if ('query' === type) return this.aiQuery(taskPrompt);
|
|
412
|
+
if ('assert' === type) return this.aiAssert(taskPrompt);
|
|
413
|
+
if ('tap' === type) return this.aiTap(taskPrompt);
|
|
414
|
+
if ('rightClick' === type) return this.aiRightClick(taskPrompt);
|
|
415
|
+
throw new Error(`Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'rightClick'`);
|
|
416
|
+
}
|
|
417
|
+
async runYaml(yamlScriptContent) {
|
|
418
|
+
const script = (0, index_js_namespaceObject.parseYamlScript)(yamlScriptContent, 'yaml', true);
|
|
419
|
+
const player = new index_js_namespaceObject.ScriptPlayer(script, async (target)=>({
|
|
420
|
+
agent: this,
|
|
421
|
+
freeFn: []
|
|
422
|
+
}));
|
|
423
|
+
await player.run();
|
|
424
|
+
if ('error' === player.status) {
|
|
425
|
+
const errors = player.taskStatusList.filter((task)=>'error' === task.status).map((task)=>{
|
|
426
|
+
var _task_error;
|
|
427
|
+
return `task - ${task.name}: ${null == (_task_error = task.error) ? void 0 : _task_error.message}`;
|
|
428
|
+
}).join('\n');
|
|
429
|
+
throw new Error(`Error(s) occurred in running yaml script:\n${errors}`);
|
|
430
|
+
}
|
|
431
|
+
return {
|
|
432
|
+
result: player.result
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
async evaluateJavaScript(script) {
|
|
436
|
+
(0, utils_namespaceObject.assert)(this.page.evaluateJavaScript, 'evaluateJavaScript is not supported in current agent');
|
|
437
|
+
return this.page.evaluateJavaScript(script);
|
|
438
|
+
}
|
|
439
|
+
async destroy() {
|
|
440
|
+
await this.page.destroy();
|
|
441
|
+
this.resetDump();
|
|
442
|
+
this.destroyed = true;
|
|
443
|
+
}
|
|
444
|
+
async logScreenshot(title, opt) {
|
|
445
|
+
const base64 = await this.page.screenshotBase64();
|
|
446
|
+
const now = Date.now();
|
|
447
|
+
const recorder = [
|
|
448
|
+
{
|
|
449
|
+
type: 'screenshot',
|
|
450
|
+
ts: now,
|
|
451
|
+
screenshot: base64
|
|
452
|
+
}
|
|
453
|
+
];
|
|
454
|
+
const task = {
|
|
455
|
+
type: 'Log',
|
|
456
|
+
subType: 'Screenshot',
|
|
457
|
+
status: 'finished',
|
|
458
|
+
recorder,
|
|
459
|
+
timing: {
|
|
460
|
+
start: now,
|
|
461
|
+
end: now,
|
|
462
|
+
cost: 0
|
|
463
|
+
},
|
|
464
|
+
param: {
|
|
465
|
+
content: (null == opt ? void 0 : opt.content) || ''
|
|
466
|
+
},
|
|
467
|
+
executor: async ()=>{}
|
|
468
|
+
};
|
|
469
|
+
const executionDump = {
|
|
470
|
+
sdkVersion: '',
|
|
471
|
+
logTime: now,
|
|
472
|
+
name: `Log - ${title || 'untitled'}`,
|
|
473
|
+
description: (null == opt ? void 0 : opt.content) || '',
|
|
474
|
+
tasks: [
|
|
475
|
+
task
|
|
476
|
+
]
|
|
477
|
+
};
|
|
478
|
+
this.appendExecutionDump(executionDump);
|
|
479
|
+
try {
|
|
480
|
+
var _this_onDumpUpdate, _this;
|
|
481
|
+
null == (_this_onDumpUpdate = (_this = this).onDumpUpdate) || _this_onDumpUpdate.call(_this, this.dumpDataString());
|
|
482
|
+
} catch (error) {
|
|
483
|
+
console.error('Failed to update dump', error);
|
|
484
|
+
}
|
|
485
|
+
this.writeOutActionDumps();
|
|
486
|
+
}
|
|
487
|
+
_unstableLogContent() {
|
|
488
|
+
const { groupName, groupDescription, executions } = this.dump;
|
|
489
|
+
const newExecutions = Array.isArray(executions) ? executions.map((execution)=>{
|
|
490
|
+
const { tasks, ...restExecution } = execution;
|
|
491
|
+
let newTasks = tasks;
|
|
492
|
+
if (Array.isArray(tasks)) newTasks = tasks.map((task)=>{
|
|
493
|
+
const { pageContext, log, ...restTask } = task;
|
|
494
|
+
return restTask;
|
|
495
|
+
});
|
|
496
|
+
return {
|
|
497
|
+
...restExecution,
|
|
498
|
+
...newTasks ? {
|
|
499
|
+
tasks: newTasks
|
|
500
|
+
} : {}
|
|
501
|
+
};
|
|
502
|
+
}) : [];
|
|
503
|
+
return {
|
|
504
|
+
groupName,
|
|
505
|
+
groupDescription,
|
|
506
|
+
executions: newExecutions
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
async freezePageContext() {
|
|
510
|
+
debug('Freezing page context');
|
|
511
|
+
const context = await this._snapshotContext();
|
|
512
|
+
context._isFrozen = true;
|
|
513
|
+
this.frozenPageContext = context;
|
|
514
|
+
debug('Page context frozen successfully');
|
|
515
|
+
}
|
|
516
|
+
async unfreezePageContext() {
|
|
517
|
+
debug('Unfreezing page context');
|
|
518
|
+
this.frozenPageContext = void 0;
|
|
519
|
+
debug('Page context unfrozen successfully');
|
|
520
|
+
}
|
|
521
|
+
constructor(page, opts){
|
|
522
|
+
_define_property(this, "page", void 0);
|
|
523
|
+
_define_property(this, "insight", void 0);
|
|
524
|
+
_define_property(this, "dump", void 0);
|
|
525
|
+
_define_property(this, "reportFile", void 0);
|
|
526
|
+
_define_property(this, "reportFileName", void 0);
|
|
527
|
+
_define_property(this, "taskExecutor", void 0);
|
|
528
|
+
_define_property(this, "opts", void 0);
|
|
529
|
+
_define_property(this, "dryMode", false);
|
|
530
|
+
_define_property(this, "onTaskStartTip", void 0);
|
|
531
|
+
_define_property(this, "taskCache", void 0);
|
|
532
|
+
_define_property(this, "onDumpUpdate", void 0);
|
|
533
|
+
_define_property(this, "destroyed", false);
|
|
534
|
+
_define_property(this, "frozenPageContext", void 0);
|
|
535
|
+
this.page = page;
|
|
536
|
+
this.opts = Object.assign({
|
|
537
|
+
generateReport: true,
|
|
538
|
+
autoPrintReportMsg: true,
|
|
539
|
+
groupName: 'Midscene Report',
|
|
540
|
+
groupDescription: ''
|
|
541
|
+
}, opts || {});
|
|
542
|
+
if ('function' == typeof (null == opts ? void 0 : opts.modelConfig)) env_namespaceObject.globalConfigManger.registerModelConfigFn(null == opts ? void 0 : opts.modelConfig);
|
|
543
|
+
this.onTaskStartTip = this.opts.onTaskStartTip;
|
|
544
|
+
this.insight = new external_index_js_namespaceObject.Insight(async (action)=>this.getUIContext(action));
|
|
545
|
+
if ((null == opts ? void 0 : opts.cacheId) && 'android' !== this.page.pageType) this.taskCache = new external_task_cache_js_namespaceObject.TaskCache(opts.cacheId, (0, env_namespaceObject.getAIConfigInBoolean)('MIDSCENE_CACHE'));
|
|
546
|
+
this.taskExecutor = new external_tasks_js_namespaceObject.PageTaskExecutor(this.page, this.insight, {
|
|
547
|
+
taskCache: this.taskCache,
|
|
548
|
+
onTaskStart: this.callbackOnTaskStartTip.bind(this)
|
|
549
|
+
});
|
|
550
|
+
this.dump = this.resetDump();
|
|
551
|
+
this.reportFileName = (null == opts ? void 0 : opts.reportFileName) || (0, external_utils_js_namespaceObject_1.getReportFileName)((null == opts ? void 0 : opts.testId) || this.page.pageType || 'web');
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
exports.Agent = __webpack_exports__.Agent;
|
|
555
|
+
for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
556
|
+
"Agent"
|
|
557
|
+
].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
|
|
558
|
+
Object.defineProperty(exports, '__esModule', {
|
|
559
|
+
value: true
|
|
560
|
+
});
|
|
561
|
+
|
|
562
|
+
//# sourceMappingURL=agent.js.map
|