@midscene/core 0.27.0 → 0.27.1-beta-20250822094725.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/es/agent/agent.mjs +518 -0
  2. package/dist/es/agent/agent.mjs.map +1 -0
  3. package/dist/es/agent/index.mjs +6 -0
  4. package/dist/es/agent/task-cache.mjs +149 -0
  5. package/dist/es/agent/task-cache.mjs.map +1 -0
  6. package/dist/es/agent/tasks.mjs +767 -0
  7. package/dist/es/agent/tasks.mjs.map +1 -0
  8. package/dist/es/agent/ui-utils.mjs +89 -0
  9. package/dist/es/agent/ui-utils.mjs.map +1 -0
  10. package/dist/es/agent/utils.mjs +337 -0
  11. package/dist/es/agent/utils.mjs.map +1 -0
  12. package/dist/es/device/index.mjs +20 -0
  13. package/dist/es/device/index.mjs.map +1 -0
  14. package/dist/es/index.mjs +2 -1
  15. package/dist/es/index.mjs.map +1 -1
  16. package/dist/es/types.mjs.map +1 -1
  17. package/dist/es/utils.mjs +2 -2
  18. package/dist/es/yaml/builder.mjs +13 -0
  19. package/dist/es/yaml/builder.mjs.map +1 -0
  20. package/dist/es/yaml/index.mjs +3 -0
  21. package/dist/es/yaml/player.mjs +375 -0
  22. package/dist/es/yaml/player.mjs.map +1 -0
  23. package/dist/es/yaml/utils.mjs +39 -0
  24. package/dist/es/yaml/utils.mjs.map +1 -0
  25. package/dist/lib/agent/agent.js +562 -0
  26. package/dist/lib/agent/agent.js.map +1 -0
  27. package/dist/lib/agent/index.js +90 -0
  28. package/dist/lib/agent/index.js.map +1 -0
  29. package/dist/lib/agent/task-cache.js +201 -0
  30. package/dist/lib/agent/task-cache.js.map +1 -0
  31. package/dist/lib/agent/tasks.js +804 -0
  32. package/dist/lib/agent/tasks.js.map +1 -0
  33. package/dist/lib/agent/ui-utils.js +141 -0
  34. package/dist/lib/agent/ui-utils.js.map +1 -0
  35. package/dist/lib/agent/utils.js +417 -0
  36. package/dist/lib/agent/utils.js.map +1 -0
  37. package/dist/lib/device/index.js +54 -0
  38. package/dist/lib/device/index.js.map +1 -0
  39. package/dist/lib/index.js +5 -1
  40. package/dist/lib/index.js.map +1 -1
  41. package/dist/lib/types.js.map +1 -1
  42. package/dist/lib/utils.js +2 -2
  43. package/dist/lib/yaml/builder.js +57 -0
  44. package/dist/lib/yaml/builder.js.map +1 -0
  45. package/dist/lib/yaml/index.js +80 -0
  46. package/dist/lib/yaml/index.js.map +1 -0
  47. package/dist/lib/yaml/player.js +409 -0
  48. package/dist/lib/yaml/player.js.map +1 -0
  49. package/dist/lib/yaml/utils.js +86 -0
  50. package/dist/lib/yaml/utils.js.map +1 -0
  51. package/dist/types/agent/agent.d.ts +123 -0
  52. package/dist/types/agent/index.d.ts +9 -0
  53. package/dist/types/agent/task-cache.d.ts +38 -0
  54. package/dist/types/agent/tasks.d.ts +56 -0
  55. package/dist/types/agent/ui-utils.d.ts +11 -0
  56. package/dist/types/agent/utils.d.ts +54 -0
  57. package/dist/types/device/index.d.ts +55 -0
  58. package/dist/types/index.d.ts +1 -0
  59. package/dist/types/types.d.ts +2 -0
  60. package/dist/types/yaml/builder.d.ts +2 -0
  61. package/dist/types/yaml/index.d.ts +3 -0
  62. package/dist/types/yaml/player.d.ts +34 -0
  63. package/dist/types/yaml/utils.d.ts +3 -0
  64. package/package.json +31 -3
@@ -0,0 +1,562 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.n = (module)=>{
5
+ var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
6
+ __webpack_require__.d(getter, {
7
+ a: getter
8
+ });
9
+ return getter;
10
+ };
11
+ })();
12
+ (()=>{
13
+ __webpack_require__.d = (exports1, definition)=>{
14
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
15
+ enumerable: true,
16
+ get: definition[key]
17
+ });
18
+ };
19
+ })();
20
+ (()=>{
21
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
22
+ })();
23
+ (()=>{
24
+ __webpack_require__.r = (exports1)=>{
25
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
26
+ value: 'Module'
27
+ });
28
+ Object.defineProperty(exports1, '__esModule', {
29
+ value: true
30
+ });
31
+ };
32
+ })();
33
+ var __webpack_exports__ = {};
34
+ __webpack_require__.r(__webpack_exports__);
35
+ __webpack_require__.d(__webpack_exports__, {
36
+ Agent: ()=>Agent
37
+ });
38
+ const external_index_js_namespaceObject = require("../index.js");
39
+ const external_js_yaml_namespaceObject = require("js-yaml");
40
+ var external_js_yaml_default = /*#__PURE__*/ __webpack_require__.n(external_js_yaml_namespaceObject);
41
+ const external_utils_js_namespaceObject = require("../utils.js");
42
+ const index_js_namespaceObject = require("../yaml/index.js");
43
+ const env_namespaceObject = require("@midscene/shared/env");
44
+ const logger_namespaceObject = require("@midscene/shared/logger");
45
+ const utils_namespaceObject = require("@midscene/shared/utils");
46
+ const external_task_cache_js_namespaceObject = require("./task-cache.js");
47
+ const external_tasks_js_namespaceObject = require("./tasks.js");
48
+ const external_ui_utils_js_namespaceObject = require("./ui-utils.js");
49
+ const external_utils_js_namespaceObject_1 = require("./utils.js");
50
+ function _define_property(obj, key, value) {
51
+ if (key in obj) Object.defineProperty(obj, key, {
52
+ value: value,
53
+ enumerable: true,
54
+ configurable: true,
55
+ writable: true
56
+ });
57
+ else obj[key] = value;
58
+ return obj;
59
+ }
60
+ const debug = (0, logger_namespaceObject.getDebug)('agent');
61
+ const distanceOfTwoPoints = (p1, p2)=>{
62
+ const [x1, y1] = p1;
63
+ const [x2, y2] = p2;
64
+ return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));
65
+ };
66
+ const includedInRect = (point, rect)=>{
67
+ const [x, y] = point;
68
+ const { left, top, width, height } = rect;
69
+ return x >= left && x <= left + width && y >= top && y <= top + height;
70
+ };
71
+ const defaultInsightExtractOption = {
72
+ domIncluded: false,
73
+ screenshotIncluded: true
74
+ };
75
+ class Agent {
76
+ async getActionSpace() {
77
+ return this.page.actionSpace();
78
+ }
79
+ async getUIContext(action) {
80
+ if (this.frozenPageContext) {
81
+ debug('Using frozen page context for action:', action);
82
+ return this.frozenPageContext;
83
+ }
84
+ if (this.page.getContext) {
85
+ debug('Using page.getContext for action:', action);
86
+ return await this.page.getContext();
87
+ }
88
+ debug('Using commonContextParser for action:', action);
89
+ return await (0, external_utils_js_namespaceObject_1.commonContextParser)(this.page);
90
+ }
91
+ async _snapshotContext() {
92
+ return await this.getUIContext('locate');
93
+ }
94
+ async setAIActionContext(prompt) {
95
+ this.opts.aiActionContext = prompt;
96
+ }
97
+ resetDump() {
98
+ this.dump = {
99
+ groupName: this.opts.groupName,
100
+ groupDescription: this.opts.groupDescription,
101
+ executions: [],
102
+ modelBriefs: []
103
+ };
104
+ return this.dump;
105
+ }
106
+ appendExecutionDump(execution) {
107
+ const trimmedExecution = (0, external_utils_js_namespaceObject_1.trimContextByViewport)(execution);
108
+ const currentDump = this.dump;
109
+ currentDump.executions.push(trimmedExecution);
110
+ }
111
+ dumpDataString() {
112
+ this.dump.groupName = this.opts.groupName;
113
+ this.dump.groupDescription = this.opts.groupDescription;
114
+ return (0, external_utils_js_namespaceObject.stringifyDumpData)(this.dump);
115
+ }
116
+ reportHTMLString() {
117
+ return (0, external_utils_js_namespaceObject.reportHTMLContent)(this.dumpDataString());
118
+ }
119
+ writeOutActionDumps() {
120
+ if (this.destroyed) throw new Error('PageAgent has been destroyed. Cannot update report file.');
121
+ const { generateReport, autoPrintReportMsg } = this.opts;
122
+ this.reportFile = (0, external_utils_js_namespaceObject.writeLogFile)({
123
+ fileName: this.reportFileName,
124
+ fileExt: external_utils_js_namespaceObject.groupedActionDumpFileExt,
125
+ fileContent: this.dumpDataString(),
126
+ type: 'dump',
127
+ generateReport
128
+ });
129
+ debug('writeOutActionDumps', this.reportFile);
130
+ if (generateReport && autoPrintReportMsg && this.reportFile) (0, external_utils_js_namespaceObject_1.printReportMsg)(this.reportFile);
131
+ }
132
+ async callbackOnTaskStartTip(task) {
133
+ const param = (0, external_ui_utils_js_namespaceObject.paramStr)(task);
134
+ const tip = param ? `${(0, external_ui_utils_js_namespaceObject.typeStr)(task)} - ${param}` : (0, external_ui_utils_js_namespaceObject.typeStr)(task);
135
+ if (this.onTaskStartTip) await this.onTaskStartTip(tip);
136
+ }
137
+ async afterTaskRunning(executor, doNotThrowError = false) {
138
+ this.appendExecutionDump(executor.dump());
139
+ try {
140
+ var _this_onDumpUpdate, _this;
141
+ await (null == (_this_onDumpUpdate = (_this = this).onDumpUpdate) ? void 0 : _this_onDumpUpdate.call(_this, this.dumpDataString()));
142
+ } catch (error) {
143
+ console.error('Error in onDumpUpdate', error);
144
+ }
145
+ this.writeOutActionDumps();
146
+ if (executor.isInErrorState() && !doNotThrowError) {
147
+ const errorTask = executor.latestErrorTask();
148
+ throw new Error(`${null == errorTask ? void 0 : errorTask.errorMessage}\n${null == errorTask ? void 0 : errorTask.errorStack}`, {
149
+ cause: null == errorTask ? void 0 : errorTask.error
150
+ });
151
+ }
152
+ }
153
+ async callActionInActionSpace(type, opt) {
154
+ debug('callActionInActionSpace', type, ',', opt, ',', opt);
155
+ const actionPlan = {
156
+ type: type,
157
+ param: opt || {},
158
+ thought: ''
159
+ };
160
+ debug('actionPlan', actionPlan);
161
+ const plans = [
162
+ actionPlan
163
+ ].filter(Boolean);
164
+ const title = (0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, (0, external_ui_utils_js_namespaceObject.locateParamStr)((null == opt ? void 0 : opt.locate) || {}));
165
+ const { output, executor } = await this.taskExecutor.runPlans(title, plans);
166
+ await this.afterTaskRunning(executor);
167
+ return output;
168
+ }
169
+ async aiTap(locatePrompt, opt) {
170
+ (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for tap');
171
+ const detailedLocateParam = (0, external_utils_js_namespaceObject_1.buildDetailedLocateParam)(locatePrompt, opt);
172
+ return this.callActionInActionSpace('Tap', {
173
+ locate: detailedLocateParam
174
+ });
175
+ }
176
+ async aiRightClick(locatePrompt, opt) {
177
+ (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for right click');
178
+ const detailedLocateParam = (0, external_utils_js_namespaceObject_1.buildDetailedLocateParam)(locatePrompt, opt);
179
+ return this.callActionInActionSpace('RightClick', {
180
+ locate: detailedLocateParam
181
+ });
182
+ }
183
+ async aiHover(locatePrompt, opt) {
184
+ (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for hover');
185
+ const detailedLocateParam = (0, external_utils_js_namespaceObject_1.buildDetailedLocateParam)(locatePrompt, opt);
186
+ return this.callActionInActionSpace('Hover', {
187
+ locate: detailedLocateParam
188
+ });
189
+ }
190
+ async aiInput(locatePromptOrValue, locatePromptOrOpt, optOrUndefined) {
191
+ let value;
192
+ let locatePrompt;
193
+ let opt;
194
+ if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'value' in locatePromptOrOpt) {
195
+ locatePrompt = locatePromptOrValue;
196
+ const optWithValue = locatePromptOrOpt;
197
+ value = optWithValue.value;
198
+ opt = optWithValue;
199
+ } else {
200
+ value = locatePromptOrValue;
201
+ locatePrompt = locatePromptOrOpt;
202
+ opt = {
203
+ ...optOrUndefined,
204
+ value
205
+ };
206
+ }
207
+ (0, utils_namespaceObject.assert)('string' == typeof value, 'input value must be a string, use empty string if you want to clear the input');
208
+ (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for input');
209
+ const detailedLocateParam = (0, external_utils_js_namespaceObject_1.buildDetailedLocateParam)(locatePrompt, opt);
210
+ return this.callActionInActionSpace('Input', {
211
+ ...opt || {},
212
+ locate: detailedLocateParam
213
+ });
214
+ }
215
+ async aiKeyboardPress(locatePromptOrKeyName, locatePromptOrOpt, optOrUndefined) {
216
+ let keyName;
217
+ let locatePrompt;
218
+ let opt;
219
+ if ('object' == typeof locatePromptOrOpt && null !== locatePromptOrOpt && 'keyName' in locatePromptOrOpt) {
220
+ locatePrompt = locatePromptOrKeyName;
221
+ opt = locatePromptOrOpt;
222
+ } else {
223
+ keyName = locatePromptOrKeyName;
224
+ locatePrompt = locatePromptOrOpt;
225
+ opt = {
226
+ ...optOrUndefined || {},
227
+ keyName
228
+ };
229
+ }
230
+ (0, utils_namespaceObject.assert)(null == opt ? void 0 : opt.keyName, 'missing keyName for keyboard press');
231
+ const detailedLocateParam = locatePrompt ? (0, external_utils_js_namespaceObject_1.buildDetailedLocateParam)(locatePrompt, opt) : void 0;
232
+ return this.callActionInActionSpace('KeyboardPress', {
233
+ ...opt || {},
234
+ locate: detailedLocateParam
235
+ });
236
+ }
237
+ async aiScroll(locatePromptOrScrollParam, locatePromptOrOpt, optOrUndefined) {
238
+ let scrollParam;
239
+ let locatePrompt;
240
+ let opt;
241
+ if ('object' == typeof locatePromptOrOpt && ('direction' in locatePromptOrOpt || 'scrollType' in locatePromptOrOpt || 'distance' in locatePromptOrOpt)) {
242
+ locatePrompt = locatePromptOrScrollParam;
243
+ opt = locatePromptOrOpt;
244
+ } else {
245
+ scrollParam = locatePromptOrScrollParam;
246
+ locatePrompt = locatePromptOrOpt;
247
+ opt = {
248
+ ...optOrUndefined || {},
249
+ ...scrollParam || {}
250
+ };
251
+ }
252
+ const detailedLocateParam = (0, external_utils_js_namespaceObject_1.buildDetailedLocateParam)(locatePrompt || '', opt);
253
+ return this.callActionInActionSpace('Scroll', {
254
+ ...opt || {},
255
+ locate: detailedLocateParam
256
+ });
257
+ }
258
+ async aiAction(taskPrompt, opt) {
259
+ var _this_taskCache, _this_taskCache1;
260
+ const modelPreferences = {
261
+ intent: 'planning'
262
+ };
263
+ const cacheable = null == opt ? void 0 : opt.cacheable;
264
+ const isVlmUiTars = 'vlm-ui-tars' === (0, env_namespaceObject.vlLocateMode)(modelPreferences);
265
+ const matchedCache = isVlmUiTars || false === cacheable ? void 0 : null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchPlanCache(taskPrompt);
266
+ if (matchedCache && (null == (_this_taskCache1 = this.taskCache) ? void 0 : _this_taskCache1.isCacheResultUsed)) {
267
+ var _matchedCache_cacheContent, _matchedCache_cacheContent1;
268
+ const { executor } = await this.taskExecutor.loadYamlFlowAsPlanning(taskPrompt, null == (_matchedCache_cacheContent = matchedCache.cacheContent) ? void 0 : _matchedCache_cacheContent.yamlWorkflow);
269
+ await await this.afterTaskRunning(executor);
270
+ debug('matched cache, will call .runYaml to run the action');
271
+ const yaml = null == (_matchedCache_cacheContent1 = matchedCache.cacheContent) ? void 0 : _matchedCache_cacheContent1.yamlWorkflow;
272
+ return this.runYaml(yaml);
273
+ }
274
+ const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext));
275
+ if (this.taskCache && (null == output ? void 0 : output.yamlFlow) && false !== cacheable) {
276
+ const yamlContent = {
277
+ tasks: [
278
+ {
279
+ name: taskPrompt,
280
+ flow: output.yamlFlow
281
+ }
282
+ ]
283
+ };
284
+ const yamlFlowStr = external_js_yaml_default().dump(yamlContent);
285
+ this.taskCache.updateOrAppendCacheRecord({
286
+ type: 'plan',
287
+ prompt: taskPrompt,
288
+ yamlWorkflow: yamlFlowStr
289
+ }, matchedCache);
290
+ }
291
+ await this.afterTaskRunning(executor);
292
+ return output;
293
+ }
294
+ async aiQuery(demand, opt = defaultInsightExtractOption) {
295
+ const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Query', demand, opt);
296
+ await this.afterTaskRunning(executor);
297
+ return output;
298
+ }
299
+ async aiBoolean(prompt, opt = defaultInsightExtractOption) {
300
+ const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
301
+ const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Boolean', textPrompt, opt, multimodalPrompt);
302
+ await this.afterTaskRunning(executor);
303
+ return output;
304
+ }
305
+ async aiNumber(prompt, opt = defaultInsightExtractOption) {
306
+ const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
307
+ const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Number', textPrompt, opt, multimodalPrompt);
308
+ await this.afterTaskRunning(executor);
309
+ return output;
310
+ }
311
+ async aiString(prompt, opt = defaultInsightExtractOption) {
312
+ const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
313
+ const { output, executor } = await this.taskExecutor.createTypeQueryExecution('String', textPrompt, opt, multimodalPrompt);
314
+ await this.afterTaskRunning(executor);
315
+ return output;
316
+ }
317
+ async aiAsk(prompt, opt = defaultInsightExtractOption) {
318
+ return this.aiString(prompt, opt);
319
+ }
320
+ async describeElementAtPoint(center, opt) {
321
+ const { verifyPrompt = true, retryLimit = 3 } = opt || {};
322
+ let success = false;
323
+ let retryCount = 0;
324
+ let resultPrompt = '';
325
+ let deepThink = (null == opt ? void 0 : opt.deepThink) || false;
326
+ let verifyResult;
327
+ while(!success && retryCount < retryLimit){
328
+ if (retryCount >= 2) deepThink = true;
329
+ debug('aiDescribe', center, 'verifyPrompt', verifyPrompt, 'retryCount', retryCount, 'deepThink', deepThink);
330
+ const text = await this.insight.describe(center, {
331
+ deepThink
332
+ });
333
+ debug('aiDescribe text', text);
334
+ (0, utils_namespaceObject.assert)(text.description, `failed to describe element at [${center}]`);
335
+ resultPrompt = text.description;
336
+ verifyResult = await this.verifyLocator(resultPrompt, deepThink ? {
337
+ deepThink: true
338
+ } : void 0, center, opt);
339
+ if (verifyResult.pass) success = true;
340
+ else retryCount++;
341
+ }
342
+ return {
343
+ prompt: resultPrompt,
344
+ deepThink,
345
+ verifyResult
346
+ };
347
+ }
348
+ async verifyLocator(prompt, locateOpt, expectCenter, verifyLocateOption) {
349
+ debug('verifyLocator', prompt, locateOpt, expectCenter, verifyLocateOption);
350
+ const { center: verifyCenter, rect: verifyRect } = await this.aiLocate(prompt, locateOpt);
351
+ const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
352
+ const included = includedInRect(expectCenter, verifyRect);
353
+ const pass = distance <= ((null == verifyLocateOption ? void 0 : verifyLocateOption.centerDistanceThreshold) || 20) || included;
354
+ const verifyResult = {
355
+ pass,
356
+ rect: verifyRect,
357
+ center: verifyCenter,
358
+ centerDistance: distance
359
+ };
360
+ debug('aiDescribe verifyResult', verifyResult);
361
+ return verifyResult;
362
+ }
363
+ async aiLocate(prompt, opt) {
364
+ const locateParam = (0, external_utils_js_namespaceObject_1.buildDetailedLocateParam)(prompt, opt);
365
+ (0, utils_namespaceObject.assert)(locateParam, 'cannot get locate param for aiLocate');
366
+ const locatePlan = (0, external_tasks_js_namespaceObject.locatePlanForLocate)(locateParam);
367
+ const plans = [
368
+ locatePlan
369
+ ];
370
+ const { executor, output } = await this.taskExecutor.runPlans((0, external_ui_utils_js_namespaceObject.taskTitleStr)('Locate', (0, external_ui_utils_js_namespaceObject.locateParamStr)(locateParam)), plans);
371
+ await this.afterTaskRunning(executor);
372
+ const { element } = output;
373
+ return {
374
+ rect: null == element ? void 0 : element.rect,
375
+ center: null == element ? void 0 : element.center,
376
+ scale: (await this.page.size()).dpr
377
+ };
378
+ }
379
+ async aiAssert(assertion, msg, opt) {
380
+ var _executor_latestErrorTask;
381
+ const insightOpt = {
382
+ domIncluded: (null == opt ? void 0 : opt.domIncluded) ?? defaultInsightExtractOption.domIncluded,
383
+ screenshotIncluded: (null == opt ? void 0 : opt.screenshotIncluded) ?? defaultInsightExtractOption.screenshotIncluded,
384
+ returnThought: (null == opt ? void 0 : opt.returnThought) ?? true,
385
+ isWaitForAssert: null == opt ? void 0 : opt.isWaitForAssert,
386
+ doNotThrowError: null == opt ? void 0 : opt.doNotThrowError
387
+ };
388
+ const { output, executor, thought } = await this.taskExecutor.assert(assertion, insightOpt);
389
+ await this.afterTaskRunning(executor, true);
390
+ const message = output ? void 0 : `Assertion failed: ${msg || assertion}\nReason: ${thought || (null == (_executor_latestErrorTask = executor.latestErrorTask()) ? void 0 : _executor_latestErrorTask.error) || '(no_reason)'}`;
391
+ if (null == opt ? void 0 : opt.keepRawResponse) return {
392
+ pass: output,
393
+ thought,
394
+ message
395
+ };
396
+ if (!output) throw new Error(message);
397
+ }
398
+ async aiWaitFor(assertion, opt) {
399
+ const { executor } = await this.taskExecutor.waitFor(assertion, {
400
+ timeoutMs: (null == opt ? void 0 : opt.timeoutMs) || 15000,
401
+ checkIntervalMs: (null == opt ? void 0 : opt.checkIntervalMs) || 3000
402
+ });
403
+ await this.afterTaskRunning(executor, true);
404
+ if (executor.isInErrorState()) {
405
+ const errorTask = executor.latestErrorTask();
406
+ throw new Error(`${null == errorTask ? void 0 : errorTask.error}\n${null == errorTask ? void 0 : errorTask.errorStack}`);
407
+ }
408
+ }
409
+ async ai(taskPrompt, type = 'action') {
410
+ if ('action' === type) return this.aiAction(taskPrompt);
411
+ if ('query' === type) return this.aiQuery(taskPrompt);
412
+ if ('assert' === type) return this.aiAssert(taskPrompt);
413
+ if ('tap' === type) return this.aiTap(taskPrompt);
414
+ if ('rightClick' === type) return this.aiRightClick(taskPrompt);
415
+ throw new Error(`Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'rightClick'`);
416
+ }
417
+ async runYaml(yamlScriptContent) {
418
+ const script = (0, index_js_namespaceObject.parseYamlScript)(yamlScriptContent, 'yaml', true);
419
+ const player = new index_js_namespaceObject.ScriptPlayer(script, async (target)=>({
420
+ agent: this,
421
+ freeFn: []
422
+ }));
423
+ await player.run();
424
+ if ('error' === player.status) {
425
+ const errors = player.taskStatusList.filter((task)=>'error' === task.status).map((task)=>{
426
+ var _task_error;
427
+ return `task - ${task.name}: ${null == (_task_error = task.error) ? void 0 : _task_error.message}`;
428
+ }).join('\n');
429
+ throw new Error(`Error(s) occurred in running yaml script:\n${errors}`);
430
+ }
431
+ return {
432
+ result: player.result
433
+ };
434
+ }
435
+ async evaluateJavaScript(script) {
436
+ (0, utils_namespaceObject.assert)(this.page.evaluateJavaScript, 'evaluateJavaScript is not supported in current agent');
437
+ return this.page.evaluateJavaScript(script);
438
+ }
439
+ async destroy() {
440
+ await this.page.destroy();
441
+ this.resetDump();
442
+ this.destroyed = true;
443
+ }
444
+ async logScreenshot(title, opt) {
445
+ const base64 = await this.page.screenshotBase64();
446
+ const now = Date.now();
447
+ const recorder = [
448
+ {
449
+ type: 'screenshot',
450
+ ts: now,
451
+ screenshot: base64
452
+ }
453
+ ];
454
+ const task = {
455
+ type: 'Log',
456
+ subType: 'Screenshot',
457
+ status: 'finished',
458
+ recorder,
459
+ timing: {
460
+ start: now,
461
+ end: now,
462
+ cost: 0
463
+ },
464
+ param: {
465
+ content: (null == opt ? void 0 : opt.content) || ''
466
+ },
467
+ executor: async ()=>{}
468
+ };
469
+ const executionDump = {
470
+ sdkVersion: '',
471
+ logTime: now,
472
+ name: `Log - ${title || 'untitled'}`,
473
+ description: (null == opt ? void 0 : opt.content) || '',
474
+ tasks: [
475
+ task
476
+ ]
477
+ };
478
+ this.appendExecutionDump(executionDump);
479
+ try {
480
+ var _this_onDumpUpdate, _this;
481
+ null == (_this_onDumpUpdate = (_this = this).onDumpUpdate) || _this_onDumpUpdate.call(_this, this.dumpDataString());
482
+ } catch (error) {
483
+ console.error('Failed to update dump', error);
484
+ }
485
+ this.writeOutActionDumps();
486
+ }
487
+ _unstableLogContent() {
488
+ const { groupName, groupDescription, executions } = this.dump;
489
+ const newExecutions = Array.isArray(executions) ? executions.map((execution)=>{
490
+ const { tasks, ...restExecution } = execution;
491
+ let newTasks = tasks;
492
+ if (Array.isArray(tasks)) newTasks = tasks.map((task)=>{
493
+ const { pageContext, log, ...restTask } = task;
494
+ return restTask;
495
+ });
496
+ return {
497
+ ...restExecution,
498
+ ...newTasks ? {
499
+ tasks: newTasks
500
+ } : {}
501
+ };
502
+ }) : [];
503
+ return {
504
+ groupName,
505
+ groupDescription,
506
+ executions: newExecutions
507
+ };
508
+ }
509
+ async freezePageContext() {
510
+ debug('Freezing page context');
511
+ const context = await this._snapshotContext();
512
+ context._isFrozen = true;
513
+ this.frozenPageContext = context;
514
+ debug('Page context frozen successfully');
515
+ }
516
+ async unfreezePageContext() {
517
+ debug('Unfreezing page context');
518
+ this.frozenPageContext = void 0;
519
+ debug('Page context unfrozen successfully');
520
+ }
521
+ constructor(page, opts){
522
+ _define_property(this, "page", void 0);
523
+ _define_property(this, "insight", void 0);
524
+ _define_property(this, "dump", void 0);
525
+ _define_property(this, "reportFile", void 0);
526
+ _define_property(this, "reportFileName", void 0);
527
+ _define_property(this, "taskExecutor", void 0);
528
+ _define_property(this, "opts", void 0);
529
+ _define_property(this, "dryMode", false);
530
+ _define_property(this, "onTaskStartTip", void 0);
531
+ _define_property(this, "taskCache", void 0);
532
+ _define_property(this, "onDumpUpdate", void 0);
533
+ _define_property(this, "destroyed", false);
534
+ _define_property(this, "frozenPageContext", void 0);
535
+ this.page = page;
536
+ this.opts = Object.assign({
537
+ generateReport: true,
538
+ autoPrintReportMsg: true,
539
+ groupName: 'Midscene Report',
540
+ groupDescription: ''
541
+ }, opts || {});
542
+ if ('function' == typeof (null == opts ? void 0 : opts.modelConfig)) env_namespaceObject.globalConfigManger.registerModelConfigFn(null == opts ? void 0 : opts.modelConfig);
543
+ this.onTaskStartTip = this.opts.onTaskStartTip;
544
+ this.insight = new external_index_js_namespaceObject.Insight(async (action)=>this.getUIContext(action));
545
+ if ((null == opts ? void 0 : opts.cacheId) && 'android' !== this.page.pageType) this.taskCache = new external_task_cache_js_namespaceObject.TaskCache(opts.cacheId, (0, env_namespaceObject.getAIConfigInBoolean)('MIDSCENE_CACHE'));
546
+ this.taskExecutor = new external_tasks_js_namespaceObject.PageTaskExecutor(this.page, this.insight, {
547
+ taskCache: this.taskCache,
548
+ onTaskStart: this.callbackOnTaskStartTip.bind(this)
549
+ });
550
+ this.dump = this.resetDump();
551
+ this.reportFileName = (null == opts ? void 0 : opts.reportFileName) || (0, external_utils_js_namespaceObject_1.getReportFileName)((null == opts ? void 0 : opts.testId) || this.page.pageType || 'web');
552
+ }
553
+ }
554
+ exports.Agent = __webpack_exports__.Agent;
555
+ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
556
+ "Agent"
557
+ ].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
558
+ Object.defineProperty(exports, '__esModule', {
559
+ value: true
560
+ });
561
+
562
+ //# sourceMappingURL=agent.js.map