misoai-web 1.0.3 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +349 -5
  2. package/dist/es/agent.js +2676 -0
  3. package/dist/es/agent.js.map +1 -0
  4. package/dist/es/bridge-mode-browser.js +955 -0
  5. package/dist/es/bridge-mode-browser.js.map +1 -0
  6. package/dist/es/bridge-mode.js +3037 -0
  7. package/dist/es/bridge-mode.js.map +1 -0
  8. package/dist/es/chrome-extension.js +3424 -0
  9. package/dist/es/chrome-extension.js.map +1 -0
  10. package/dist/es/index.js +3292 -0
  11. package/dist/es/index.js.map +1 -0
  12. package/dist/es/midscene-playground.js +3012 -0
  13. package/dist/es/midscene-playground.js.map +1 -0
  14. package/dist/es/midscene-server.js +247 -0
  15. package/dist/es/midscene-server.js.map +1 -0
  16. package/dist/es/playground.js +2783 -0
  17. package/dist/es/playground.js.map +1 -0
  18. package/dist/es/playwright-report.js +120 -0
  19. package/dist/es/playwright-report.js.map +1 -0
  20. package/dist/es/playwright.js +3237 -0
  21. package/dist/es/playwright.js.map +1 -0
  22. package/dist/es/puppeteer-agent-launcher.js +3187 -0
  23. package/dist/es/puppeteer-agent-launcher.js.map +1 -0
  24. package/dist/es/puppeteer.js +3034 -0
  25. package/dist/es/puppeteer.js.map +1 -0
  26. package/dist/es/ui-utils.js +106 -0
  27. package/dist/es/ui-utils.js.map +1 -0
  28. package/dist/es/utils.js +197 -0
  29. package/dist/es/utils.js.map +1 -0
  30. package/dist/es/yaml.js +351 -0
  31. package/dist/es/yaml.js.map +1 -0
  32. package/dist/lib/agent.js +2691 -0
  33. package/dist/lib/agent.js.map +1 -0
  34. package/dist/lib/bridge-mode-browser.js +989 -0
  35. package/dist/lib/bridge-mode-browser.js.map +1 -0
  36. package/dist/lib/bridge-mode.js +3057 -0
  37. package/dist/lib/bridge-mode.js.map +1 -0
  38. package/dist/lib/chrome-extension.js +3441 -0
  39. package/dist/lib/chrome-extension.js.map +1 -0
  40. package/dist/lib/index.js +3308 -0
  41. package/dist/lib/index.js.map +1 -0
  42. package/dist/lib/midscene-playground.js +3016 -0
  43. package/dist/lib/midscene-playground.js.map +1 -0
  44. package/dist/lib/midscene-server.js +273 -0
  45. package/dist/lib/midscene-server.js.map +1 -0
  46. package/dist/lib/playground.js +2802 -0
  47. package/dist/lib/playground.js.map +1 -0
  48. package/dist/lib/playwright-report.js +148 -0
  49. package/dist/lib/playwright-report.js.map +1 -0
  50. package/dist/lib/playwright.js +3254 -0
  51. package/dist/lib/playwright.js.map +1 -0
  52. package/dist/lib/puppeteer-agent-launcher.js +3200 -0
  53. package/dist/lib/puppeteer-agent-launcher.js.map +1 -0
  54. package/dist/lib/puppeteer.js +3045 -0
  55. package/dist/lib/puppeteer.js.map +1 -0
  56. package/dist/lib/ui-utils.js +137 -0
  57. package/dist/lib/ui-utils.js.map +1 -0
  58. package/dist/lib/utils.js +235 -0
  59. package/dist/lib/utils.js.map +1 -0
  60. package/dist/lib/yaml.js +390 -0
  61. package/dist/lib/yaml.js.map +1 -0
  62. package/dist/types/agent.d.ts +290 -0
  63. package/dist/types/bridge-mode-browser.d.ts +9 -0
  64. package/dist/types/bridge-mode.d.ts +40 -0
  65. package/dist/types/browser-a1877d18.d.ts +37 -0
  66. package/dist/types/chrome-extension.d.ts +18 -0
  67. package/dist/types/index.d.ts +16 -0
  68. package/dist/types/midscene-playground.d.ts +2 -0
  69. package/dist/types/midscene-server.d.ts +31 -0
  70. package/dist/types/page-663ece08.d.ts +333 -0
  71. package/dist/types/playground.d.ts +17 -0
  72. package/dist/types/playwright-report.d.ts +11 -0
  73. package/dist/types/playwright.d.ts +87 -0
  74. package/dist/types/puppeteer-agent-launcher.d.ts +40 -0
  75. package/dist/types/puppeteer.d.ts +17 -0
  76. package/dist/types/ui-utils.d.ts +14 -0
  77. package/dist/types/utils-badc824e.d.ts +34 -0
  78. package/dist/types/utils.d.ts +8 -0
  79. package/dist/types/yaml.d.ts +15 -0
  80. package/iife-script/htmlElement.js +99 -37
  81. package/iife-script/htmlElementDebug.js +92 -9
  82. package/package.json +2 -2
@@ -0,0 +1,3045 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/puppeteer/index.ts
31
+ var puppeteer_exports = {};
32
+ __export(puppeteer_exports, {
33
+ PuppeteerAgent: () => PuppeteerAgent,
34
+ PuppeteerWebPage: () => WebPage,
35
+ overrideAIConfig: () => import_env3.overrideAIConfig
36
+ });
37
+ module.exports = __toCommonJS(puppeteer_exports);
38
+
39
+ // src/common/agent.ts
40
+ var import_misoai_core2 = require("misoai-core");
41
+ var import_js_yaml4 = __toESM(require("js-yaml"));
42
+
43
+ // src/yaml/player.ts
44
+ var import_node_fs = require("fs");
45
+ var import_node_path = require("path");
46
+ var import_utils = require("misoai-shared/utils");
47
+ var import_common = require("misoai-shared/common");
48
+ var ScriptPlayer = class {
49
+ constructor(script, setupAgent, onTaskStatusChange) {
50
+ this.script = script;
51
+ this.setupAgent = setupAgent;
52
+ this.onTaskStatusChange = onTaskStatusChange;
53
+ this.taskStatusList = [];
54
+ this.status = "init";
55
+ this.unnamedResultIndex = 0;
56
+ this.pageAgent = null;
57
+ this.result = {};
58
+ if (import_utils.ifInBrowser) {
59
+ this.output = void 0;
60
+ } else if (script.target?.output) {
61
+ this.output = (0, import_node_path.resolve)(process.cwd(), script.target.output);
62
+ } else {
63
+ this.output = (0, import_node_path.join)((0, import_common.getMidsceneRunSubDir)("output"), `${process.pid}.json`);
64
+ }
65
+ this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({
66
+ ...task,
67
+ index: taskIndex,
68
+ status: "init",
69
+ totalSteps: task.flow?.length || 0
70
+ }));
71
+ }
72
+ setResult(key, value) {
73
+ const keyToUse = key || this.unnamedResultIndex++;
74
+ if (this.result[keyToUse]) {
75
+ console.warn(`result key ${keyToUse} already exists, will overwrite`);
76
+ }
77
+ this.result[keyToUse] = value;
78
+ this.flushResult();
79
+ }
80
+ setPlayerStatus(status, error) {
81
+ this.status = status;
82
+ this.errorInSetup = error;
83
+ }
84
+ notifyCurrentTaskStatusChange(taskIndex) {
85
+ const taskIndexToNotify = typeof taskIndex === "number" ? taskIndex : this.currentTaskIndex;
86
+ if (typeof taskIndexToNotify !== "number") {
87
+ return;
88
+ }
89
+ const taskStatus = this.taskStatusList[taskIndexToNotify];
90
+ if (this.onTaskStatusChange) {
91
+ this.onTaskStatusChange(taskStatus);
92
+ }
93
+ }
94
+ async setTaskStatus(index, statusValue, error) {
95
+ this.taskStatusList[index].status = statusValue;
96
+ if (error) {
97
+ this.taskStatusList[index].error = error;
98
+ }
99
+ this.notifyCurrentTaskStatusChange(index);
100
+ }
101
+ setTaskIndex(taskIndex) {
102
+ this.currentTaskIndex = taskIndex;
103
+ }
104
+ flushResult() {
105
+ if (Object.keys(this.result).length && this.output) {
106
+ const output = (0, import_node_path.resolve)(process.cwd(), this.output);
107
+ const outputDir = (0, import_node_path.dirname)(output);
108
+ if (!(0, import_node_fs.existsSync)(outputDir)) {
109
+ (0, import_node_fs.mkdirSync)(outputDir, { recursive: true });
110
+ }
111
+ (0, import_node_fs.writeFileSync)(output, JSON.stringify(this.result, void 0, 2));
112
+ }
113
+ }
114
+ async playTask(taskStatus, agent) {
115
+ const { flow } = taskStatus;
116
+ (0, import_utils.assert)(flow, "missing flow in task");
117
+ for (const flowItemIndex in flow) {
118
+ const currentStep = Number.parseInt(flowItemIndex, 10);
119
+ taskStatus.currentStep = currentStep;
120
+ const flowItem = flow[flowItemIndex];
121
+ if ("aiAction" in flowItem || "ai" in flowItem) {
122
+ const actionTask = flowItem;
123
+ const prompt = actionTask.aiAction || actionTask.ai;
124
+ (0, import_utils.assert)(prompt, "missing prompt for ai (aiAction)");
125
+ (0, import_utils.assert)(
126
+ typeof prompt === "string",
127
+ "prompt for aiAction must be a string"
128
+ );
129
+ await agent.aiAction(prompt, {
130
+ cacheable: actionTask.cacheable
131
+ });
132
+ } else if ("aiAssert" in flowItem) {
133
+ const assertTask = flowItem;
134
+ const prompt = assertTask.aiAssert;
135
+ (0, import_utils.assert)(prompt, "missing prompt for aiAssert");
136
+ (0, import_utils.assert)(
137
+ typeof prompt === "string",
138
+ "prompt for aiAssert must be a string"
139
+ );
140
+ await agent.aiAssert(prompt);
141
+ } else if ("aiQuery" in flowItem) {
142
+ const queryTask = flowItem;
143
+ const prompt = queryTask.aiQuery;
144
+ (0, import_utils.assert)(prompt, "missing prompt for aiQuery");
145
+ (0, import_utils.assert)(
146
+ typeof prompt === "string",
147
+ "prompt for aiQuery must be a string"
148
+ );
149
+ const queryResult = await agent.aiQuery(prompt);
150
+ this.setResult(queryTask.name, queryResult);
151
+ } else if ("aiNumber" in flowItem) {
152
+ const numberTask = flowItem;
153
+ const prompt = numberTask.aiNumber;
154
+ (0, import_utils.assert)(prompt, "missing prompt for number");
155
+ (0, import_utils.assert)(
156
+ typeof prompt === "string",
157
+ "prompt for number must be a string"
158
+ );
159
+ const numberResult = await agent.aiNumber(prompt);
160
+ this.setResult(numberTask.name, numberResult);
161
+ } else if ("aiString" in flowItem) {
162
+ const stringTask = flowItem;
163
+ const prompt = stringTask.aiString;
164
+ (0, import_utils.assert)(prompt, "missing prompt for string");
165
+ (0, import_utils.assert)(
166
+ typeof prompt === "string",
167
+ "prompt for string must be a string"
168
+ );
169
+ const stringResult = await agent.aiString(prompt);
170
+ this.setResult(stringTask.name, stringResult);
171
+ } else if ("aiBoolean" in flowItem) {
172
+ const booleanTask = flowItem;
173
+ const prompt = booleanTask.aiBoolean;
174
+ (0, import_utils.assert)(prompt, "missing prompt for boolean");
175
+ (0, import_utils.assert)(
176
+ typeof prompt === "string",
177
+ "prompt for boolean must be a string"
178
+ );
179
+ const booleanResult = await agent.aiBoolean(prompt);
180
+ this.setResult(booleanTask.name, booleanResult);
181
+ } else if ("aiLocate" in flowItem) {
182
+ const locateTask = flowItem;
183
+ const prompt = locateTask.aiLocate;
184
+ (0, import_utils.assert)(prompt, "missing prompt for aiLocate");
185
+ (0, import_utils.assert)(
186
+ typeof prompt === "string",
187
+ "prompt for aiLocate must be a string"
188
+ );
189
+ const locateResult = await agent.aiLocate(prompt);
190
+ this.setResult(locateTask.name, locateResult);
191
+ } else if ("aiWaitFor" in flowItem) {
192
+ const waitForTask = flowItem;
193
+ const prompt = waitForTask.aiWaitFor;
194
+ (0, import_utils.assert)(prompt, "missing prompt for aiWaitFor");
195
+ (0, import_utils.assert)(
196
+ typeof prompt === "string",
197
+ "prompt for aiWaitFor must be a string"
198
+ );
199
+ const timeout = waitForTask.timeout;
200
+ await agent.aiWaitFor(prompt, { timeoutMs: timeout });
201
+ } else if ("sleep" in flowItem) {
202
+ const sleepTask = flowItem;
203
+ const ms = sleepTask.sleep;
204
+ let msNumber = ms;
205
+ if (typeof ms === "string") {
206
+ msNumber = Number.parseInt(ms, 10);
207
+ }
208
+ (0, import_utils.assert)(
209
+ msNumber && msNumber > 0,
210
+ `ms for sleep must be greater than 0, but got ${ms}`
211
+ );
212
+ await new Promise((resolve2) => setTimeout(resolve2, msNumber));
213
+ } else if ("aiTap" in flowItem) {
214
+ const tapTask = flowItem;
215
+ await agent.aiTap(tapTask.aiTap, tapTask);
216
+ } else if ("aiHover" in flowItem) {
217
+ const hoverTask = flowItem;
218
+ await agent.aiHover(hoverTask.aiHover, hoverTask);
219
+ } else if ("aiInput" in flowItem) {
220
+ const inputTask = flowItem;
221
+ await agent.aiInput(inputTask.aiInput, inputTask.locate, inputTask);
222
+ } else if ("aiKeyboardPress" in flowItem) {
223
+ const keyboardPressTask = flowItem;
224
+ await agent.aiKeyboardPress(
225
+ keyboardPressTask.aiKeyboardPress,
226
+ keyboardPressTask.locate,
227
+ keyboardPressTask
228
+ );
229
+ } else if ("aiScroll" in flowItem) {
230
+ const scrollTask = flowItem;
231
+ await agent.aiScroll(scrollTask, scrollTask.locate, scrollTask);
232
+ } else if ("javascript" in flowItem) {
233
+ const evaluateJavaScriptTask = flowItem;
234
+ const result = await agent.evaluateJavaScript(
235
+ evaluateJavaScriptTask.javascript
236
+ );
237
+ this.setResult(evaluateJavaScriptTask.name, result);
238
+ } else {
239
+ throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
240
+ }
241
+ }
242
+ this.reportFile = agent.reportFile;
243
+ }
244
+ async run() {
245
+ const { target, web, android, tasks } = this.script;
246
+ const webEnv = web || target;
247
+ const androidEnv = android;
248
+ const platform = webEnv || androidEnv;
249
+ this.setPlayerStatus("running");
250
+ let agent = null;
251
+ let freeFn = [];
252
+ try {
253
+ const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(
254
+ platform
255
+ );
256
+ agent = newAgent;
257
+ const originalOnTaskStartTip = agent.onTaskStartTip;
258
+ agent.onTaskStartTip = (tip) => {
259
+ if (this.status === "running") {
260
+ this.agentStatusTip = tip;
261
+ }
262
+ originalOnTaskStartTip?.(tip);
263
+ };
264
+ freeFn = [
265
+ ...newFreeFn || [],
266
+ {
267
+ name: "restore-agent-onTaskStartTip",
268
+ fn: () => {
269
+ if (agent) {
270
+ agent.onTaskStartTip = originalOnTaskStartTip;
271
+ }
272
+ }
273
+ }
274
+ ];
275
+ } catch (e) {
276
+ this.setPlayerStatus("error", e);
277
+ return;
278
+ }
279
+ this.pageAgent = agent;
280
+ let taskIndex = 0;
281
+ this.setPlayerStatus("running");
282
+ let errorFlag = false;
283
+ while (taskIndex < tasks.length) {
284
+ const taskStatus = this.taskStatusList[taskIndex];
285
+ this.setTaskStatus(taskIndex, "running");
286
+ this.setTaskIndex(taskIndex);
287
+ try {
288
+ await this.playTask(taskStatus, this.pageAgent);
289
+ this.setTaskStatus(taskIndex, "done");
290
+ } catch (e) {
291
+ this.setTaskStatus(taskIndex, "error", e);
292
+ if (taskStatus.continueOnError) {
293
+ } else {
294
+ this.reportFile = agent.reportFile;
295
+ errorFlag = true;
296
+ break;
297
+ }
298
+ }
299
+ this.reportFile = agent.reportFile;
300
+ taskIndex++;
301
+ }
302
+ if (errorFlag) {
303
+ this.setPlayerStatus("error");
304
+ } else {
305
+ this.setPlayerStatus("done");
306
+ }
307
+ this.agentStatusTip = "";
308
+ for (const fn of freeFn) {
309
+ try {
310
+ await fn.fn();
311
+ } catch (e) {
312
+ }
313
+ }
314
+ }
315
+ };
316
+
317
+ // src/yaml/builder.ts
318
+ var import_js_yaml = __toESM(require("js-yaml"));
319
+
320
+ // src/yaml/utils.ts
321
+ var import_utils2 = require("misoai-shared/utils");
322
+ var import_js_yaml2 = __toESM(require("js-yaml"));
323
+ function interpolateEnvVars(content) {
324
+ return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
325
+ const value = process.env[envVar.trim()];
326
+ if (value === void 0) {
327
+ throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
328
+ }
329
+ return value;
330
+ });
331
+ }
332
+ function parseYamlScript(content, filePath, ignoreCheckingTarget) {
333
+ let processedContent = content;
334
+ if (content.indexOf("android") !== -1 && content.match(/deviceId:\s*(\d+)/)) {
335
+ let matchedDeviceId;
336
+ processedContent = content.replace(
337
+ /deviceId:\s*(\d+)/g,
338
+ (match, deviceId) => {
339
+ matchedDeviceId = deviceId;
340
+ return `deviceId: '${deviceId}'`;
341
+ }
342
+ );
343
+ console.warn(
344
+ `please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`
345
+ );
346
+ }
347
+ const interpolatedContent = interpolateEnvVars(processedContent);
348
+ const obj = import_js_yaml2.default.load(interpolatedContent, {
349
+ schema: import_js_yaml2.default.JSON_SCHEMA
350
+ });
351
+ const pathTip = filePath ? `, failed to load ${filePath}` : "";
352
+ const android = typeof obj.android !== "undefined" ? Object.assign({}, obj.android || {}) : void 0;
353
+ const webConfig = obj.web || obj.target;
354
+ const web = typeof webConfig !== "undefined" ? Object.assign({}, webConfig || {}) : void 0;
355
+ if (!ignoreCheckingTarget) {
356
+ (0, import_utils2.assert)(
357
+ web || android,
358
+ `at least one of "target", "web", or "android" properties is required in yaml script${pathTip}`
359
+ );
360
+ (0, import_utils2.assert)(
361
+ web && !android || !web && android,
362
+ `only one of "target", "web", or "android" properties is allowed in yaml script${pathTip}`
363
+ );
364
+ if (web || android) {
365
+ (0, import_utils2.assert)(
366
+ typeof web === "object" || typeof android === "object",
367
+ `property "target/web/android" must be an object${pathTip}`
368
+ );
369
+ }
370
+ }
371
+ (0, import_utils2.assert)(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
372
+ (0, import_utils2.assert)(
373
+ Array.isArray(obj.tasks),
374
+ `property "tasks" must be an array in yaml script, but got ${obj.tasks}`
375
+ );
376
+ return obj;
377
+ }
378
+
379
+ // src/common/agent.ts
380
+ var import_utils11 = require("misoai-core/utils");
381
+ var import_constants2 = require("misoai-shared/constants");
382
+ var import_env2 = require("misoai-shared/env");
383
+ var import_logger4 = require("misoai-shared/logger");
384
+ var import_utils12 = require("misoai-shared/utils");
385
+
386
+ // src/common/tasks.ts
387
+ var import_misoai_core = require("misoai-core");
388
+ var import_ai_model2 = require("misoai-core/ai-model");
389
+ var import_utils5 = require("misoai-core/utils");
390
+ var import_constants = require("misoai-shared/constants");
391
+ var import_logger = require("misoai-shared/logger");
392
+ var import_utils6 = require("misoai-shared/utils");
393
+
394
+ // src/common/ui-utils.ts
395
+ function typeStr(task) {
396
+ return task.subType && task.subType !== "Plan" ? `${task.type} / ${task.subType || ""}` : task.type;
397
+ }
398
+ function getKeyCommands(value) {
399
+ const keys = Array.isArray(value) ? value : [value];
400
+ return keys.reduce((acc, k) => {
401
+ const includeMeta = keys.includes("Meta") || keys.includes("Control");
402
+ if (includeMeta && (k === "a" || k === "A")) {
403
+ return acc.concat([{ key: k, command: "SelectAll" }]);
404
+ }
405
+ if (includeMeta && (k === "c" || k === "C")) {
406
+ return acc.concat([{ key: k, command: "Copy" }]);
407
+ }
408
+ if (includeMeta && (k === "v" || k === "V")) {
409
+ return acc.concat([{ key: k, command: "Paste" }]);
410
+ }
411
+ return acc.concat([{ key: k }]);
412
+ }, []);
413
+ }
414
+ function locateParamStr(locate) {
415
+ if (!locate) {
416
+ return "";
417
+ }
418
+ if (typeof locate === "string") {
419
+ return locate;
420
+ }
421
+ return locate.prompt;
422
+ }
423
+ function scrollParamStr(scrollParam) {
424
+ if (!scrollParam) {
425
+ return "";
426
+ }
427
+ return `${scrollParam.direction || "down"}, ${scrollParam.scrollType || "once"}, ${scrollParam.distance || "distance-not-set"}`;
428
+ }
429
+ function taskTitleStr(type, prompt) {
430
+ if (prompt) {
431
+ return `${type} - ${prompt}`;
432
+ }
433
+ return type;
434
+ }
435
+ function paramStr(task) {
436
+ let value;
437
+ if (task.type === "Planning") {
438
+ value = task?.param?.userInstruction;
439
+ }
440
+ if (task.type === "Insight") {
441
+ value = task?.param?.prompt || task?.param?.id || task?.param?.dataDemand || task?.param?.assertion;
442
+ }
443
+ if (task.type === "Action") {
444
+ const locate = task?.locate;
445
+ const locateStr = locate ? locateParamStr(locate) : "";
446
+ value = task.thought || "";
447
+ if (typeof task?.param?.timeMs === "number") {
448
+ value = `${task?.param?.timeMs}ms`;
449
+ } else if (typeof task?.param?.scrollType === "string") {
450
+ value = scrollParamStr(task?.param);
451
+ } else if (typeof task?.param?.value !== "undefined") {
452
+ value = task?.param?.value;
453
+ }
454
+ if (locateStr) {
455
+ if (value) {
456
+ value = `${locateStr} - ${value}`;
457
+ } else {
458
+ value = locateStr;
459
+ }
460
+ }
461
+ }
462
+ if (typeof value === "undefined")
463
+ return "";
464
+ return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
465
+ }
466
+
467
+ // src/common/utils.ts
468
+ var import_ai_model = require("misoai-core/ai-model");
469
+ var import_utils3 = require("misoai-core/utils");
470
+ var import_env = require("misoai-shared/env");
471
+ var import_extractor = require("misoai-shared/extractor");
472
+ var import_img = require("misoai-shared/img");
473
+ var import_utils4 = require("misoai-shared/utils");
474
+ var import_dayjs = __toESM(require("dayjs"));
475
+
476
+ // src/web-element.ts
477
+ var WebElementInfo = class {
478
+ constructor({
479
+ content,
480
+ rect,
481
+ // page,
482
+ locator,
483
+ id,
484
+ attributes,
485
+ indexId,
486
+ xpaths
487
+ }) {
488
+ this.content = content;
489
+ this.rect = rect;
490
+ this.center = [
491
+ Math.floor(rect.left + rect.width / 2),
492
+ Math.floor(rect.top + rect.height / 2)
493
+ ];
494
+ this.locator = locator;
495
+ this.id = id;
496
+ this.attributes = attributes;
497
+ this.indexId = indexId;
498
+ this.xpaths = xpaths;
499
+ }
500
+ };
501
+
502
+ // src/common/utils.ts
503
+ async function parseContextFromWebPage(page, _opt) {
504
+ (0, import_utils4.assert)(page, "page is required");
505
+ if (page._forceUsePageContext) {
506
+ return await page._forceUsePageContext();
507
+ }
508
+ const url = await page.url();
509
+ (0, import_utils3.uploadTestInfoToServer)({ testUrl: url });
510
+ let screenshotBase64;
511
+ let tree;
512
+ await Promise.all([
513
+ page.screenshotBase64().then((base64) => {
514
+ screenshotBase64 = base64;
515
+ }),
516
+ page.getElementsNodeTree().then(async (treeRoot) => {
517
+ tree = treeRoot;
518
+ })
519
+ ]);
520
+ const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
521
+ const { rect, id, content, attributes, locator, indexId } = elementInfo;
522
+ return new WebElementInfo({
523
+ rect,
524
+ locator,
525
+ id,
526
+ content,
527
+ attributes,
528
+ indexId
529
+ });
530
+ });
531
+ (0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
532
+ const elementsInfo = (0, import_extractor.treeToList)(webTree);
533
+ const size = await page.size();
534
+ if (size.dpr && size.dpr > 1) {
535
+ screenshotBase64 = await (0, import_img.resizeImgBase64)(screenshotBase64, {
536
+ width: size.width,
537
+ height: size.height
538
+ });
539
+ }
540
+ return {
541
+ content: elementsInfo,
542
+ tree: webTree,
543
+ size,
544
+ screenshotBase64,
545
+ url
546
+ };
547
+ }
548
+ function reportFileName(tag = "web") {
549
+ const reportTagName = (0, import_env.getAIConfig)(import_env.MIDSCENE_REPORT_TAG_NAME);
550
+ const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss");
551
+ const uniqueId = (0, import_utils4.uuid)().substring(0, 8);
552
+ return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
553
+ }
554
+ function printReportMsg(filepath) {
555
+ (0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
556
+ }
557
+ function replaceIllegalPathCharsAndSpace(str) {
558
+ return str.replace(/[/\\:*?"<>| ]/g, "-");
559
+ }
560
+ function forceClosePopup(page, debug6) {
561
+ page.on("popup", async (popup) => {
562
+ if (!popup) {
563
+ console.warn("got a popup event, but the popup is not ready yet, skip");
564
+ return;
565
+ }
566
+ const url = await popup.url();
567
+ console.log(`Popup opened: ${url}`);
568
+ if (!popup.isClosed()) {
569
+ try {
570
+ await popup.close();
571
+ } catch (error) {
572
+ debug6(`failed to close popup ${url}, error: ${error}`);
573
+ }
574
+ } else {
575
+ debug6(`popup is already closed, skip close ${url}`);
576
+ }
577
+ if (!page.isClosed()) {
578
+ try {
579
+ await page.goto(url);
580
+ } catch (error) {
581
+ debug6(`failed to goto ${url}, error: ${error}`);
582
+ }
583
+ } else {
584
+ debug6(`page is already closed, skip goto ${url}`);
585
+ }
586
+ });
587
+ }
588
+ function matchElementFromPlan(planLocateParam, tree) {
589
+ if (!planLocateParam) {
590
+ return void 0;
591
+ }
592
+ if (planLocateParam.id) {
593
+ return (0, import_extractor.getNodeFromCacheList)(planLocateParam.id);
594
+ }
595
+ if (planLocateParam.bbox) {
596
+ const centerPosition = {
597
+ x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
598
+ y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
599
+ };
600
+ let element = (0, import_ai_model.elementByPositionWithElementInfo)(tree, centerPosition);
601
+ if (!element) {
602
+ element = (0, import_extractor.generateElementByPosition)(centerPosition);
603
+ }
604
+ return element;
605
+ }
606
+ return void 0;
607
+ }
608
+
609
+ // src/common/tasks.ts
610
+ var debug = (0, import_logger.getDebug)("page-task-executor");
611
+ var replanningCountLimit = 10;
612
+ var isAndroidPage = (page) => {
613
+ return page.pageType === "android";
614
+ };
615
+ var PageTaskExecutor = class {
616
+ constructor(page, insight, opts) {
617
+ this.conversationHistory = [];
618
+ this.page = page;
619
+ this.insight = insight;
620
+ this.taskCache = opts.taskCache;
621
+ this.onTaskStartCallback = opts?.onTaskStart;
622
+ }
623
+ async recordScreenshot(timing) {
624
+ const base64 = await this.page.screenshotBase64();
625
+ const item = {
626
+ type: "screenshot",
627
+ ts: Date.now(),
628
+ screenshot: base64,
629
+ timing
630
+ };
631
+ return item;
632
+ }
633
+ async getElementXpath(pageContext, element) {
634
+ let elementId = element?.id;
635
+ if (element?.attributes?.nodeType === import_constants.NodeType.POSITION) {
636
+ await this.insight.contextRetrieverFn("locate");
637
+ const info = (0, import_ai_model2.elementByPositionWithElementInfo)(
638
+ pageContext.tree,
639
+ {
640
+ x: element.center[0],
641
+ y: element.center[1]
642
+ },
643
+ {
644
+ requireStrictDistance: false,
645
+ filterPositionElements: true
646
+ }
647
+ );
648
+ if (info?.id) {
649
+ elementId = info.id;
650
+ } else {
651
+ debug(
652
+ "no element id found for position node, will not update cache",
653
+ element
654
+ );
655
+ }
656
+ }
657
+ if (!elementId) {
658
+ return void 0;
659
+ }
660
+ try {
661
+ const result = await this.page.getXpathsById(elementId);
662
+ return result;
663
+ } catch (error) {
664
+ debug("getXpathsById error: ", error);
665
+ }
666
+ }
667
+ prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
668
+ const taskWithScreenshot = {
669
+ ...taskApply,
670
+ executor: async (param, context, ...args) => {
671
+ const recorder = [];
672
+ const { task } = context;
673
+ task.recorder = recorder;
674
+ const shot = await this.recordScreenshot(`before ${task.type}`);
675
+ recorder.push(shot);
676
+ const result = await taskApply.executor(param, context, ...args);
677
+ if (taskApply.type === "Action") {
678
+ await Promise.all([
679
+ (async () => {
680
+ await (0, import_utils5.sleep)(100);
681
+ if (this.page.waitUntilNetworkIdle) {
682
+ try {
683
+ await this.page.waitUntilNetworkIdle();
684
+ } catch (error) {
685
+ }
686
+ }
687
+ })(),
688
+ (0, import_utils5.sleep)(200)
689
+ ]);
690
+ }
691
+ if (appendAfterExecution) {
692
+ const shot2 = await this.recordScreenshot("after Action");
693
+ recorder.push(shot2);
694
+ }
695
+ return result;
696
+ }
697
+ };
698
+ return taskWithScreenshot;
699
+ }
700
+ async convertPlanToExecutable(plans, opts) {
701
+ const tasks = [];
702
+ plans.forEach((plan2) => {
703
+ if (plan2.type === "Locate") {
704
+ if (plan2.locate === null || plan2.locate?.id === null || plan2.locate?.id === "null") {
705
+ return;
706
+ }
707
+ const taskFind = {
708
+ type: "Insight",
709
+ subType: "Locate",
710
+ param: plan2.locate ? {
711
+ ...plan2.locate,
712
+ cacheable: opts?.cacheable
713
+ } : void 0,
714
+ thought: plan2.thought,
715
+ locate: plan2.locate,
716
+ executor: async (param, taskContext) => {
717
+ const { task } = taskContext;
718
+ (0, import_utils6.assert)(
719
+ param?.prompt || param?.id || param?.bbox,
720
+ "No prompt or id or position or bbox to locate"
721
+ );
722
+ let insightDump;
723
+ let usage;
724
+ const dumpCollector = (dump) => {
725
+ insightDump = dump;
726
+ usage = dump?.taskInfo?.usage;
727
+ task.log = {
728
+ dump: insightDump
729
+ };
730
+ task.usage = usage;
731
+ };
732
+ this.insight.onceDumpUpdatedFn = dumpCollector;
733
+ const shotTime = Date.now();
734
+ const pageContext = await this.insight.contextRetrieverFn("locate");
735
+ task.pageContext = pageContext;
736
+ const recordItem = {
737
+ type: "screenshot",
738
+ ts: shotTime,
739
+ screenshot: pageContext.screenshotBase64,
740
+ timing: "before locate"
741
+ };
742
+ task.recorder = [recordItem];
743
+ let cacheHitFlag = false;
744
+ const cachePrompt = param.prompt;
745
+ const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
746
+ const xpaths = locateCacheRecord?.cacheContent?.xpaths;
747
+ let elementFromCache = null;
748
+ try {
749
+ if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
750
+ for (let i = 0; i < xpaths.length; i++) {
751
+ const element2 = await this.page.getElementInfoByXpath(
752
+ xpaths[i]
753
+ );
754
+ if (element2?.id) {
755
+ elementFromCache = element2;
756
+ debug("cache hit, prompt: %s", cachePrompt);
757
+ cacheHitFlag = true;
758
+ debug(
759
+ "found a new new element with same xpath, xpath: %s, id: %s",
760
+ xpaths[i],
761
+ element2?.id
762
+ );
763
+ break;
764
+ }
765
+ }
766
+ }
767
+ } catch (error) {
768
+ debug("get element info by xpath error: ", error);
769
+ }
770
+ const startTime = Date.now();
771
+ const element = elementFromCache || // try to match element from cache
772
+ matchElementFromPlan(param, pageContext.tree) || // try to match element from plan
773
+ (await this.insight.locate(param, {
774
+ context: pageContext
775
+ })).element;
776
+ const aiCost = Date.now() - startTime;
777
+ let currentXpaths;
778
+ if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
779
+ const elementXpaths = await this.getElementXpath(
780
+ pageContext,
781
+ element
782
+ );
783
+ if (elementXpaths?.length) {
784
+ currentXpaths = elementXpaths;
785
+ this.taskCache.updateOrAppendCacheRecord(
786
+ {
787
+ type: "locate",
788
+ prompt: cachePrompt,
789
+ xpaths: elementXpaths
790
+ },
791
+ locateCacheRecord
792
+ );
793
+ } else {
794
+ debug(
795
+ "no xpaths found, will not update cache",
796
+ cachePrompt,
797
+ elementXpaths
798
+ );
799
+ }
800
+ }
801
+ if (!element) {
802
+ throw new Error(`Element not found: ${param.prompt}`);
803
+ }
804
+ return {
805
+ output: {
806
+ element
807
+ },
808
+ pageContext,
809
+ cache: {
810
+ hit: cacheHitFlag,
811
+ originalXpaths: xpaths,
812
+ currentXpaths
813
+ },
814
+ aiCost
815
+ };
816
+ }
817
+ };
818
+ tasks.push(taskFind);
819
+ } else if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
820
+ const assertPlan = plan2;
821
+ const taskAssert = {
822
+ type: "Insight",
823
+ subType: "Assert",
824
+ param: assertPlan.param,
825
+ thought: assertPlan.thought,
826
+ locate: assertPlan.locate,
827
+ executor: async (param, taskContext) => {
828
+ const { task } = taskContext;
829
+ let insightDump;
830
+ const dumpCollector = (dump) => {
831
+ insightDump = dump;
832
+ };
833
+ this.insight.onceDumpUpdatedFn = dumpCollector;
834
+ const assertion = await this.insight.assert(
835
+ assertPlan.param.assertion
836
+ );
837
+ if (!assertion.pass) {
838
+ if (plan2.type === "Assert") {
839
+ task.output = assertion;
840
+ task.log = {
841
+ dump: insightDump
842
+ };
843
+ throw new Error(
844
+ assertion.thought || "Assertion failed without reason"
845
+ );
846
+ }
847
+ task.error = assertion.thought;
848
+ }
849
+ return {
850
+ output: assertion,
851
+ log: {
852
+ dump: insightDump
853
+ },
854
+ usage: assertion.usage
855
+ };
856
+ }
857
+ };
858
+ tasks.push(taskAssert);
859
+ } else if (plan2.type === "Input") {
860
+ const taskActionInput = {
861
+ type: "Action",
862
+ subType: "Input",
863
+ param: plan2.param,
864
+ thought: plan2.thought,
865
+ locate: plan2.locate,
866
+ executor: async (taskParam, { element }) => {
867
+ if (element) {
868
+ await this.page.clearInput(element);
869
+ if (!taskParam || !taskParam.value) {
870
+ return;
871
+ }
872
+ await this.page.keyboard.type(taskParam.value);
873
+ } else {
874
+ await this.page.keyboard.type(taskParam.value);
875
+ }
876
+ }
877
+ };
878
+ tasks.push(taskActionInput);
879
+ } else if (plan2.type === "KeyboardPress") {
880
+ const taskActionKeyboardPress = {
881
+ type: "Action",
882
+ subType: "KeyboardPress",
883
+ param: plan2.param,
884
+ thought: plan2.thought,
885
+ locate: plan2.locate,
886
+ executor: async (taskParam) => {
887
+ const keys = getKeyCommands(taskParam.value);
888
+ await this.page.keyboard.press(keys);
889
+ }
890
+ };
891
+ tasks.push(taskActionKeyboardPress);
892
+ } else if (plan2.type === "Tap") {
893
+ const taskActionTap = {
894
+ type: "Action",
895
+ subType: "Tap",
896
+ thought: plan2.thought,
897
+ locate: plan2.locate,
898
+ executor: async (param, { element }) => {
899
+ (0, import_utils6.assert)(element, "Element not found, cannot tap");
900
+ await this.page.mouse.click(element.center[0], element.center[1]);
901
+ }
902
+ };
903
+ tasks.push(taskActionTap);
904
+ } else if (plan2.type === "Drag") {
905
+ const taskActionDrag = {
906
+ type: "Action",
907
+ subType: "Drag",
908
+ param: plan2.param,
909
+ thought: plan2.thought,
910
+ locate: plan2.locate,
911
+ executor: async (taskParam) => {
912
+ (0, import_utils6.assert)(
913
+ taskParam?.start_box && taskParam?.end_box,
914
+ "No start_box or end_box to drag"
915
+ );
916
+ await this.page.mouse.drag(taskParam.start_box, taskParam.end_box);
917
+ }
918
+ };
919
+ tasks.push(taskActionDrag);
920
+ } else if (plan2.type === "Hover") {
921
+ const taskActionHover = {
922
+ type: "Action",
923
+ subType: "Hover",
924
+ thought: plan2.thought,
925
+ locate: plan2.locate,
926
+ executor: async (param, { element }) => {
927
+ (0, import_utils6.assert)(element, "Element not found, cannot hover");
928
+ await this.page.mouse.move(element.center[0], element.center[1]);
929
+ }
930
+ };
931
+ tasks.push(taskActionHover);
932
+ } else if (plan2.type === "Scroll") {
933
+ const taskActionScroll = {
934
+ type: "Action",
935
+ subType: "Scroll",
936
+ param: plan2.param,
937
+ thought: plan2.thought,
938
+ locate: plan2.locate,
939
+ executor: async (taskParam, { element }) => {
940
+ const startingPoint = element ? {
941
+ left: element.center[0],
942
+ top: element.center[1]
943
+ } : void 0;
944
+ const scrollToEventName = taskParam?.scrollType;
945
+ if (scrollToEventName === "untilTop") {
946
+ await this.page.scrollUntilTop(startingPoint);
947
+ } else if (scrollToEventName === "untilBottom") {
948
+ await this.page.scrollUntilBottom(startingPoint);
949
+ } else if (scrollToEventName === "untilRight") {
950
+ await this.page.scrollUntilRight(startingPoint);
951
+ } else if (scrollToEventName === "untilLeft") {
952
+ await this.page.scrollUntilLeft(startingPoint);
953
+ } else if (scrollToEventName === "once" || !scrollToEventName) {
954
+ if (taskParam?.direction === "down" || !taskParam || !taskParam.direction) {
955
+ await this.page.scrollDown(
956
+ taskParam?.distance || void 0,
957
+ startingPoint
958
+ );
959
+ } else if (taskParam.direction === "up") {
960
+ await this.page.scrollUp(
961
+ taskParam.distance || void 0,
962
+ startingPoint
963
+ );
964
+ } else if (taskParam.direction === "left") {
965
+ await this.page.scrollLeft(
966
+ taskParam.distance || void 0,
967
+ startingPoint
968
+ );
969
+ } else if (taskParam.direction === "right") {
970
+ await this.page.scrollRight(
971
+ taskParam.distance || void 0,
972
+ startingPoint
973
+ );
974
+ } else {
975
+ throw new Error(
976
+ `Unknown scroll direction: ${taskParam.direction}`
977
+ );
978
+ }
979
+ await (0, import_utils5.sleep)(500);
980
+ } else {
981
+ throw new Error(
982
+ `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
983
+ taskParam
984
+ )}`
985
+ );
986
+ }
987
+ }
988
+ };
989
+ tasks.push(taskActionScroll);
990
+ } else if (plan2.type === "Sleep") {
991
+ const taskActionSleep = {
992
+ type: "Action",
993
+ subType: "Sleep",
994
+ param: plan2.param,
995
+ thought: plan2.thought,
996
+ locate: plan2.locate,
997
+ executor: async (taskParam) => {
998
+ await (0, import_utils5.sleep)(taskParam?.timeMs || 3e3);
999
+ }
1000
+ };
1001
+ tasks.push(taskActionSleep);
1002
+ } else if (plan2.type === "Error") {
1003
+ const taskActionError = {
1004
+ type: "Action",
1005
+ subType: "Error",
1006
+ param: plan2.param,
1007
+ thought: plan2.thought || plan2.param?.thought,
1008
+ locate: plan2.locate,
1009
+ executor: async () => {
1010
+ throw new Error(
1011
+ plan2?.thought || plan2.param?.thought || "error without thought"
1012
+ );
1013
+ }
1014
+ };
1015
+ tasks.push(taskActionError);
1016
+ } else if (plan2.type === "ExpectedFalsyCondition") {
1017
+ const taskActionFalsyConditionStatement = {
1018
+ type: "Action",
1019
+ subType: "ExpectedFalsyCondition",
1020
+ param: null,
1021
+ thought: plan2.param?.reason,
1022
+ locate: plan2.locate,
1023
+ executor: async () => {
1024
+ }
1025
+ };
1026
+ tasks.push(taskActionFalsyConditionStatement);
1027
+ } else if (plan2.type === "Finished") {
1028
+ const taskActionFinished = {
1029
+ type: "Action",
1030
+ subType: "Finished",
1031
+ param: null,
1032
+ thought: plan2.thought,
1033
+ locate: plan2.locate,
1034
+ executor: async (param) => {
1035
+ }
1036
+ };
1037
+ tasks.push(taskActionFinished);
1038
+ } else if (plan2.type === "AndroidHomeButton") {
1039
+ const taskActionAndroidHomeButton = {
1040
+ type: "Action",
1041
+ subType: "AndroidHomeButton",
1042
+ param: null,
1043
+ thought: plan2.thought,
1044
+ locate: plan2.locate,
1045
+ executor: async (param) => {
1046
+ (0, import_utils6.assert)(
1047
+ isAndroidPage(this.page),
1048
+ "Cannot use home button on non-Android devices"
1049
+ );
1050
+ await this.page.home();
1051
+ }
1052
+ };
1053
+ tasks.push(taskActionAndroidHomeButton);
1054
+ } else if (plan2.type === "AndroidBackButton") {
1055
+ const taskActionAndroidBackButton = {
1056
+ type: "Action",
1057
+ subType: "AndroidBackButton",
1058
+ param: null,
1059
+ thought: plan2.thought,
1060
+ locate: plan2.locate,
1061
+ executor: async (param) => {
1062
+ (0, import_utils6.assert)(
1063
+ isAndroidPage(this.page),
1064
+ "Cannot use back button on non-Android devices"
1065
+ );
1066
+ await this.page.back();
1067
+ }
1068
+ };
1069
+ tasks.push(taskActionAndroidBackButton);
1070
+ } else if (plan2.type === "AndroidRecentAppsButton") {
1071
+ const taskActionAndroidRecentAppsButton = {
1072
+ type: "Action",
1073
+ subType: "AndroidRecentAppsButton",
1074
+ param: null,
1075
+ thought: plan2.thought,
1076
+ locate: plan2.locate,
1077
+ executor: async (param) => {
1078
+ (0, import_utils6.assert)(
1079
+ isAndroidPage(this.page),
1080
+ "Cannot use recent apps button on non-Android devices"
1081
+ );
1082
+ await this.page.recentApps();
1083
+ }
1084
+ };
1085
+ tasks.push(taskActionAndroidRecentAppsButton);
1086
+ } else {
1087
+ throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
1088
+ }
1089
+ });
1090
+ const wrappedTasks = tasks.map(
1091
+ (task, index) => {
1092
+ if (task.type === "Action") {
1093
+ return this.prependExecutorWithScreenshot(
1094
+ task,
1095
+ index === tasks.length - 1
1096
+ );
1097
+ }
1098
+ return task;
1099
+ }
1100
+ );
1101
+ return {
1102
+ tasks: wrappedTasks
1103
+ };
1104
+ }
1105
+ async setupPlanningContext(executorContext) {
1106
+ const shotTime = Date.now();
1107
+ const pageContext = await this.insight.contextRetrieverFn("locate");
1108
+ const recordItem = {
1109
+ type: "screenshot",
1110
+ ts: shotTime,
1111
+ screenshot: pageContext.screenshotBase64,
1112
+ timing: "before planning"
1113
+ };
1114
+ executorContext.task.recorder = [recordItem];
1115
+ executorContext.task.pageContext = pageContext;
1116
+ return {
1117
+ pageContext
1118
+ };
1119
+ }
1120
+ async loadYamlFlowAsPlanning(userInstruction, yamlString) {
1121
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userInstruction), {
1122
+ onTaskStart: this.onTaskStartCallback
1123
+ });
1124
+ const task = {
1125
+ type: "Planning",
1126
+ subType: "LoadYaml",
1127
+ locate: null,
1128
+ param: {
1129
+ userInstruction
1130
+ },
1131
+ executor: async (param, executorContext) => {
1132
+ await this.setupPlanningContext(executorContext);
1133
+ return {
1134
+ output: {
1135
+ actions: [],
1136
+ more_actions_needed_by_instruction: false,
1137
+ log: "",
1138
+ yamlString
1139
+ },
1140
+ cache: {
1141
+ hit: true
1142
+ }
1143
+ };
1144
+ }
1145
+ };
1146
+ await taskExecutor.append(task);
1147
+ await taskExecutor.flush();
1148
+ return {
1149
+ executor: taskExecutor
1150
+ };
1151
+ }
1152
+ planningTaskFromPrompt(userInstruction, log, actionContext) {
1153
+ const task = {
1154
+ type: "Planning",
1155
+ subType: "Plan",
1156
+ locate: null,
1157
+ param: {
1158
+ userInstruction,
1159
+ log
1160
+ },
1161
+ executor: async (param, executorContext) => {
1162
+ const startTime = Date.now();
1163
+ const { pageContext } = await this.setupPlanningContext(executorContext);
1164
+ const planResult = await (0, import_misoai_core.plan)(param.userInstruction, {
1165
+ context: pageContext,
1166
+ log: param.log,
1167
+ actionContext,
1168
+ pageType: this.page.pageType
1169
+ });
1170
+ const {
1171
+ actions,
1172
+ log: log2,
1173
+ more_actions_needed_by_instruction,
1174
+ error,
1175
+ usage,
1176
+ rawResponse,
1177
+ sleep: sleep3
1178
+ } = planResult;
1179
+ executorContext.task.log = {
1180
+ ...executorContext.task.log || {},
1181
+ rawResponse
1182
+ };
1183
+ executorContext.task.usage = usage;
1184
+ let stopCollecting = false;
1185
+ let bboxCollected = false;
1186
+ let planParsingError = "";
1187
+ const finalActions = (actions || []).reduce(
1188
+ (acc, planningAction) => {
1189
+ if (stopCollecting) {
1190
+ return acc;
1191
+ }
1192
+ if (planningAction.locate) {
1193
+ if (bboxCollected && planningAction.locate.bbox) {
1194
+ delete planningAction.locate.bbox;
1195
+ }
1196
+ if (planningAction.locate.bbox) {
1197
+ bboxCollected = true;
1198
+ }
1199
+ acc.push({
1200
+ type: "Locate",
1201
+ locate: planningAction.locate,
1202
+ param: null,
1203
+ thought: planningAction.locate.prompt
1204
+ });
1205
+ } else if (["Tap", "Hover", "Input"].includes(planningAction.type)) {
1206
+ planParsingError = `invalid planning response: ${JSON.stringify(planningAction)}`;
1207
+ stopCollecting = true;
1208
+ return acc;
1209
+ }
1210
+ acc.push(planningAction);
1211
+ return acc;
1212
+ },
1213
+ []
1214
+ );
1215
+ if (sleep3) {
1216
+ const timeNow = Date.now();
1217
+ const timeRemaining = sleep3 - (timeNow - startTime);
1218
+ if (timeRemaining > 0) {
1219
+ finalActions.push({
1220
+ type: "Sleep",
1221
+ param: {
1222
+ timeMs: timeRemaining
1223
+ },
1224
+ locate: null
1225
+ });
1226
+ }
1227
+ }
1228
+ if (finalActions.length === 0) {
1229
+ (0, import_utils6.assert)(
1230
+ !more_actions_needed_by_instruction || sleep3,
1231
+ error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
1232
+ );
1233
+ }
1234
+ return {
1235
+ output: {
1236
+ actions: finalActions,
1237
+ more_actions_needed_by_instruction,
1238
+ log: log2,
1239
+ yamlFlow: planResult.yamlFlow
1240
+ },
1241
+ cache: {
1242
+ hit: false
1243
+ },
1244
+ pageContext
1245
+ };
1246
+ }
1247
+ };
1248
+ return task;
1249
+ }
1250
+ planningTaskToGoal(userInstruction) {
1251
+ const task = {
1252
+ type: "Planning",
1253
+ subType: "Plan",
1254
+ locate: null,
1255
+ param: {
1256
+ userInstruction
1257
+ },
1258
+ executor: async (param, executorContext) => {
1259
+ const { pageContext } = await this.setupPlanningContext(executorContext);
1260
+ const imagePayload = await (0, import_ai_model2.resizeImageForUiTars)(
1261
+ pageContext.screenshotBase64,
1262
+ pageContext.size
1263
+ );
1264
+ this.appendConversationHistory({
1265
+ role: "user",
1266
+ content: [
1267
+ {
1268
+ type: "image_url",
1269
+ image_url: {
1270
+ url: imagePayload
1271
+ }
1272
+ }
1273
+ ]
1274
+ });
1275
+ const startTime = Date.now();
1276
+ const planResult = await (0, import_ai_model2.vlmPlanning)({
1277
+ userInstruction: param.userInstruction,
1278
+ conversationHistory: this.conversationHistory,
1279
+ size: pageContext.size
1280
+ });
1281
+ const aiCost = Date.now() - startTime;
1282
+ const { actions, action_summary } = planResult;
1283
+ this.appendConversationHistory({
1284
+ role: "assistant",
1285
+ content: action_summary
1286
+ });
1287
+ return {
1288
+ output: {
1289
+ actions,
1290
+ thought: actions[0]?.thought,
1291
+ actionType: actions[0].type,
1292
+ more_actions_needed_by_instruction: true,
1293
+ log: "",
1294
+ yamlFlow: planResult.yamlFlow
1295
+ },
1296
+ cache: {
1297
+ hit: false
1298
+ },
1299
+ aiCost
1300
+ };
1301
+ }
1302
+ };
1303
+ return task;
1304
+ }
1305
+ async runPlans(title, plans, opts) {
1306
+ const taskExecutor = new import_misoai_core.Executor(title, {
1307
+ onTaskStart: this.onTaskStartCallback
1308
+ });
1309
+ const { tasks } = await this.convertPlanToExecutable(plans, opts);
1310
+ await taskExecutor.append(tasks);
1311
+ const result = await taskExecutor.flush();
1312
+ return {
1313
+ output: result,
1314
+ executor: taskExecutor
1315
+ };
1316
+ }
1317
+ async action(userPrompt, actionContext, opts) {
1318
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
1319
+ onTaskStart: this.onTaskStartCallback
1320
+ });
1321
+ let planningTask = this.planningTaskFromPrompt(userPrompt, void 0, actionContext);
1322
+ let replanCount = 0;
1323
+ const logList = [];
1324
+ const yamlFlow = [];
1325
+ while (planningTask) {
1326
+ if (replanCount > replanningCountLimit) {
1327
+ const errorMsg = "Replanning too many times, please split the task into multiple steps";
1328
+ return this.appendErrorPlan(taskExecutor, errorMsg);
1329
+ }
1330
+ await taskExecutor.append(planningTask);
1331
+ const planResult = await taskExecutor.flush();
1332
+ if (taskExecutor.isInErrorState()) {
1333
+ return {
1334
+ output: planResult,
1335
+ executor: taskExecutor
1336
+ };
1337
+ }
1338
+ const plans = planResult.actions || [];
1339
+ yamlFlow.push(...planResult.yamlFlow || []);
1340
+ let executables;
1341
+ try {
1342
+ executables = await this.convertPlanToExecutable(plans, opts);
1343
+ taskExecutor.append(executables.tasks);
1344
+ } catch (error) {
1345
+ return this.appendErrorPlan(
1346
+ taskExecutor,
1347
+ `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
1348
+ plans
1349
+ )}`
1350
+ );
1351
+ }
1352
+ await taskExecutor.flush();
1353
+ if (taskExecutor.isInErrorState()) {
1354
+ return {
1355
+ output: void 0,
1356
+ executor: taskExecutor
1357
+ };
1358
+ }
1359
+ if (planResult?.log) {
1360
+ logList.push(planResult.log);
1361
+ }
1362
+ if (!planResult.more_actions_needed_by_instruction) {
1363
+ planningTask = null;
1364
+ break;
1365
+ }
1366
+ planningTask = this.planningTaskFromPrompt(
1367
+ userPrompt,
1368
+ logList.length > 0 ? `- ${logList.join("\n- ")}` : void 0,
1369
+ actionContext
1370
+ );
1371
+ replanCount++;
1372
+ }
1373
+ return {
1374
+ output: {
1375
+ yamlFlow
1376
+ },
1377
+ executor: taskExecutor
1378
+ };
1379
+ }
1380
+ async actionToGoal(userPrompt, opts) {
1381
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
1382
+ onTaskStart: this.onTaskStartCallback
1383
+ });
1384
+ this.conversationHistory = [];
1385
+ const isCompleted = false;
1386
+ let currentActionNumber = 0;
1387
+ const maxActionNumber = 40;
1388
+ const yamlFlow = [];
1389
+ while (!isCompleted && currentActionNumber < maxActionNumber) {
1390
+ currentActionNumber++;
1391
+ const planningTask = this.planningTaskToGoal(userPrompt);
1392
+ await taskExecutor.append(planningTask);
1393
+ const output = await taskExecutor.flush();
1394
+ if (taskExecutor.isInErrorState()) {
1395
+ return {
1396
+ output: void 0,
1397
+ executor: taskExecutor
1398
+ };
1399
+ }
1400
+ const plans = output.actions;
1401
+ yamlFlow.push(...output.yamlFlow || []);
1402
+ let executables;
1403
+ try {
1404
+ executables = await this.convertPlanToExecutable(plans, opts);
1405
+ taskExecutor.append(executables.tasks);
1406
+ } catch (error) {
1407
+ return this.appendErrorPlan(
1408
+ taskExecutor,
1409
+ `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
1410
+ plans
1411
+ )}`
1412
+ );
1413
+ }
1414
+ await taskExecutor.flush();
1415
+ if (taskExecutor.isInErrorState()) {
1416
+ return {
1417
+ output: void 0,
1418
+ executor: taskExecutor
1419
+ };
1420
+ }
1421
+ if (plans[0].type === "Finished") {
1422
+ break;
1423
+ }
1424
+ }
1425
+ return {
1426
+ output: {
1427
+ yamlFlow
1428
+ },
1429
+ executor: taskExecutor
1430
+ };
1431
+ }
1432
+ async createTypeQueryTask(type, demand) {
1433
+ const taskExecutor = new import_misoai_core.Executor(
1434
+ taskTitleStr(
1435
+ type,
1436
+ typeof demand === "string" ? demand : JSON.stringify(demand)
1437
+ ),
1438
+ {
1439
+ onTaskStart: this.onTaskStartCallback
1440
+ }
1441
+ );
1442
+ const queryTask = {
1443
+ type: "Insight",
1444
+ subType: type,
1445
+ locate: null,
1446
+ param: {
1447
+ dataDemand: demand
1448
+ // for user param presentation in report right sidebar
1449
+ },
1450
+ executor: async (param) => {
1451
+ let insightDump;
1452
+ const dumpCollector = (dump) => {
1453
+ insightDump = dump;
1454
+ };
1455
+ this.insight.onceDumpUpdatedFn = dumpCollector;
1456
+ const ifTypeRestricted = type !== "Query";
1457
+ let demandInput = demand;
1458
+ if (ifTypeRestricted) {
1459
+ demandInput = {
1460
+ result: `${type}, ${demand}`
1461
+ };
1462
+ }
1463
+ const { data, usage } = await this.insight.extract(demandInput);
1464
+ let outputResult = data;
1465
+ if (ifTypeRestricted) {
1466
+ (0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
1467
+ outputResult = data.result;
1468
+ }
1469
+ return {
1470
+ output: outputResult,
1471
+ log: { dump: insightDump },
1472
+ usage
1473
+ };
1474
+ }
1475
+ };
1476
+ await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
1477
+ const output = await taskExecutor.flush();
1478
+ return {
1479
+ output,
1480
+ executor: taskExecutor
1481
+ };
1482
+ }
1483
+ async query(demand) {
1484
+ return this.createTypeQueryTask("Query", demand);
1485
+ }
1486
+ async boolean(prompt) {
1487
+ return this.createTypeQueryTask("Boolean", prompt);
1488
+ }
1489
+ async number(prompt) {
1490
+ return this.createTypeQueryTask("Number", prompt);
1491
+ }
1492
+ async string(prompt) {
1493
+ return this.createTypeQueryTask("String", prompt);
1494
+ }
1495
+ async assert(assertion) {
1496
+ const description = `assert: ${assertion}`;
1497
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
1498
+ onTaskStart: this.onTaskStartCallback
1499
+ });
1500
+ const assertionPlan = {
1501
+ type: "Assert",
1502
+ param: {
1503
+ assertion
1504
+ },
1505
+ locate: null
1506
+ };
1507
+ const { tasks } = await this.convertPlanToExecutable([assertionPlan]);
1508
+ await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
1509
+ const output = await taskExecutor.flush();
1510
+ return {
1511
+ output,
1512
+ executor: taskExecutor
1513
+ };
1514
+ }
1515
+ /**
1516
+ * Append a message to the conversation history
1517
+ * For user messages with images:
1518
+ * - Keep max 4 user image messages in history
1519
+ * - Remove oldest user image message when limit reached
1520
+ * For assistant messages:
1521
+ * - Simply append to history
1522
+ * @param conversationHistory Message to append
1523
+ */
1524
+ appendConversationHistory(conversationHistory) {
1525
+ if (conversationHistory.role === "user") {
1526
+ const userImgItems = this.conversationHistory.filter(
1527
+ (item) => item.role === "user"
1528
+ );
1529
+ if (userImgItems.length >= 4 && conversationHistory.role === "user") {
1530
+ const firstUserImgIndex = this.conversationHistory.findIndex(
1531
+ (item) => item.role === "user"
1532
+ );
1533
+ if (firstUserImgIndex >= 0) {
1534
+ this.conversationHistory.splice(firstUserImgIndex, 1);
1535
+ }
1536
+ }
1537
+ }
1538
+ this.conversationHistory.push(conversationHistory);
1539
+ }
1540
+ async appendErrorPlan(taskExecutor, errorMsg) {
1541
+ const errorPlan = {
1542
+ type: "Error",
1543
+ param: {
1544
+ thought: errorMsg
1545
+ },
1546
+ locate: null
1547
+ };
1548
+ const { tasks } = await this.convertPlanToExecutable([errorPlan]);
1549
+ await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
1550
+ await taskExecutor.flush();
1551
+ return {
1552
+ output: void 0,
1553
+ executor: taskExecutor
1554
+ };
1555
+ }
1556
+ async waitFor(assertion, opt) {
1557
+ const description = `waitFor: ${assertion}`;
1558
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("WaitFor", description), {
1559
+ onTaskStart: this.onTaskStartCallback
1560
+ });
1561
+ const { timeoutMs, checkIntervalMs } = opt;
1562
+ (0, import_utils6.assert)(assertion, "No assertion for waitFor");
1563
+ (0, import_utils6.assert)(timeoutMs, "No timeoutMs for waitFor");
1564
+ (0, import_utils6.assert)(checkIntervalMs, "No checkIntervalMs for waitFor");
1565
+ const overallStartTime = Date.now();
1566
+ let startTime = Date.now();
1567
+ let errorThought = "";
1568
+ while (Date.now() - overallStartTime < timeoutMs) {
1569
+ startTime = Date.now();
1570
+ const assertPlan = {
1571
+ type: "AssertWithoutThrow",
1572
+ param: {
1573
+ assertion
1574
+ },
1575
+ locate: null
1576
+ };
1577
+ const { tasks: assertTasks } = await this.convertPlanToExecutable([
1578
+ assertPlan
1579
+ ]);
1580
+ await taskExecutor.append(
1581
+ this.prependExecutorWithScreenshot(assertTasks[0])
1582
+ );
1583
+ const output = await taskExecutor.flush();
1584
+ if (output?.pass) {
1585
+ return {
1586
+ output: void 0,
1587
+ executor: taskExecutor
1588
+ };
1589
+ }
1590
+ errorThought = output?.thought || `unknown error when waiting for assertion: ${assertion}`;
1591
+ const now = Date.now();
1592
+ if (now - startTime < checkIntervalMs) {
1593
+ const timeRemaining = checkIntervalMs - (now - startTime);
1594
+ const sleepPlan = {
1595
+ type: "Sleep",
1596
+ param: {
1597
+ timeMs: timeRemaining
1598
+ },
1599
+ locate: null
1600
+ };
1601
+ const { tasks: sleepTasks } = await this.convertPlanToExecutable([
1602
+ sleepPlan
1603
+ ]);
1604
+ await taskExecutor.append(
1605
+ this.prependExecutorWithScreenshot(sleepTasks[0])
1606
+ );
1607
+ await taskExecutor.flush();
1608
+ }
1609
+ }
1610
+ return this.appendErrorPlan(
1611
+ taskExecutor,
1612
+ `waitFor timeout: ${errorThought}`
1613
+ );
1614
+ }
1615
+ };
1616
+
1617
+ // src/common/plan-builder.ts
1618
+ var import_logger2 = require("misoai-shared/logger");
1619
+ var import_utils8 = require("misoai-shared/utils");
1620
+ var debug2 = (0, import_logger2.getDebug)("plan-builder");
1621
+ function buildPlans(type, locateParam, param) {
1622
+ let returnPlans = [];
1623
+ const locatePlan = locateParam ? {
1624
+ type: "Locate",
1625
+ locate: locateParam,
1626
+ param: locateParam,
1627
+ thought: ""
1628
+ } : null;
1629
+ if (type === "Tap" || type === "Hover") {
1630
+ (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
1631
+ (0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
1632
+ const tapPlan = {
1633
+ type,
1634
+ param: null,
1635
+ thought: "",
1636
+ locate: locateParam
1637
+ };
1638
+ returnPlans = [locatePlan, tapPlan];
1639
+ }
1640
+ if (type === "Input" || type === "KeyboardPress") {
1641
+ if (type === "Input") {
1642
+ (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
1643
+ }
1644
+ (0, import_utils8.assert)(param, `missing param for action "${type}"`);
1645
+ const inputPlan = {
1646
+ type,
1647
+ param,
1648
+ thought: "",
1649
+ locate: locateParam
1650
+ };
1651
+ if (locatePlan) {
1652
+ returnPlans = [locatePlan, inputPlan];
1653
+ } else {
1654
+ returnPlans = [inputPlan];
1655
+ }
1656
+ }
1657
+ if (type === "Scroll") {
1658
+ (0, import_utils8.assert)(param, `missing param for action "${type}"`);
1659
+ const scrollPlan = {
1660
+ type,
1661
+ param,
1662
+ thought: "",
1663
+ locate: locateParam
1664
+ };
1665
+ if (locatePlan) {
1666
+ returnPlans = [locatePlan, scrollPlan];
1667
+ } else {
1668
+ returnPlans = [scrollPlan];
1669
+ }
1670
+ }
1671
+ if (type === "Sleep") {
1672
+ (0, import_utils8.assert)(param, `missing param for action "${type}"`);
1673
+ const sleepPlan = {
1674
+ type,
1675
+ param,
1676
+ thought: "",
1677
+ locate: null
1678
+ };
1679
+ returnPlans = [sleepPlan];
1680
+ }
1681
+ if (type === "Locate") {
1682
+ (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
1683
+ const locatePlan2 = {
1684
+ type,
1685
+ param: locateParam,
1686
+ locate: locateParam,
1687
+ thought: ""
1688
+ };
1689
+ returnPlans = [locatePlan2];
1690
+ }
1691
+ if (returnPlans) {
1692
+ debug2("buildPlans", returnPlans);
1693
+ return returnPlans;
1694
+ }
1695
+ throw new Error(`Not supported type: ${type}`);
1696
+ }
1697
+
1698
+ // src/common/task-cache.ts
1699
+ var import_node_assert = __toESM(require("assert"));
1700
+ var import_node_fs2 = require("fs");
1701
+ var import_node_path2 = require("path");
1702
+ var import_common2 = require("misoai-shared/common");
1703
+ var import_logger3 = require("misoai-shared/logger");
1704
+ var import_utils9 = require("misoai-shared/utils");
1705
+ var import_js_yaml3 = __toESM(require("js-yaml"));
1706
+ var import_semver = __toESM(require("semver"));
1707
+
1708
+ // package.json
1709
+ var version = "1.0.5";
1710
+
1711
+ // src/common/task-cache.ts
1712
+ var debug3 = (0, import_logger3.getDebug)("cache");
1713
+ var lowestSupportedMidsceneVersion = "0.16.10";
1714
+ var cacheFileExt = ".cache.yaml";
1715
+ var TaskCache = class {
1716
+ // Track matched records
1717
+ constructor(cacheId, isCacheResultUsed, cacheFilePath) {
1718
+ this.matchedCacheIndices = /* @__PURE__ */ new Set();
1719
+ (0, import_node_assert.default)(cacheId, "cacheId is required");
1720
+ this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
1721
+ this.cacheFilePath = import_utils9.ifInBrowser ? void 0 : cacheFilePath || (0, import_node_path2.join)((0, import_common2.getMidsceneRunSubDir)("cache"), `${this.cacheId}${cacheFileExt}`);
1722
+ this.isCacheResultUsed = isCacheResultUsed;
1723
+ let cacheContent;
1724
+ if (this.cacheFilePath) {
1725
+ cacheContent = this.loadCacheFromFile();
1726
+ }
1727
+ if (!cacheContent) {
1728
+ cacheContent = {
1729
+ midsceneVersion: version,
1730
+ cacheId: this.cacheId,
1731
+ caches: []
1732
+ };
1733
+ }
1734
+ this.cache = cacheContent;
1735
+ this.cacheOriginalLength = this.cache.caches.length;
1736
+ }
1737
+ matchCache(prompt, type) {
1738
+ for (let i = 0; i < this.cacheOriginalLength; i++) {
1739
+ const item = this.cache.caches[i];
1740
+ const key = `${type}:${prompt}:${i}`;
1741
+ if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
1742
+ this.matchedCacheIndices.add(key);
1743
+ debug3(
1744
+ "cache found and marked as used, type: %s, prompt: %s, index: %d",
1745
+ type,
1746
+ prompt,
1747
+ i
1748
+ );
1749
+ return {
1750
+ cacheContent: item,
1751
+ updateFn: (cb) => {
1752
+ debug3(
1753
+ "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
1754
+ type,
1755
+ prompt,
1756
+ i
1757
+ );
1758
+ cb(item);
1759
+ debug3(
1760
+ "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
1761
+ type,
1762
+ prompt,
1763
+ i
1764
+ );
1765
+ this.flushCacheToFile();
1766
+ }
1767
+ };
1768
+ }
1769
+ }
1770
+ debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
1771
+ return void 0;
1772
+ }
1773
+ matchPlanCache(prompt) {
1774
+ return this.matchCache(prompt, "plan");
1775
+ }
1776
+ matchLocateCache(prompt) {
1777
+ return this.matchCache(prompt, "locate");
1778
+ }
1779
+ appendCache(cache) {
1780
+ debug3("will append cache", cache);
1781
+ this.cache.caches.push(cache);
1782
+ this.flushCacheToFile();
1783
+ }
1784
+ loadCacheFromFile() {
1785
+ const cacheFile = this.cacheFilePath;
1786
+ (0, import_node_assert.default)(cacheFile, "cache file path is required");
1787
+ if (!(0, import_node_fs2.existsSync)(cacheFile)) {
1788
+ debug3("no cache file found, path: %s", cacheFile);
1789
+ return void 0;
1790
+ }
1791
+ const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
1792
+ if ((0, import_node_fs2.existsSync)(jsonTypeCacheFile) && this.isCacheResultUsed) {
1793
+ console.warn(
1794
+ `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
1795
+ );
1796
+ return void 0;
1797
+ }
1798
+ try {
1799
+ const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
1800
+ const jsonData = import_js_yaml3.default.load(data);
1801
+ if (!version) {
1802
+ debug3("no midscene version info, will not read cache from file");
1803
+ return void 0;
1804
+ }
1805
+ if (import_semver.default.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
1806
+ console.warn(
1807
+ `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
1808
+ Please delete the existing cache and rebuild it. Sorry for the inconvenience.
1809
+ cache file: ${cacheFile}`
1810
+ );
1811
+ return void 0;
1812
+ }
1813
+ debug3(
1814
+ "cache loaded from file, path: %s, cache version: %s, record length: %s",
1815
+ cacheFile,
1816
+ jsonData.midsceneVersion,
1817
+ jsonData.caches.length
1818
+ );
1819
+ jsonData.midsceneVersion = version;
1820
+ return jsonData;
1821
+ } catch (err) {
1822
+ debug3(
1823
+ "cache file exists but load failed, path: %s, error: %s",
1824
+ cacheFile,
1825
+ err
1826
+ );
1827
+ return void 0;
1828
+ }
1829
+ }
1830
+ flushCacheToFile() {
1831
+ if (!version) {
1832
+ debug3("no midscene version info, will not write cache to file");
1833
+ return;
1834
+ }
1835
+ if (!this.cacheFilePath) {
1836
+ debug3("no cache file path, will not write cache to file");
1837
+ return;
1838
+ }
1839
+ try {
1840
+ const yamlData = import_js_yaml3.default.dump(this.cache);
1841
+ (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
1842
+ } catch (err) {
1843
+ debug3(
1844
+ "write cache to file failed, path: %s, error: %s",
1845
+ this.cacheFilePath,
1846
+ err
1847
+ );
1848
+ }
1849
+ }
1850
+ updateOrAppendCacheRecord(newRecord, cachedRecord) {
1851
+ if (cachedRecord) {
1852
+ if (newRecord.type === "plan") {
1853
+ cachedRecord.updateFn((cache) => {
1854
+ cache.yamlWorkflow = newRecord.yamlWorkflow;
1855
+ });
1856
+ } else {
1857
+ cachedRecord.updateFn((cache) => {
1858
+ cache.xpaths = newRecord.xpaths;
1859
+ });
1860
+ }
1861
+ } else {
1862
+ this.appendCache(newRecord);
1863
+ }
1864
+ }
1865
+ };
1866
+
1867
+ // src/common/agent.ts
1868
+ var debug4 = (0, import_logger4.getDebug)("web-integration");
1869
+ var distanceOfTwoPoints = (p1, p2) => {
1870
+ const [x1, y1] = p1;
1871
+ const [x2, y2] = p2;
1872
+ return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));
1873
+ };
1874
+ var includedInRect = (point, rect) => {
1875
+ const [x, y] = point;
1876
+ const { left, top, width, height } = rect;
1877
+ return x >= left && x <= left + width && y >= top && y <= top + height;
1878
+ };
1879
+ var PageAgent = class {
1880
+ constructor(page, opts) {
1881
+ /**
1882
+ * If true, the agent will not perform any actions
1883
+ */
1884
+ this.dryMode = false;
1885
+ this.page = page;
1886
+ this.opts = Object.assign(
1887
+ {
1888
+ generateReport: true,
1889
+ autoPrintReportMsg: true,
1890
+ groupName: "Midscene Report",
1891
+ groupDescription: "",
1892
+ enableCumulativeContext: true,
1893
+ autoClearContext: false
1894
+ },
1895
+ opts || {}
1896
+ );
1897
+ this.initializeContextStore();
1898
+ if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
1899
+ this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
1900
+ this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
1901
+ }
1902
+ this.onTaskStartTip = this.opts.onTaskStartTip;
1903
+ this.insight = new import_misoai_core2.Insight(
1904
+ async (action) => {
1905
+ return this.getUIContext(action);
1906
+ }
1907
+ );
1908
+ if (opts?.cacheId && this.page.pageType !== "android") {
1909
+ this.taskCache = new TaskCache(
1910
+ opts.cacheId,
1911
+ (0, import_env2.getAIConfigInBoolean)("MIDSCENE_CACHE")
1912
+ // if we should use cache to match the element
1913
+ );
1914
+ }
1915
+ this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {
1916
+ taskCache: this.taskCache,
1917
+ onTaskStart: this.callbackOnTaskStartTip.bind(this)
1918
+ });
1919
+ this.dump = this.resetDump();
1920
+ this.reportFileName = reportFileName(
1921
+ opts?.testId || this.page.pageType || "web"
1922
+ );
1923
+ }
1924
+ /**
1925
+ * Initialize context store for cumulative context functionality
1926
+ */
1927
+ async initializeContextStore() {
1928
+ if (!this.opts.enableCumulativeContext) {
1929
+ debug4("Cumulative context disabled via options");
1930
+ return;
1931
+ }
1932
+ try {
1933
+ const aiModel = await import("misoai-core/ai-model");
1934
+ this.contextStore = aiModel.getContextStore();
1935
+ debug4("Context store initialized successfully", {
1936
+ autoClearContext: this.opts.autoClearContext,
1937
+ testId: this.opts.testId
1938
+ });
1939
+ if (this.opts.autoClearContext) {
1940
+ this.contextStore.clear();
1941
+ debug4("Context store cleared due to autoClearContext option");
1942
+ } else {
1943
+ const existingData = this.contextStore.getAllData();
1944
+ const existingSteps = this.contextStore.getRecentSteps(100).length;
1945
+ debug4("Context store preserving existing data", {
1946
+ existingDataKeys: Object.keys(existingData),
1947
+ existingStepsCount: existingSteps
1948
+ });
1949
+ }
1950
+ } catch (error) {
1951
+ debug4("Failed to initialize context store:", error);
1952
+ console.warn("⚠️ Could not initialize context store:", error);
1953
+ }
1954
+ }
1955
+ /**
1956
+ * Get the context store instance
1957
+ */
1958
+ getContextStore() {
1959
+ return this.contextStore;
1960
+ }
1961
+ /**
1962
+ * Clear the context store
1963
+ */
1964
+ clearContext() {
1965
+ if (this.contextStore) {
1966
+ this.contextStore.clear();
1967
+ }
1968
+ }
1969
+ /**
1970
+ * Get all stored data from context store
1971
+ */
1972
+ getStoredData() {
1973
+ if (this.contextStore) {
1974
+ return this.contextStore.getAllData();
1975
+ }
1976
+ return {};
1977
+ }
1978
+ /**
1979
+ * Get step summary from context store
1980
+ */
1981
+ getStepSummary() {
1982
+ if (this.contextStore) {
1983
+ return this.contextStore.getStepSummary();
1984
+ }
1985
+ return "";
1986
+ }
1987
+ async getUIContext(action) {
1988
+ if (action && (action === "extract" || action === "assert" || action === "captcha")) {
1989
+ return await parseContextFromWebPage(this.page, {
1990
+ ignoreMarker: true
1991
+ });
1992
+ }
1993
+ return await parseContextFromWebPage(this.page, {
1994
+ ignoreMarker: !!(0, import_env2.vlLocateMode)()
1995
+ });
1996
+ }
1997
+ // Helper method to call the insight.captcha method
1998
+ async _callInsightCaptcha(options) {
1999
+ const context = await this.getUIContext();
2000
+ if (this.page.url) {
2001
+ const url = await this.page.url();
2002
+ context.url = url;
2003
+ }
2004
+ return this.insight.captcha(context, options);
2005
+ }
2006
+ async setAIActionContext(prompt) {
2007
+ this.opts.aiActionContext = prompt;
2008
+ }
2009
+ resetDump() {
2010
+ this.dump = {
2011
+ groupName: this.opts.groupName,
2012
+ groupDescription: this.opts.groupDescription,
2013
+ executions: []
2014
+ };
2015
+ return this.dump;
2016
+ }
2017
+ appendExecutionDump(execution) {
2018
+ const currentDump = this.dump;
2019
+ currentDump.executions.push(execution);
2020
+ }
2021
+ dumpDataString() {
2022
+ this.dump.groupName = this.opts.groupName;
2023
+ this.dump.groupDescription = this.opts.groupDescription;
2024
+ return (0, import_utils11.stringifyDumpData)(this.dump);
2025
+ }
2026
+ reportHTMLString() {
2027
+ return (0, import_utils11.reportHTMLContent)(this.dumpDataString());
2028
+ }
2029
+ writeOutActionDumps() {
2030
+ const { generateReport, autoPrintReportMsg } = this.opts;
2031
+ this.reportFile = (0, import_utils11.writeLogFile)({
2032
+ fileName: this.reportFileName,
2033
+ fileExt: import_utils11.groupedActionDumpFileExt,
2034
+ fileContent: this.dumpDataString(),
2035
+ type: "dump",
2036
+ generateReport
2037
+ });
2038
+ debug4("writeOutActionDumps", this.reportFile);
2039
+ if (generateReport && autoPrintReportMsg && this.reportFile) {
2040
+ printReportMsg(this.reportFile);
2041
+ }
2042
+ }
2043
+ async callbackOnTaskStartTip(task) {
2044
+ const param = paramStr(task);
2045
+ const tip = param ? `${typeStr(task)} - ${param}` : typeStr(task);
2046
+ if (this.onTaskStartTip) {
2047
+ await this.onTaskStartTip(tip);
2048
+ }
2049
+ }
2050
+ afterTaskRunning(executor, doNotThrowError = false) {
2051
+ this.appendExecutionDump(executor.dump());
2052
+ this.writeOutActionDumps();
2053
+ if (executor.isInErrorState() && !doNotThrowError) {
2054
+ const errorTask = executor.latestErrorTask();
2055
+ throw new Error(`${errorTask?.error}`);
2056
+ }
2057
+ const lastTask = executor.tasks[executor.tasks.length - 1];
2058
+ const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
2059
+ const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
2060
+ const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
2061
+ const planningTasks = executor.tasks.filter((task) => task.type === "Planning");
2062
+ const insightTasks = executor.tasks.filter((task) => task.type === "Insight");
2063
+ const actionTasks = executor.tasks.filter((task) => task.type === "Action");
2064
+ const planning = planningTasks.length > 0 ? {
2065
+ type: "Planning",
2066
+ description: `Planning for task execution`,
2067
+ steps: planningTasks.map((task) => task.thought || "Planning step")
2068
+ } : void 0;
2069
+ const insight = insightTasks.length > 0 ? {
2070
+ type: "Insight",
2071
+ description: `Insight for task execution`,
2072
+ elements: insightTasks.map((task) => task.thought || "Insight element")
2073
+ } : void 0;
2074
+ const action = actionTasks.length > 0 ? {
2075
+ type: "Action",
2076
+ description: `Action for task execution`,
2077
+ result: lastTask?.output
2078
+ } : void 0;
2079
+ const actionDetails = executor.tasks.map((task) => ({
2080
+ type: task.type,
2081
+ subType: task.subType,
2082
+ status: task.status,
2083
+ thought: task.thought
2084
+ }));
2085
+ const metadata = {
2086
+ status: lastTask?.status,
2087
+ start: lastTask?.timing?.start,
2088
+ end: lastTask?.timing?.end,
2089
+ totalTime: lastTask?.timing?.cost,
2090
+ cache: lastTask?.cache,
2091
+ usage: lastTask?.usage,
2092
+ thought: allThoughts.length > 0 ? allThoughts.join("\n") : lastTask?.thought,
2093
+ locate: allLocates.length > 0 ? allLocates : lastTask?.locate,
2094
+ plan: allPlans.length > 0 ? allPlans : lastTask?.param?.plans,
2095
+ // Add planning, insight, and action information
2096
+ planning,
2097
+ insight,
2098
+ action,
2099
+ actionDetails,
2100
+ // Include raw tasks for debugging
2101
+ tasks: executor.tasks.map((task) => ({
2102
+ type: task.type,
2103
+ subType: task.subType,
2104
+ status: task.status,
2105
+ thought: task.thought,
2106
+ locate: task.locate,
2107
+ timing: task.timing,
2108
+ usage: task.usage,
2109
+ cache: task.cache,
2110
+ error: task.error
2111
+ }))
2112
+ };
2113
+ return metadata;
2114
+ }
2115
+ buildDetailedLocateParam(locatePrompt, opt) {
2116
+ (0, import_utils12.assert)(locatePrompt, "missing locate prompt");
2117
+ if (typeof opt === "object") {
2118
+ const prompt = opt.prompt ?? locatePrompt;
2119
+ const deepThink = opt.deepThink ?? false;
2120
+ const cacheable = opt.cacheable ?? true;
2121
+ return {
2122
+ prompt,
2123
+ deepThink,
2124
+ cacheable
2125
+ };
2126
+ }
2127
+ return {
2128
+ prompt: locatePrompt
2129
+ };
2130
+ }
2131
+ async aiTap(locatePrompt, opt) {
2132
+ const detailedLocateParam = this.buildDetailedLocateParam(
2133
+ locatePrompt,
2134
+ opt
2135
+ );
2136
+ const plans = buildPlans("Tap", detailedLocateParam);
2137
+ const { executor, output } = await this.taskExecutor.runPlans(
2138
+ taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
2139
+ plans,
2140
+ { cacheable: opt?.cacheable }
2141
+ );
2142
+ const metadata = this.afterTaskRunning(executor);
2143
+ return {
2144
+ result: output,
2145
+ metadata
2146
+ };
2147
+ }
2148
+ async aiHover(locatePrompt, opt) {
2149
+ const detailedLocateParam = this.buildDetailedLocateParam(
2150
+ locatePrompt,
2151
+ opt
2152
+ );
2153
+ const plans = buildPlans("Hover", detailedLocateParam);
2154
+ const { executor, output } = await this.taskExecutor.runPlans(
2155
+ taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
2156
+ plans,
2157
+ { cacheable: opt?.cacheable }
2158
+ );
2159
+ const metadata = this.afterTaskRunning(executor);
2160
+ return {
2161
+ result: output,
2162
+ metadata
2163
+ };
2164
+ }
2165
+ async aiInput(value, locatePrompt, opt) {
2166
+ (0, import_utils12.assert)(
2167
+ typeof value === "string",
2168
+ "input value must be a string, use empty string if you want to clear the input"
2169
+ );
2170
+ (0, import_utils12.assert)(locatePrompt, "missing locate prompt for input");
2171
+ const detailedLocateParam = this.buildDetailedLocateParam(
2172
+ locatePrompt,
2173
+ opt
2174
+ );
2175
+ const plans = buildPlans("Input", detailedLocateParam, {
2176
+ value
2177
+ });
2178
+ const { executor, output } = await this.taskExecutor.runPlans(
2179
+ taskTitleStr("Input", locateParamStr(detailedLocateParam)),
2180
+ plans,
2181
+ { cacheable: opt?.cacheable }
2182
+ );
2183
+ const metadata = this.afterTaskRunning(executor);
2184
+ return {
2185
+ result: output,
2186
+ metadata
2187
+ };
2188
+ }
2189
+ async aiKeyboardPress(keyName, locatePrompt, opt) {
2190
+ (0, import_utils12.assert)(keyName, "missing keyName for keyboard press");
2191
+ const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
2192
+ const plans = buildPlans("KeyboardPress", detailedLocateParam, {
2193
+ value: keyName
2194
+ });
2195
+ const { executor, output } = await this.taskExecutor.runPlans(
2196
+ taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
2197
+ plans,
2198
+ { cacheable: opt?.cacheable }
2199
+ );
2200
+ const metadata = this.afterTaskRunning(executor);
2201
+ return {
2202
+ result: output,
2203
+ metadata
2204
+ };
2205
+ }
2206
+ async aiScroll(scrollParam, locatePrompt, opt) {
2207
+ const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
2208
+ const plans = buildPlans("Scroll", detailedLocateParam, scrollParam);
2209
+ const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
2210
+ const { executor, output } = await this.taskExecutor.runPlans(
2211
+ taskTitleStr("Scroll", paramInTitle),
2212
+ plans,
2213
+ { cacheable: opt?.cacheable }
2214
+ );
2215
+ const metadata = this.afterTaskRunning(executor);
2216
+ return {
2217
+ result: output,
2218
+ metadata
2219
+ };
2220
+ }
2221
+ async aiAction(taskPrompt, opt) {
2222
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2223
+ try {
2224
+ const originalPrompt = taskPrompt;
2225
+ const processedPrompt = this.contextStore.replaceAllReferences(taskPrompt, "action");
2226
+ if (originalPrompt !== processedPrompt) {
2227
+ debug4("Context replacement in aiAction:", {
2228
+ original: originalPrompt,
2229
+ processed: processedPrompt,
2230
+ storedData: this.contextStore.getAllData()
2231
+ });
2232
+ }
2233
+ this.contextStore.addStep({
2234
+ type: "action",
2235
+ summary: `Action: ${processedPrompt}`,
2236
+ prompt: processedPrompt
2237
+ });
2238
+ debug4("Added action step to context store:", {
2239
+ stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
2240
+ totalSteps: this.contextStore.getRecentSteps(100).length
2241
+ });
2242
+ taskPrompt = processedPrompt;
2243
+ } catch (error) {
2244
+ debug4("Context store operation failed:", error);
2245
+ }
2246
+ }
2247
+ const cacheable = opt?.cacheable;
2248
+ const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
2249
+ const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
2250
+ if (matchedCache && this.taskCache?.isCacheResultUsed) {
2251
+ const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
2252
+ taskPrompt,
2253
+ matchedCache.cacheContent?.yamlWorkflow
2254
+ );
2255
+ const metadata2 = this.afterTaskRunning(executor2);
2256
+ debug4("matched cache, will call .runYaml to run the action");
2257
+ const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
2258
+ const result = await this.runYaml(yaml5);
2259
+ return {
2260
+ result: result.result,
2261
+ metadata: metadata2
2262
+ };
2263
+ }
2264
+ const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
2265
+ cacheable
2266
+ }));
2267
+ if (this.taskCache && output?.yamlFlow && cacheable !== false) {
2268
+ const yamlContent = {
2269
+ tasks: [
2270
+ {
2271
+ name: taskPrompt,
2272
+ flow: output.yamlFlow
2273
+ }
2274
+ ]
2275
+ };
2276
+ const yamlFlowStr = import_js_yaml4.default.dump(yamlContent);
2277
+ this.taskCache.updateOrAppendCacheRecord(
2278
+ {
2279
+ type: "plan",
2280
+ prompt: taskPrompt,
2281
+ yamlWorkflow: yamlFlowStr
2282
+ },
2283
+ matchedCache
2284
+ );
2285
+ }
2286
+ const metadata = this.afterTaskRunning(executor);
2287
+ return {
2288
+ result: output,
2289
+ metadata
2290
+ };
2291
+ }
2292
+ async aiQuery(demand) {
2293
+ let processedDemand = demand;
2294
+ let storageKey;
2295
+ try {
2296
+ const aiModel = await import("misoai-core/ai-model");
2297
+ const contextStore = aiModel.getContextStore();
2298
+ if (typeof demand === "string") {
2299
+ const storageInstruction = contextStore.parseStorageInstruction(demand);
2300
+ if (storageInstruction) {
2301
+ storageKey = storageInstruction.key;
2302
+ processedDemand = storageInstruction.cleanText;
2303
+ contextStore._pendingAliases = storageInstruction.aliases;
2304
+ } else {
2305
+ const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
2306
+ if (storageMatch) {
2307
+ storageKey = storageMatch[1];
2308
+ processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
2309
+ }
2310
+ }
2311
+ }
2312
+ } catch (error) {
2313
+ debug4("Context store not available:", error);
2314
+ }
2315
+ const { output, executor } = await this.taskExecutor.query(processedDemand);
2316
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2317
+ if (storageKey && output) {
2318
+ try {
2319
+ const pendingAliases = this.contextStore._pendingAliases;
2320
+ if (pendingAliases) {
2321
+ this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
2322
+ delete this.contextStore._pendingAliases;
2323
+ debug4("Stored query result with aliases:", {
2324
+ key: storageKey,
2325
+ value: output,
2326
+ aliases: pendingAliases
2327
+ });
2328
+ } else {
2329
+ this.contextStore.storeData(storageKey, output);
2330
+ debug4("Stored query result:", {
2331
+ key: storageKey,
2332
+ value: output
2333
+ });
2334
+ }
2335
+ this.contextStore.addStep({
2336
+ type: "query",
2337
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
2338
+ data: output,
2339
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2340
+ });
2341
+ debug4("Added query step to context store:", {
2342
+ storageKey,
2343
+ totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
2344
+ totalSteps: this.contextStore.getRecentSteps(100).length
2345
+ });
2346
+ } catch (error) {
2347
+ debug4("Failed to store query result:", error);
2348
+ }
2349
+ } else {
2350
+ try {
2351
+ this.contextStore.addStep({
2352
+ type: "query",
2353
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
2354
+ data: output,
2355
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2356
+ });
2357
+ } catch (error) {
2358
+ debug4("Failed to add query step:", error);
2359
+ }
2360
+ }
2361
+ }
2362
+ const metadata = this.afterTaskRunning(executor);
2363
+ return {
2364
+ result: output,
2365
+ metadata
2366
+ };
2367
+ }
2368
+ async aiBoolean(prompt) {
2369
+ const { output, executor } = await this.taskExecutor.boolean(prompt);
2370
+ const metadata = this.afterTaskRunning(executor);
2371
+ return {
2372
+ result: output,
2373
+ metadata
2374
+ };
2375
+ }
2376
+ async aiNumber(prompt) {
2377
+ const { output, executor } = await this.taskExecutor.number(prompt);
2378
+ const metadata = this.afterTaskRunning(executor);
2379
+ return {
2380
+ result: output,
2381
+ metadata
2382
+ };
2383
+ }
2384
+ async aiString(prompt) {
2385
+ const { output, executor } = await this.taskExecutor.string(prompt);
2386
+ const metadata = this.afterTaskRunning(executor);
2387
+ return {
2388
+ result: output,
2389
+ metadata
2390
+ };
2391
+ }
2392
+ async describeElementAtPoint(center, opt) {
2393
+ const { verifyPrompt = true, retryLimit = 3 } = opt || {};
2394
+ let success = false;
2395
+ let retryCount = 0;
2396
+ let resultPrompt = "";
2397
+ let deepThink = opt?.deepThink || false;
2398
+ let verifyResult;
2399
+ while (!success && retryCount < retryLimit) {
2400
+ if (retryCount >= 2) {
2401
+ deepThink = true;
2402
+ }
2403
+ debug4(
2404
+ "aiDescribe",
2405
+ center,
2406
+ "verifyPrompt",
2407
+ verifyPrompt,
2408
+ "retryCount",
2409
+ retryCount,
2410
+ "deepThink",
2411
+ deepThink
2412
+ );
2413
+ const text = await this.insight.describe(center, { deepThink });
2414
+ debug4("aiDescribe text", text);
2415
+ (0, import_utils12.assert)(text.description, `failed to describe element at [${center}]`);
2416
+ resultPrompt = text.description;
2417
+ verifyResult = await this.verifyLocator(
2418
+ resultPrompt,
2419
+ deepThink ? { deepThink: true } : void 0,
2420
+ center,
2421
+ opt
2422
+ );
2423
+ if (verifyResult.pass) {
2424
+ success = true;
2425
+ } else {
2426
+ retryCount++;
2427
+ }
2428
+ }
2429
+ return {
2430
+ prompt: resultPrompt,
2431
+ deepThink,
2432
+ verifyResult
2433
+ };
2434
+ }
2435
+ async verifyLocator(prompt, locateOpt, expectCenter, verifyLocateOption) {
2436
+ debug4("verifyLocator", prompt, locateOpt, expectCenter, verifyLocateOption);
2437
+ const locateResult = await this.aiLocate(prompt, locateOpt);
2438
+ const { center: verifyCenter, rect: verifyRect } = locateResult.result;
2439
+ const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
2440
+ const included = includedInRect(expectCenter, verifyRect);
2441
+ const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;
2442
+ const verifyResult = {
2443
+ pass,
2444
+ rect: verifyRect,
2445
+ center: verifyCenter,
2446
+ centerDistance: distance
2447
+ };
2448
+ debug4("aiDescribe verifyResult", verifyResult);
2449
+ return verifyResult;
2450
+ }
2451
+ async aiLocate(prompt, opt) {
2452
+ const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
2453
+ const plans = buildPlans("Locate", detailedLocateParam);
2454
+ const { executor, output } = await this.taskExecutor.runPlans(
2455
+ taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
2456
+ plans,
2457
+ { cacheable: opt?.cacheable }
2458
+ );
2459
+ const metadata = this.afterTaskRunning(executor);
2460
+ const { element } = output;
2461
+ const result = {
2462
+ rect: element?.rect,
2463
+ center: element?.center
2464
+ };
2465
+ return {
2466
+ result,
2467
+ metadata
2468
+ };
2469
+ }
2470
+ async aiAssert(assertion, msg, opt) {
2471
+ let processedAssertion = assertion;
2472
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2473
+ try {
2474
+ const originalAssertion = assertion;
2475
+ processedAssertion = this.contextStore.replaceAllReferences(assertion, "assertion");
2476
+ if (originalAssertion !== processedAssertion) {
2477
+ debug4("Context replacement in aiAssert:", {
2478
+ original: originalAssertion,
2479
+ processed: processedAssertion,
2480
+ context: "assertion",
2481
+ storedData: this.contextStore.getAllData()
2482
+ });
2483
+ }
2484
+ this.contextStore.addStep({
2485
+ type: "assertion",
2486
+ summary: `Assertion: ${processedAssertion}`,
2487
+ prompt: processedAssertion
2488
+ });
2489
+ debug4("Added assertion step to context store:", {
2490
+ totalSteps: this.contextStore.getRecentSteps(100).length
2491
+ });
2492
+ } catch (error) {
2493
+ debug4("Context store operation failed:", error);
2494
+ }
2495
+ }
2496
+ let currentUrl = "";
2497
+ if (this.page.url) {
2498
+ try {
2499
+ currentUrl = await this.page.url();
2500
+ } catch (e) {
2501
+ }
2502
+ }
2503
+ const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
2504
+ const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
2505
+ const metadata = this.afterTaskRunning(executor, true);
2506
+ if (output && opt?.keepRawResponse) {
2507
+ return {
2508
+ result: output,
2509
+ metadata
2510
+ };
2511
+ }
2512
+ if (!output?.pass) {
2513
+ const errMsg = msg || `Assertion failed: ${assertion}`;
2514
+ const reasonMsg = `Reason: ${output?.thought || executor.latestErrorTask()?.error || "(no_reason)"}`;
2515
+ throw new Error(`${errMsg}
2516
+ ${reasonMsg}`);
2517
+ }
2518
+ return {
2519
+ result: true,
2520
+ metadata
2521
+ };
2522
+ }
2523
+ async aiCaptcha(options) {
2524
+ const { deepThink = false, autoDetectComplexity = true } = options || {};
2525
+ let shouldUseDeepThink = deepThink;
2526
+ if (autoDetectComplexity && !deepThink) {
2527
+ const context = await this.getUIContext();
2528
+ const { screenshotBase64 } = context;
2529
+ try {
2530
+ const complexityAnalysisPrompt = `
2531
+ Analyze this screenshot and determine if it contains a complex CAPTCHA that would benefit from deep thinking.
2532
+ A complex CAPTCHA typically has one or more of these characteristics:
2533
+ - Distorted or overlapping text that is hard to read
2534
+ - Multiple images that need to be selected based on a specific criteria
2535
+ - Puzzles that require spatial reasoning
2536
+ - Multiple steps or verification methods
2537
+ - Small or hard-to-distinguish elements
2538
+
2539
+ Return only "complex" or "simple" based on your analysis.
2540
+ `;
2541
+ const complexityMsgs = [
2542
+ { role: "system", content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity." },
2543
+ {
2544
+ role: "user",
2545
+ content: [
2546
+ {
2547
+ type: "image_url",
2548
+ image_url: {
2549
+ url: screenshotBase64,
2550
+ detail: "high"
2551
+ }
2552
+ },
2553
+ {
2554
+ type: "text",
2555
+ text: complexityAnalysisPrompt
2556
+ }
2557
+ ]
2558
+ }
2559
+ ];
2560
+ const complexityResult = await this.insight.aiVendorFn(
2561
+ complexityMsgs,
2562
+ { type: "extract_data" }
2563
+ );
2564
+ const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
2565
+ shouldUseDeepThink = responseText.includes("complex");
2566
+ debug4("CAPTCHA complexity analysis:", responseText, "Using deep think:", shouldUseDeepThink);
2567
+ } catch (error) {
2568
+ debug4("Failed to analyze CAPTCHA complexity:", error);
2569
+ }
2570
+ }
2571
+ const captchaResponse = await this._callInsightCaptcha({
2572
+ deepThink: shouldUseDeepThink
2573
+ });
2574
+ const captchaResult = captchaResponse.content;
2575
+ const usage = captchaResponse.usage;
2576
+ const actualDeepThink = captchaResponse.deepThink || false;
2577
+ if (captchaResult.captchaType === "text") {
2578
+ for (const action of captchaResult.actions) {
2579
+ if (action.type === "click" && action.target) {
2580
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2581
+ } else if (action.type === "input" && action.value) {
2582
+ if (action.target) {
2583
+ await this.aiInput(action.value, action.target, { deepThink: shouldUseDeepThink });
2584
+ }
2585
+ } else if (action.type === "verify" && action.target) {
2586
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2587
+ }
2588
+ }
2589
+ } else if (captchaResult.captchaType === "image") {
2590
+ for (const action of captchaResult.actions) {
2591
+ if (action.type === "click") {
2592
+ if (action.coordinates) {
2593
+ const x = action.coordinates[0];
2594
+ const y = action.coordinates[1];
2595
+ await this.aiTap(`element at coordinates (${x}, ${y})`, { deepThink: shouldUseDeepThink });
2596
+ } else if (action.target) {
2597
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2598
+ }
2599
+ } else if (action.type === "verify" && action.target) {
2600
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2601
+ }
2602
+ }
2603
+ }
2604
+ await new Promise((resolve2) => setTimeout(resolve2, 3e3));
2605
+ const metadata = {
2606
+ status: "finished",
2607
+ usage,
2608
+ thought: captchaResult.thought
2609
+ };
2610
+ metadata.deepThink = actualDeepThink;
2611
+ if (autoDetectComplexity && !deepThink) {
2612
+ metadata.autoDetectedComplexity = shouldUseDeepThink;
2613
+ }
2614
+ return {
2615
+ result: captchaResult,
2616
+ metadata
2617
+ };
2618
+ }
2619
+ async aiWaitFor(assertion, opt) {
2620
+ const startTime = Date.now();
2621
+ const { executor } = await this.taskExecutor.waitFor(assertion, {
2622
+ timeoutMs: opt?.timeoutMs || 15 * 1e3,
2623
+ checkIntervalMs: opt?.checkIntervalMs || 3 * 1e3,
2624
+ assertion
2625
+ });
2626
+ const metadata = {
2627
+ status: executor.isInErrorState() ? "failed" : "finished",
2628
+ start: startTime,
2629
+ end: Date.now(),
2630
+ totalTime: Date.now() - startTime,
2631
+ thought: executor.latestErrorTask()?.thought,
2632
+ actionDetails: executor.tasks.map((task) => ({
2633
+ type: task.type,
2634
+ subType: task.subType,
2635
+ status: task.status,
2636
+ thought: task.thought
2637
+ }))
2638
+ };
2639
+ this.appendExecutionDump(executor.dump());
2640
+ this.writeOutActionDumps();
2641
+ if (executor.isInErrorState()) {
2642
+ const errorTask = executor.latestErrorTask();
2643
+ throw new Error(`${errorTask?.error}
2644
+ ${errorTask?.errorStack}`);
2645
+ }
2646
+ return {
2647
+ result: true,
2648
+ // Successfully waited
2649
+ metadata
2650
+ };
2651
+ }
2652
+ async ai(taskPrompt, type = "action", options) {
2653
+ if (type === "action") {
2654
+ return this.aiAction(taskPrompt);
2655
+ }
2656
+ if (type === "query") {
2657
+ return this.aiQuery(taskPrompt);
2658
+ }
2659
+ if (type === "assert") {
2660
+ return this.aiAssert(taskPrompt);
2661
+ }
2662
+ if (type === "tap") {
2663
+ return this.aiTap(taskPrompt, options);
2664
+ }
2665
+ if (type === "captcha") {
2666
+ return this.aiCaptcha(options);
2667
+ }
2668
+ throw new Error(
2669
+ `Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'captcha'`
2670
+ );
2671
+ }
2672
+ async runYaml(yamlScriptContent) {
2673
+ const startTime = Date.now();
2674
+ const script = parseYamlScript(yamlScriptContent, "yaml", true);
2675
+ const player = new ScriptPlayer(script, async () => {
2676
+ return { agent: this, freeFn: [] };
2677
+ });
2678
+ await player.run();
2679
+ const endTime = Date.now();
2680
+ const metadata = {
2681
+ status: player.status,
2682
+ start: startTime,
2683
+ end: endTime,
2684
+ totalTime: endTime - startTime,
2685
+ tasks: player.taskStatusList.map((task) => ({
2686
+ type: "yaml-task",
2687
+ subType: task.name,
2688
+ status: task.status,
2689
+ error: task.error?.message
2690
+ }))
2691
+ };
2692
+ if (player.status === "error") {
2693
+ const errors = player.taskStatusList.filter((task) => task.status === "error").map((task) => {
2694
+ return `task - ${task.name}: ${task.error?.message}`;
2695
+ }).join("\n");
2696
+ throw new Error(`Error(s) occurred in running yaml script:
2697
+ ${errors}`);
2698
+ }
2699
+ return {
2700
+ result: player.result,
2701
+ metadata
2702
+ };
2703
+ }
2704
+ async evaluateJavaScript(script) {
2705
+ (0, import_utils12.assert)(
2706
+ this.page.evaluateJavaScript,
2707
+ "evaluateJavaScript is not supported in current agent"
2708
+ );
2709
+ if (this.page.evaluateJavaScript) {
2710
+ return this.page.evaluateJavaScript(script);
2711
+ }
2712
+ throw new Error("evaluateJavaScript is not supported in current agent");
2713
+ }
2714
+ async destroy() {
2715
+ await this.page.destroy();
2716
+ }
2717
+ };
2718
+
2719
+ // src/puppeteer/index.ts
2720
+ var import_logger6 = require("misoai-shared/logger");
2721
+
2722
+ // src/puppeteer/page.ts
2723
+ var import_constants4 = require("misoai-shared/constants");
2724
+
2725
+ // src/puppeteer/base-page.ts
2726
+ var import_utils15 = require("misoai-core/utils");
2727
+ var import_constants3 = require("misoai-shared/constants");
2728
+ var import_extractor2 = require("misoai-shared/extractor");
2729
+ var import_fs = require("misoai-shared/fs");
2730
+ var import_logger5 = require("misoai-shared/logger");
2731
+ var import_utils16 = require("misoai-shared/utils");
2732
+ var debugPage = (0, import_logger5.getDebug)("web:page");
2733
+ var Page = class {
2734
+ constructor(underlyingPage, pageType, opts) {
2735
+ this.everMoved = false;
2736
+ this.underlyingPage = underlyingPage;
2737
+ this.pageType = pageType;
2738
+ this.waitForNavigationTimeout = opts?.waitForNavigationTimeout || import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
2739
+ }
2740
+ async evaluate(pageFunction, arg) {
2741
+ let result;
2742
+ debugPage("evaluate function begin");
2743
+ if (this.pageType === "puppeteer") {
2744
+ result = await this.underlyingPage.evaluate(
2745
+ pageFunction,
2746
+ arg
2747
+ );
2748
+ } else {
2749
+ result = await this.underlyingPage.evaluate(
2750
+ pageFunction,
2751
+ arg
2752
+ );
2753
+ }
2754
+ debugPage("evaluate function end");
2755
+ return result;
2756
+ }
2757
+ async evaluateJavaScript(script) {
2758
+ return this.evaluate(script);
2759
+ }
2760
+ async waitForNavigation() {
2761
+ if (this.pageType === "puppeteer" || this.pageType === "playwright") {
2762
+ debugPage("waitForNavigation begin");
2763
+ debugPage(`waitForNavigation timeout: ${this.waitForNavigationTimeout}`);
2764
+ try {
2765
+ await this.underlyingPage.waitForSelector("html", {
2766
+ timeout: this.waitForNavigationTimeout
2767
+ });
2768
+ } catch (error) {
2769
+ console.warn(
2770
+ "[midscene:warning] Waiting for the navigation has timed out, but Midscene will continue execution. Please check https://midscenejs.com/faq.html#customize-the-network-timeout for more information on customizing the network timeout"
2771
+ );
2772
+ }
2773
+ debugPage("waitForNavigation end");
2774
+ }
2775
+ }
2776
+ // @deprecated
2777
+ async getElementsInfo() {
2778
+ await this.waitForNavigation();
2779
+ debugPage("getElementsInfo begin");
2780
+ const tree = await this.getElementsNodeTree();
2781
+ debugPage("getElementsInfo end");
2782
+ return (0, import_extractor2.treeToList)(tree);
2783
+ }
2784
+ async getXpathsById(id) {
2785
+ const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
2786
+ return this.evaluateJavaScript(
2787
+ `${elementInfosScriptContent}midscene_element_inspector.getXpathsById('${id}')`
2788
+ );
2789
+ }
2790
+ async getElementInfoByXpath(xpath) {
2791
+ const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
2792
+ return this.evaluateJavaScript(
2793
+ `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByXpath('${xpath}')`
2794
+ );
2795
+ }
2796
+ async getElementsNodeTree() {
2797
+ await this.waitForNavigation();
2798
+ const scripts = await (0, import_fs.getExtraReturnLogic)(true);
2799
+ (0, import_utils16.assert)(scripts, "scripts should be set before writing report in browser");
2800
+ const captureElementSnapshot = await this.evaluate(scripts);
2801
+ return captureElementSnapshot;
2802
+ }
2803
+ async size() {
2804
+ if (this.viewportSize)
2805
+ return this.viewportSize;
2806
+ const sizeInfo = await this.evaluate(() => {
2807
+ return {
2808
+ width: document.documentElement.clientWidth,
2809
+ height: document.documentElement.clientHeight,
2810
+ dpr: window.devicePixelRatio
2811
+ };
2812
+ });
2813
+ this.viewportSize = sizeInfo;
2814
+ return sizeInfo;
2815
+ }
2816
+ async screenshotBase64() {
2817
+ const imgType = "jpeg";
2818
+ const quality = 90;
2819
+ await this.waitForNavigation();
2820
+ debugPage("screenshotBase64 begin");
2821
+ let base64;
2822
+ if (this.pageType === "puppeteer") {
2823
+ const result = await this.underlyingPage.screenshot({
2824
+ type: imgType,
2825
+ quality,
2826
+ encoding: "base64"
2827
+ });
2828
+ base64 = `data:image/jpeg;base64,${result}`;
2829
+ } else if (this.pageType === "playwright") {
2830
+ const buffer = await this.underlyingPage.screenshot({
2831
+ type: imgType,
2832
+ quality,
2833
+ timeout: 10 * 1e3
2834
+ });
2835
+ base64 = `data:image/jpeg;base64,${buffer.toString("base64")}`;
2836
+ } else {
2837
+ throw new Error("Unsupported page type for screenshot");
2838
+ }
2839
+ debugPage("screenshotBase64 end");
2840
+ return base64;
2841
+ }
2842
+ async url() {
2843
+ return this.underlyingPage.url();
2844
+ }
2845
+ get mouse() {
2846
+ return {
2847
+ click: async (x, y, options) => {
2848
+ await this.mouse.move(x, y);
2849
+ this.underlyingPage.mouse.click(x, y, {
2850
+ button: options?.button || "left",
2851
+ count: options?.count || 1
2852
+ });
2853
+ },
2854
+ wheel: async (deltaX, deltaY) => {
2855
+ if (this.pageType === "puppeteer") {
2856
+ await this.underlyingPage.mouse.wheel({
2857
+ deltaX,
2858
+ deltaY
2859
+ });
2860
+ } else if (this.pageType === "playwright") {
2861
+ await this.underlyingPage.mouse.wheel(
2862
+ deltaX,
2863
+ deltaY
2864
+ );
2865
+ }
2866
+ },
2867
+ move: async (x, y) => {
2868
+ this.everMoved = true;
2869
+ return this.underlyingPage.mouse.move(x, y);
2870
+ },
2871
+ drag: async (from, to) => {
2872
+ if (this.pageType === "puppeteer") {
2873
+ await this.underlyingPage.mouse.drag(
2874
+ {
2875
+ x: from.x,
2876
+ y: from.y
2877
+ },
2878
+ {
2879
+ x: to.x,
2880
+ y: to.y
2881
+ }
2882
+ );
2883
+ } else if (this.pageType === "playwright") {
2884
+ await this.underlyingPage.mouse.move(
2885
+ from.x,
2886
+ from.y
2887
+ );
2888
+ await this.underlyingPage.mouse.down();
2889
+ await this.underlyingPage.mouse.move(to.x, to.y);
2890
+ await this.underlyingPage.mouse.up();
2891
+ }
2892
+ }
2893
+ };
2894
+ }
2895
+ get keyboard() {
2896
+ return {
2897
+ type: async (text) => this.underlyingPage.keyboard.type(text, { delay: 80 }),
2898
+ press: async (action) => {
2899
+ const keys = Array.isArray(action) ? action : [action];
2900
+ for (const k of keys) {
2901
+ const commands = k.command ? [k.command] : [];
2902
+ await this.underlyingPage.keyboard.down(k.key, { commands });
2903
+ }
2904
+ for (const k of [...keys].reverse()) {
2905
+ await this.underlyingPage.keyboard.up(k.key);
2906
+ }
2907
+ },
2908
+ down: async (key) => {
2909
+ this.underlyingPage.keyboard.down(key);
2910
+ },
2911
+ up: async (key) => {
2912
+ this.underlyingPage.keyboard.up(key);
2913
+ }
2914
+ };
2915
+ }
2916
+ async clearInput(element) {
2917
+ if (!element) {
2918
+ console.warn("No element to clear input");
2919
+ return;
2920
+ }
2921
+ const isMac = process.platform === "darwin";
2922
+ if (isMac) {
2923
+ if (this.pageType === "puppeteer") {
2924
+ await this.mouse.click(element.center[0], element.center[1], {
2925
+ count: 3
2926
+ });
2927
+ } else {
2928
+ await this.mouse.click(element.center[0], element.center[1]);
2929
+ await this.underlyingPage.keyboard.down("Meta");
2930
+ await this.underlyingPage.keyboard.press("a");
2931
+ await this.underlyingPage.keyboard.up("Meta");
2932
+ }
2933
+ } else {
2934
+ await this.mouse.click(element.center[0], element.center[1]);
2935
+ await this.underlyingPage.keyboard.down("Control");
2936
+ await this.underlyingPage.keyboard.press("a");
2937
+ await this.underlyingPage.keyboard.up("Control");
2938
+ }
2939
+ await (0, import_utils15.sleep)(100);
2940
+ await this.keyboard.press([{ key: "Backspace" }]);
2941
+ }
2942
+ async moveToPointBeforeScroll(point) {
2943
+ if (point) {
2944
+ await this.mouse.move(point.left, point.top);
2945
+ } else if (!this.everMoved) {
2946
+ const size = await this.size();
2947
+ const targetX = Math.floor(size.width / 2);
2948
+ const targetY = Math.floor(size.height / 2);
2949
+ await this.mouse.move(targetX, targetY);
2950
+ }
2951
+ }
2952
+ async scrollUntilTop(startingPoint) {
2953
+ await this.moveToPointBeforeScroll(startingPoint);
2954
+ return this.mouse.wheel(0, -9999999);
2955
+ }
2956
+ async scrollUntilBottom(startingPoint) {
2957
+ await this.moveToPointBeforeScroll(startingPoint);
2958
+ return this.mouse.wheel(0, 9999999);
2959
+ }
2960
+ async scrollUntilLeft(startingPoint) {
2961
+ await this.moveToPointBeforeScroll(startingPoint);
2962
+ return this.mouse.wheel(-9999999, 0);
2963
+ }
2964
+ async scrollUntilRight(startingPoint) {
2965
+ await this.moveToPointBeforeScroll(startingPoint);
2966
+ return this.mouse.wheel(9999999, 0);
2967
+ }
2968
+ async scrollUp(distance, startingPoint) {
2969
+ const innerHeight = await this.evaluate(() => window.innerHeight);
2970
+ const scrollDistance = distance || innerHeight * 0.7;
2971
+ await this.moveToPointBeforeScroll(startingPoint);
2972
+ return this.mouse.wheel(0, -scrollDistance);
2973
+ }
2974
+ async scrollDown(distance, startingPoint) {
2975
+ const innerHeight = await this.evaluate(() => window.innerHeight);
2976
+ const scrollDistance = distance || innerHeight * 0.7;
2977
+ await this.moveToPointBeforeScroll(startingPoint);
2978
+ return this.mouse.wheel(0, scrollDistance);
2979
+ }
2980
+ async scrollLeft(distance, startingPoint) {
2981
+ const innerWidth = await this.evaluate(() => window.innerWidth);
2982
+ const scrollDistance = distance || innerWidth * 0.7;
2983
+ await this.moveToPointBeforeScroll(startingPoint);
2984
+ return this.mouse.wheel(-scrollDistance, 0);
2985
+ }
2986
+ async scrollRight(distance, startingPoint) {
2987
+ const innerWidth = await this.evaluate(() => window.innerWidth);
2988
+ const scrollDistance = distance || innerWidth * 0.7;
2989
+ await this.moveToPointBeforeScroll(startingPoint);
2990
+ return this.mouse.wheel(scrollDistance, 0);
2991
+ }
2992
+ async navigate(url) {
2993
+ if (this.pageType === "puppeteer") {
2994
+ await this.underlyingPage.goto(url);
2995
+ } else if (this.pageType === "playwright") {
2996
+ await this.underlyingPage.goto(url);
2997
+ } else {
2998
+ throw new Error("Unsupported page type for navigate");
2999
+ }
3000
+ }
3001
+ async destroy() {
3002
+ }
3003
+ };
3004
+
3005
+ // src/puppeteer/page.ts
3006
+ var WebPage = class extends Page {
3007
+ constructor(page, opts) {
3008
+ super(page, "puppeteer");
3009
+ const {
3010
+ waitForNavigationTimeout = import_constants4.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
3011
+ waitForNetworkIdleTimeout = import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT
3012
+ } = opts ?? {};
3013
+ this.waitForNavigationTimeout = waitForNavigationTimeout;
3014
+ this.waitForNetworkIdleTimeout = waitForNetworkIdleTimeout;
3015
+ }
3016
+ async waitUntilNetworkIdle(options) {
3017
+ await this.underlyingPage.waitForNetworkIdle({
3018
+ idleTime: options?.idleTime || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
3019
+ concurrency: options?.concurrency || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
3020
+ timeout: options?.timeout || this.waitForNetworkIdleTimeout
3021
+ });
3022
+ }
3023
+ };
3024
+
3025
+ // src/puppeteer/index.ts
3026
+ var import_env3 = require("misoai-shared/env");
3027
+ var debug5 = (0, import_logger6.getDebug)("puppeteer:agent");
3028
+ var PuppeteerAgent = class extends PageAgent {
3029
+ constructor(page, opts) {
3030
+ const webPage = new WebPage(page);
3031
+ super(webPage, opts);
3032
+ const { forceSameTabNavigation = true } = opts ?? {};
3033
+ if (forceSameTabNavigation) {
3034
+ forceClosePopup(page, debug5);
3035
+ }
3036
+ }
3037
+ };
3038
+ // Annotate the CommonJS export names for ESM import in node:
3039
+ 0 && (module.exports = {
3040
+ PuppeteerAgent,
3041
+ PuppeteerWebPage,
3042
+ overrideAIConfig
3043
+ });
3044
+
3045
+ //# sourceMappingURL=puppeteer.js.map