misoai-web 1.0.3 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/es/agent.js +2574 -0
  2. package/dist/es/agent.js.map +1 -0
  3. package/dist/es/bridge-mode-browser.js +955 -0
  4. package/dist/es/bridge-mode-browser.js.map +1 -0
  5. package/dist/es/bridge-mode.js +2935 -0
  6. package/dist/es/bridge-mode.js.map +1 -0
  7. package/dist/es/chrome-extension.js +3322 -0
  8. package/dist/es/chrome-extension.js.map +1 -0
  9. package/dist/es/index.js +3190 -0
  10. package/dist/es/index.js.map +1 -0
  11. package/dist/es/midscene-playground.js +2910 -0
  12. package/dist/es/midscene-playground.js.map +1 -0
  13. package/dist/es/midscene-server.js +247 -0
  14. package/dist/es/midscene-server.js.map +1 -0
  15. package/dist/es/playground.js +2681 -0
  16. package/dist/es/playground.js.map +1 -0
  17. package/dist/es/playwright-report.js +120 -0
  18. package/dist/es/playwright-report.js.map +1 -0
  19. package/dist/es/playwright.js +3135 -0
  20. package/dist/es/playwright.js.map +1 -0
  21. package/dist/es/puppeteer-agent-launcher.js +3085 -0
  22. package/dist/es/puppeteer-agent-launcher.js.map +1 -0
  23. package/dist/es/puppeteer.js +2932 -0
  24. package/dist/es/puppeteer.js.map +1 -0
  25. package/dist/es/ui-utils.js +106 -0
  26. package/dist/es/ui-utils.js.map +1 -0
  27. package/dist/es/utils.js +197 -0
  28. package/dist/es/utils.js.map +1 -0
  29. package/dist/es/yaml.js +351 -0
  30. package/dist/es/yaml.js.map +1 -0
  31. package/dist/lib/agent.js +2589 -0
  32. package/dist/lib/agent.js.map +1 -0
  33. package/dist/lib/bridge-mode-browser.js +989 -0
  34. package/dist/lib/bridge-mode-browser.js.map +1 -0
  35. package/dist/lib/bridge-mode.js +2955 -0
  36. package/dist/lib/bridge-mode.js.map +1 -0
  37. package/dist/lib/chrome-extension.js +3339 -0
  38. package/dist/lib/chrome-extension.js.map +1 -0
  39. package/dist/lib/index.js +3206 -0
  40. package/dist/lib/index.js.map +1 -0
  41. package/dist/lib/midscene-playground.js +2914 -0
  42. package/dist/lib/midscene-playground.js.map +1 -0
  43. package/dist/lib/midscene-server.js +273 -0
  44. package/dist/lib/midscene-server.js.map +1 -0
  45. package/dist/lib/playground.js +2700 -0
  46. package/dist/lib/playground.js.map +1 -0
  47. package/dist/lib/playwright-report.js +148 -0
  48. package/dist/lib/playwright-report.js.map +1 -0
  49. package/dist/lib/playwright.js +3152 -0
  50. package/dist/lib/playwright.js.map +1 -0
  51. package/dist/lib/puppeteer-agent-launcher.js +3098 -0
  52. package/dist/lib/puppeteer-agent-launcher.js.map +1 -0
  53. package/dist/lib/puppeteer.js +2943 -0
  54. package/dist/lib/puppeteer.js.map +1 -0
  55. package/dist/lib/ui-utils.js +137 -0
  56. package/dist/lib/ui-utils.js.map +1 -0
  57. package/dist/lib/utils.js +235 -0
  58. package/dist/lib/utils.js.map +1 -0
  59. package/dist/lib/yaml.js +390 -0
  60. package/dist/lib/yaml.js.map +1 -0
  61. package/dist/types/agent.d.ts +264 -0
  62. package/dist/types/bridge-mode-browser.d.ts +9 -0
  63. package/dist/types/bridge-mode.d.ts +40 -0
  64. package/dist/types/browser-a1877d18.d.ts +37 -0
  65. package/dist/types/chrome-extension.d.ts +18 -0
  66. package/dist/types/index.d.ts +16 -0
  67. package/dist/types/midscene-playground.d.ts +2 -0
  68. package/dist/types/midscene-server.d.ts +31 -0
  69. package/dist/types/page-663ece08.d.ts +333 -0
  70. package/dist/types/playground.d.ts +17 -0
  71. package/dist/types/playwright-report.d.ts +11 -0
  72. package/dist/types/playwright.d.ts +87 -0
  73. package/dist/types/puppeteer-agent-launcher.d.ts +40 -0
  74. package/dist/types/puppeteer.d.ts +17 -0
  75. package/dist/types/ui-utils.d.ts +14 -0
  76. package/dist/types/utils-badc824e.d.ts +34 -0
  77. package/dist/types/utils.d.ts +8 -0
  78. package/dist/types/yaml.d.ts +15 -0
  79. package/iife-script/htmlElement.js +99 -37
  80. package/iife-script/htmlElementDebug.js +92 -9
  81. package/package.json +2 -2
@@ -0,0 +1,3098 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/puppeteer/agent-launcher.ts
31
+ var agent_launcher_exports = {};
32
+ __export(agent_launcher_exports, {
33
+ defaultUA: () => defaultUA,
34
+ defaultViewportHeight: () => defaultViewportHeight,
35
+ defaultViewportScale: () => defaultViewportScale,
36
+ defaultViewportWidth: () => defaultViewportWidth,
37
+ defaultWaitForNetworkIdleTimeout: () => defaultWaitForNetworkIdleTimeout,
38
+ launchPuppeteerPage: () => launchPuppeteerPage,
39
+ puppeteerAgentForTarget: () => puppeteerAgentForTarget
40
+ });
41
+ module.exports = __toCommonJS(agent_launcher_exports);
42
+ var import_node_fs3 = require("fs");
43
+ var import_logger7 = require("misoai-shared/logger");
44
+ var import_utils18 = require("misoai-shared/utils");
45
+
46
+ // src/common/agent.ts
47
+ var import_misoai_core2 = require("misoai-core");
48
+ var import_js_yaml4 = __toESM(require("js-yaml"));
49
+
50
+ // src/yaml/player.ts
51
+ var import_node_fs = require("fs");
52
+ var import_node_path = require("path");
53
+ var import_utils = require("misoai-shared/utils");
54
+ var import_common = require("misoai-shared/common");
55
+ var ScriptPlayer = class {
56
+ constructor(script, setupAgent, onTaskStatusChange) {
57
+ this.script = script;
58
+ this.setupAgent = setupAgent;
59
+ this.onTaskStatusChange = onTaskStatusChange;
60
+ this.taskStatusList = [];
61
+ this.status = "init";
62
+ this.unnamedResultIndex = 0;
63
+ this.pageAgent = null;
64
+ this.result = {};
65
+ if (import_utils.ifInBrowser) {
66
+ this.output = void 0;
67
+ } else if (script.target?.output) {
68
+ this.output = (0, import_node_path.resolve)(process.cwd(), script.target.output);
69
+ } else {
70
+ this.output = (0, import_node_path.join)((0, import_common.getMidsceneRunSubDir)("output"), `${process.pid}.json`);
71
+ }
72
+ this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({
73
+ ...task,
74
+ index: taskIndex,
75
+ status: "init",
76
+ totalSteps: task.flow?.length || 0
77
+ }));
78
+ }
79
+ setResult(key, value) {
80
+ const keyToUse = key || this.unnamedResultIndex++;
81
+ if (this.result[keyToUse]) {
82
+ console.warn(`result key ${keyToUse} already exists, will overwrite`);
83
+ }
84
+ this.result[keyToUse] = value;
85
+ this.flushResult();
86
+ }
87
+ setPlayerStatus(status, error) {
88
+ this.status = status;
89
+ this.errorInSetup = error;
90
+ }
91
+ notifyCurrentTaskStatusChange(taskIndex) {
92
+ const taskIndexToNotify = typeof taskIndex === "number" ? taskIndex : this.currentTaskIndex;
93
+ if (typeof taskIndexToNotify !== "number") {
94
+ return;
95
+ }
96
+ const taskStatus = this.taskStatusList[taskIndexToNotify];
97
+ if (this.onTaskStatusChange) {
98
+ this.onTaskStatusChange(taskStatus);
99
+ }
100
+ }
101
+ async setTaskStatus(index, statusValue, error) {
102
+ this.taskStatusList[index].status = statusValue;
103
+ if (error) {
104
+ this.taskStatusList[index].error = error;
105
+ }
106
+ this.notifyCurrentTaskStatusChange(index);
107
+ }
108
+ setTaskIndex(taskIndex) {
109
+ this.currentTaskIndex = taskIndex;
110
+ }
111
+ flushResult() {
112
+ if (Object.keys(this.result).length && this.output) {
113
+ const output = (0, import_node_path.resolve)(process.cwd(), this.output);
114
+ const outputDir = (0, import_node_path.dirname)(output);
115
+ if (!(0, import_node_fs.existsSync)(outputDir)) {
116
+ (0, import_node_fs.mkdirSync)(outputDir, { recursive: true });
117
+ }
118
+ (0, import_node_fs.writeFileSync)(output, JSON.stringify(this.result, void 0, 2));
119
+ }
120
+ }
121
+ async playTask(taskStatus, agent) {
122
+ const { flow } = taskStatus;
123
+ (0, import_utils.assert)(flow, "missing flow in task");
124
+ for (const flowItemIndex in flow) {
125
+ const currentStep = Number.parseInt(flowItemIndex, 10);
126
+ taskStatus.currentStep = currentStep;
127
+ const flowItem = flow[flowItemIndex];
128
+ if ("aiAction" in flowItem || "ai" in flowItem) {
129
+ const actionTask = flowItem;
130
+ const prompt = actionTask.aiAction || actionTask.ai;
131
+ (0, import_utils.assert)(prompt, "missing prompt for ai (aiAction)");
132
+ (0, import_utils.assert)(
133
+ typeof prompt === "string",
134
+ "prompt for aiAction must be a string"
135
+ );
136
+ await agent.aiAction(prompt, {
137
+ cacheable: actionTask.cacheable
138
+ });
139
+ } else if ("aiAssert" in flowItem) {
140
+ const assertTask = flowItem;
141
+ const prompt = assertTask.aiAssert;
142
+ (0, import_utils.assert)(prompt, "missing prompt for aiAssert");
143
+ (0, import_utils.assert)(
144
+ typeof prompt === "string",
145
+ "prompt for aiAssert must be a string"
146
+ );
147
+ await agent.aiAssert(prompt);
148
+ } else if ("aiQuery" in flowItem) {
149
+ const queryTask = flowItem;
150
+ const prompt = queryTask.aiQuery;
151
+ (0, import_utils.assert)(prompt, "missing prompt for aiQuery");
152
+ (0, import_utils.assert)(
153
+ typeof prompt === "string",
154
+ "prompt for aiQuery must be a string"
155
+ );
156
+ const queryResult = await agent.aiQuery(prompt);
157
+ this.setResult(queryTask.name, queryResult);
158
+ } else if ("aiNumber" in flowItem) {
159
+ const numberTask = flowItem;
160
+ const prompt = numberTask.aiNumber;
161
+ (0, import_utils.assert)(prompt, "missing prompt for number");
162
+ (0, import_utils.assert)(
163
+ typeof prompt === "string",
164
+ "prompt for number must be a string"
165
+ );
166
+ const numberResult = await agent.aiNumber(prompt);
167
+ this.setResult(numberTask.name, numberResult);
168
+ } else if ("aiString" in flowItem) {
169
+ const stringTask = flowItem;
170
+ const prompt = stringTask.aiString;
171
+ (0, import_utils.assert)(prompt, "missing prompt for string");
172
+ (0, import_utils.assert)(
173
+ typeof prompt === "string",
174
+ "prompt for string must be a string"
175
+ );
176
+ const stringResult = await agent.aiString(prompt);
177
+ this.setResult(stringTask.name, stringResult);
178
+ } else if ("aiBoolean" in flowItem) {
179
+ const booleanTask = flowItem;
180
+ const prompt = booleanTask.aiBoolean;
181
+ (0, import_utils.assert)(prompt, "missing prompt for boolean");
182
+ (0, import_utils.assert)(
183
+ typeof prompt === "string",
184
+ "prompt for boolean must be a string"
185
+ );
186
+ const booleanResult = await agent.aiBoolean(prompt);
187
+ this.setResult(booleanTask.name, booleanResult);
188
+ } else if ("aiLocate" in flowItem) {
189
+ const locateTask = flowItem;
190
+ const prompt = locateTask.aiLocate;
191
+ (0, import_utils.assert)(prompt, "missing prompt for aiLocate");
192
+ (0, import_utils.assert)(
193
+ typeof prompt === "string",
194
+ "prompt for aiLocate must be a string"
195
+ );
196
+ const locateResult = await agent.aiLocate(prompt);
197
+ this.setResult(locateTask.name, locateResult);
198
+ } else if ("aiWaitFor" in flowItem) {
199
+ const waitForTask = flowItem;
200
+ const prompt = waitForTask.aiWaitFor;
201
+ (0, import_utils.assert)(prompt, "missing prompt for aiWaitFor");
202
+ (0, import_utils.assert)(
203
+ typeof prompt === "string",
204
+ "prompt for aiWaitFor must be a string"
205
+ );
206
+ const timeout = waitForTask.timeout;
207
+ await agent.aiWaitFor(prompt, { timeoutMs: timeout });
208
+ } else if ("sleep" in flowItem) {
209
+ const sleepTask = flowItem;
210
+ const ms = sleepTask.sleep;
211
+ let msNumber = ms;
212
+ if (typeof ms === "string") {
213
+ msNumber = Number.parseInt(ms, 10);
214
+ }
215
+ (0, import_utils.assert)(
216
+ msNumber && msNumber > 0,
217
+ `ms for sleep must be greater than 0, but got ${ms}`
218
+ );
219
+ await new Promise((resolve2) => setTimeout(resolve2, msNumber));
220
+ } else if ("aiTap" in flowItem) {
221
+ const tapTask = flowItem;
222
+ await agent.aiTap(tapTask.aiTap, tapTask);
223
+ } else if ("aiHover" in flowItem) {
224
+ const hoverTask = flowItem;
225
+ await agent.aiHover(hoverTask.aiHover, hoverTask);
226
+ } else if ("aiInput" in flowItem) {
227
+ const inputTask = flowItem;
228
+ await agent.aiInput(inputTask.aiInput, inputTask.locate, inputTask);
229
+ } else if ("aiKeyboardPress" in flowItem) {
230
+ const keyboardPressTask = flowItem;
231
+ await agent.aiKeyboardPress(
232
+ keyboardPressTask.aiKeyboardPress,
233
+ keyboardPressTask.locate,
234
+ keyboardPressTask
235
+ );
236
+ } else if ("aiScroll" in flowItem) {
237
+ const scrollTask = flowItem;
238
+ await agent.aiScroll(scrollTask, scrollTask.locate, scrollTask);
239
+ } else if ("javascript" in flowItem) {
240
+ const evaluateJavaScriptTask = flowItem;
241
+ const result = await agent.evaluateJavaScript(
242
+ evaluateJavaScriptTask.javascript
243
+ );
244
+ this.setResult(evaluateJavaScriptTask.name, result);
245
+ } else {
246
+ throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
247
+ }
248
+ }
249
+ this.reportFile = agent.reportFile;
250
+ }
251
+ async run() {
252
+ const { target, web, android, tasks } = this.script;
253
+ const webEnv = web || target;
254
+ const androidEnv = android;
255
+ const platform = webEnv || androidEnv;
256
+ this.setPlayerStatus("running");
257
+ let agent = null;
258
+ let freeFn = [];
259
+ try {
260
+ const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(
261
+ platform
262
+ );
263
+ agent = newAgent;
264
+ const originalOnTaskStartTip = agent.onTaskStartTip;
265
+ agent.onTaskStartTip = (tip) => {
266
+ if (this.status === "running") {
267
+ this.agentStatusTip = tip;
268
+ }
269
+ originalOnTaskStartTip?.(tip);
270
+ };
271
+ freeFn = [
272
+ ...newFreeFn || [],
273
+ {
274
+ name: "restore-agent-onTaskStartTip",
275
+ fn: () => {
276
+ if (agent) {
277
+ agent.onTaskStartTip = originalOnTaskStartTip;
278
+ }
279
+ }
280
+ }
281
+ ];
282
+ } catch (e) {
283
+ this.setPlayerStatus("error", e);
284
+ return;
285
+ }
286
+ this.pageAgent = agent;
287
+ let taskIndex = 0;
288
+ this.setPlayerStatus("running");
289
+ let errorFlag = false;
290
+ while (taskIndex < tasks.length) {
291
+ const taskStatus = this.taskStatusList[taskIndex];
292
+ this.setTaskStatus(taskIndex, "running");
293
+ this.setTaskIndex(taskIndex);
294
+ try {
295
+ await this.playTask(taskStatus, this.pageAgent);
296
+ this.setTaskStatus(taskIndex, "done");
297
+ } catch (e) {
298
+ this.setTaskStatus(taskIndex, "error", e);
299
+ if (taskStatus.continueOnError) {
300
+ } else {
301
+ this.reportFile = agent.reportFile;
302
+ errorFlag = true;
303
+ break;
304
+ }
305
+ }
306
+ this.reportFile = agent.reportFile;
307
+ taskIndex++;
308
+ }
309
+ if (errorFlag) {
310
+ this.setPlayerStatus("error");
311
+ } else {
312
+ this.setPlayerStatus("done");
313
+ }
314
+ this.agentStatusTip = "";
315
+ for (const fn of freeFn) {
316
+ try {
317
+ await fn.fn();
318
+ } catch (e) {
319
+ }
320
+ }
321
+ }
322
+ };
323
+
324
+ // src/yaml/builder.ts
325
+ var import_js_yaml = __toESM(require("js-yaml"));
326
+
327
+ // src/yaml/utils.ts
328
+ var import_utils2 = require("misoai-shared/utils");
329
+ var import_js_yaml2 = __toESM(require("js-yaml"));
330
+ function interpolateEnvVars(content) {
331
+ return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
332
+ const value = process.env[envVar.trim()];
333
+ if (value === void 0) {
334
+ throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
335
+ }
336
+ return value;
337
+ });
338
+ }
339
+ function parseYamlScript(content, filePath, ignoreCheckingTarget) {
340
+ let processedContent = content;
341
+ if (content.indexOf("android") !== -1 && content.match(/deviceId:\s*(\d+)/)) {
342
+ let matchedDeviceId;
343
+ processedContent = content.replace(
344
+ /deviceId:\s*(\d+)/g,
345
+ (match, deviceId) => {
346
+ matchedDeviceId = deviceId;
347
+ return `deviceId: '${deviceId}'`;
348
+ }
349
+ );
350
+ console.warn(
351
+ `please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`
352
+ );
353
+ }
354
+ const interpolatedContent = interpolateEnvVars(processedContent);
355
+ const obj = import_js_yaml2.default.load(interpolatedContent, {
356
+ schema: import_js_yaml2.default.JSON_SCHEMA
357
+ });
358
+ const pathTip = filePath ? `, failed to load ${filePath}` : "";
359
+ const android = typeof obj.android !== "undefined" ? Object.assign({}, obj.android || {}) : void 0;
360
+ const webConfig = obj.web || obj.target;
361
+ const web = typeof webConfig !== "undefined" ? Object.assign({}, webConfig || {}) : void 0;
362
+ if (!ignoreCheckingTarget) {
363
+ (0, import_utils2.assert)(
364
+ web || android,
365
+ `at least one of "target", "web", or "android" properties is required in yaml script${pathTip}`
366
+ );
367
+ (0, import_utils2.assert)(
368
+ web && !android || !web && android,
369
+ `only one of "target", "web", or "android" properties is allowed in yaml script${pathTip}`
370
+ );
371
+ if (web || android) {
372
+ (0, import_utils2.assert)(
373
+ typeof web === "object" || typeof android === "object",
374
+ `property "target/web/android" must be an object${pathTip}`
375
+ );
376
+ }
377
+ }
378
+ (0, import_utils2.assert)(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
379
+ (0, import_utils2.assert)(
380
+ Array.isArray(obj.tasks),
381
+ `property "tasks" must be an array in yaml script, but got ${obj.tasks}`
382
+ );
383
+ return obj;
384
+ }
385
+
386
+ // src/common/agent.ts
387
+ var import_utils11 = require("misoai-core/utils");
388
+ var import_constants2 = require("misoai-shared/constants");
389
+ var import_env2 = require("misoai-shared/env");
390
+ var import_logger4 = require("misoai-shared/logger");
391
+ var import_utils12 = require("misoai-shared/utils");
392
+
393
+ // src/common/tasks.ts
394
+ var import_misoai_core = require("misoai-core");
395
+ var import_ai_model2 = require("misoai-core/ai-model");
396
+ var import_utils5 = require("misoai-core/utils");
397
+ var import_constants = require("misoai-shared/constants");
398
+ var import_logger = require("misoai-shared/logger");
399
+ var import_utils6 = require("misoai-shared/utils");
400
+
401
+ // src/common/ui-utils.ts
402
+ function typeStr(task) {
403
+ return task.subType && task.subType !== "Plan" ? `${task.type} / ${task.subType || ""}` : task.type;
404
+ }
405
+ function getKeyCommands(value) {
406
+ const keys = Array.isArray(value) ? value : [value];
407
+ return keys.reduce((acc, k) => {
408
+ const includeMeta = keys.includes("Meta") || keys.includes("Control");
409
+ if (includeMeta && (k === "a" || k === "A")) {
410
+ return acc.concat([{ key: k, command: "SelectAll" }]);
411
+ }
412
+ if (includeMeta && (k === "c" || k === "C")) {
413
+ return acc.concat([{ key: k, command: "Copy" }]);
414
+ }
415
+ if (includeMeta && (k === "v" || k === "V")) {
416
+ return acc.concat([{ key: k, command: "Paste" }]);
417
+ }
418
+ return acc.concat([{ key: k }]);
419
+ }, []);
420
+ }
421
+ function locateParamStr(locate) {
422
+ if (!locate) {
423
+ return "";
424
+ }
425
+ if (typeof locate === "string") {
426
+ return locate;
427
+ }
428
+ return locate.prompt;
429
+ }
430
+ function scrollParamStr(scrollParam) {
431
+ if (!scrollParam) {
432
+ return "";
433
+ }
434
+ return `${scrollParam.direction || "down"}, ${scrollParam.scrollType || "once"}, ${scrollParam.distance || "distance-not-set"}`;
435
+ }
436
+ function taskTitleStr(type, prompt) {
437
+ if (prompt) {
438
+ return `${type} - ${prompt}`;
439
+ }
440
+ return type;
441
+ }
442
+ function paramStr(task) {
443
+ let value;
444
+ if (task.type === "Planning") {
445
+ value = task?.param?.userInstruction;
446
+ }
447
+ if (task.type === "Insight") {
448
+ value = task?.param?.prompt || task?.param?.id || task?.param?.dataDemand || task?.param?.assertion;
449
+ }
450
+ if (task.type === "Action") {
451
+ const locate = task?.locate;
452
+ const locateStr = locate ? locateParamStr(locate) : "";
453
+ value = task.thought || "";
454
+ if (typeof task?.param?.timeMs === "number") {
455
+ value = `${task?.param?.timeMs}ms`;
456
+ } else if (typeof task?.param?.scrollType === "string") {
457
+ value = scrollParamStr(task?.param);
458
+ } else if (typeof task?.param?.value !== "undefined") {
459
+ value = task?.param?.value;
460
+ }
461
+ if (locateStr) {
462
+ if (value) {
463
+ value = `${locateStr} - ${value}`;
464
+ } else {
465
+ value = locateStr;
466
+ }
467
+ }
468
+ }
469
+ if (typeof value === "undefined")
470
+ return "";
471
+ return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
472
+ }
473
+
474
+ // src/common/utils.ts
475
+ var import_ai_model = require("misoai-core/ai-model");
476
+ var import_utils3 = require("misoai-core/utils");
477
+ var import_env = require("misoai-shared/env");
478
+ var import_extractor = require("misoai-shared/extractor");
479
+ var import_img = require("misoai-shared/img");
480
+ var import_utils4 = require("misoai-shared/utils");
481
+ var import_dayjs = __toESM(require("dayjs"));
482
+
483
+ // src/web-element.ts
484
+ var WebElementInfo = class {
485
+ constructor({
486
+ content,
487
+ rect,
488
+ // page,
489
+ locator,
490
+ id,
491
+ attributes,
492
+ indexId,
493
+ xpaths
494
+ }) {
495
+ this.content = content;
496
+ this.rect = rect;
497
+ this.center = [
498
+ Math.floor(rect.left + rect.width / 2),
499
+ Math.floor(rect.top + rect.height / 2)
500
+ ];
501
+ this.locator = locator;
502
+ this.id = id;
503
+ this.attributes = attributes;
504
+ this.indexId = indexId;
505
+ this.xpaths = xpaths;
506
+ }
507
+ };
508
+
509
+ // src/common/utils.ts
510
+ async function parseContextFromWebPage(page, _opt) {
511
+ (0, import_utils4.assert)(page, "page is required");
512
+ if (page._forceUsePageContext) {
513
+ return await page._forceUsePageContext();
514
+ }
515
+ const url = await page.url();
516
+ (0, import_utils3.uploadTestInfoToServer)({ testUrl: url });
517
+ let screenshotBase64;
518
+ let tree;
519
+ await Promise.all([
520
+ page.screenshotBase64().then((base64) => {
521
+ screenshotBase64 = base64;
522
+ }),
523
+ page.getElementsNodeTree().then(async (treeRoot) => {
524
+ tree = treeRoot;
525
+ })
526
+ ]);
527
+ const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
528
+ const { rect, id, content, attributes, locator, indexId } = elementInfo;
529
+ return new WebElementInfo({
530
+ rect,
531
+ locator,
532
+ id,
533
+ content,
534
+ attributes,
535
+ indexId
536
+ });
537
+ });
538
+ (0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
539
+ const elementsInfo = (0, import_extractor.treeToList)(webTree);
540
+ const size = await page.size();
541
+ if (size.dpr && size.dpr > 1) {
542
+ screenshotBase64 = await (0, import_img.resizeImgBase64)(screenshotBase64, {
543
+ width: size.width,
544
+ height: size.height
545
+ });
546
+ }
547
+ return {
548
+ content: elementsInfo,
549
+ tree: webTree,
550
+ size,
551
+ screenshotBase64,
552
+ url
553
+ };
554
+ }
555
+ function reportFileName(tag = "web") {
556
+ const reportTagName = (0, import_env.getAIConfig)(import_env.MIDSCENE_REPORT_TAG_NAME);
557
+ const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss");
558
+ const uniqueId = (0, import_utils4.uuid)().substring(0, 8);
559
+ return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
560
+ }
561
+ function printReportMsg(filepath) {
562
+ (0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
563
+ }
564
+ function replaceIllegalPathCharsAndSpace(str) {
565
+ return str.replace(/[/\\:*?"<>| ]/g, "-");
566
+ }
567
+ function forceClosePopup(page, debug6) {
568
+ page.on("popup", async (popup) => {
569
+ if (!popup) {
570
+ console.warn("got a popup event, but the popup is not ready yet, skip");
571
+ return;
572
+ }
573
+ const url = await popup.url();
574
+ console.log(`Popup opened: ${url}`);
575
+ if (!popup.isClosed()) {
576
+ try {
577
+ await popup.close();
578
+ } catch (error) {
579
+ debug6(`failed to close popup ${url}, error: ${error}`);
580
+ }
581
+ } else {
582
+ debug6(`popup is already closed, skip close ${url}`);
583
+ }
584
+ if (!page.isClosed()) {
585
+ try {
586
+ await page.goto(url);
587
+ } catch (error) {
588
+ debug6(`failed to goto ${url}, error: ${error}`);
589
+ }
590
+ } else {
591
+ debug6(`page is already closed, skip goto ${url}`);
592
+ }
593
+ });
594
+ }
595
+ function matchElementFromPlan(planLocateParam, tree) {
596
+ if (!planLocateParam) {
597
+ return void 0;
598
+ }
599
+ if (planLocateParam.id) {
600
+ return (0, import_extractor.getNodeFromCacheList)(planLocateParam.id);
601
+ }
602
+ if (planLocateParam.bbox) {
603
+ const centerPosition = {
604
+ x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
605
+ y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
606
+ };
607
+ let element = (0, import_ai_model.elementByPositionWithElementInfo)(tree, centerPosition);
608
+ if (!element) {
609
+ element = (0, import_extractor.generateElementByPosition)(centerPosition);
610
+ }
611
+ return element;
612
+ }
613
+ return void 0;
614
+ }
615
+
616
+ // src/common/tasks.ts
617
+ var debug = (0, import_logger.getDebug)("page-task-executor");
618
+ var replanningCountLimit = 10;
619
+ var isAndroidPage = (page) => {
620
+ return page.pageType === "android";
621
+ };
622
+ var PageTaskExecutor = class {
623
+ constructor(page, insight, opts) {
624
+ this.conversationHistory = [];
625
+ this.page = page;
626
+ this.insight = insight;
627
+ this.taskCache = opts.taskCache;
628
+ this.onTaskStartCallback = opts?.onTaskStart;
629
+ }
630
+ async recordScreenshot(timing) {
631
+ const base64 = await this.page.screenshotBase64();
632
+ const item = {
633
+ type: "screenshot",
634
+ ts: Date.now(),
635
+ screenshot: base64,
636
+ timing
637
+ };
638
+ return item;
639
+ }
640
+ async getElementXpath(pageContext, element) {
641
+ let elementId = element?.id;
642
+ if (element?.attributes?.nodeType === import_constants.NodeType.POSITION) {
643
+ await this.insight.contextRetrieverFn("locate");
644
+ const info = (0, import_ai_model2.elementByPositionWithElementInfo)(
645
+ pageContext.tree,
646
+ {
647
+ x: element.center[0],
648
+ y: element.center[1]
649
+ },
650
+ {
651
+ requireStrictDistance: false,
652
+ filterPositionElements: true
653
+ }
654
+ );
655
+ if (info?.id) {
656
+ elementId = info.id;
657
+ } else {
658
+ debug(
659
+ "no element id found for position node, will not update cache",
660
+ element
661
+ );
662
+ }
663
+ }
664
+ if (!elementId) {
665
+ return void 0;
666
+ }
667
+ try {
668
+ const result = await this.page.getXpathsById(elementId);
669
+ return result;
670
+ } catch (error) {
671
+ debug("getXpathsById error: ", error);
672
+ }
673
+ }
674
+ prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
675
+ const taskWithScreenshot = {
676
+ ...taskApply,
677
+ executor: async (param, context, ...args) => {
678
+ const recorder = [];
679
+ const { task } = context;
680
+ task.recorder = recorder;
681
+ const shot = await this.recordScreenshot(`before ${task.type}`);
682
+ recorder.push(shot);
683
+ const result = await taskApply.executor(param, context, ...args);
684
+ if (taskApply.type === "Action") {
685
+ await Promise.all([
686
+ (async () => {
687
+ await (0, import_utils5.sleep)(100);
688
+ if (this.page.waitUntilNetworkIdle) {
689
+ try {
690
+ await this.page.waitUntilNetworkIdle();
691
+ } catch (error) {
692
+ }
693
+ }
694
+ })(),
695
+ (0, import_utils5.sleep)(200)
696
+ ]);
697
+ }
698
+ if (appendAfterExecution) {
699
+ const shot2 = await this.recordScreenshot("after Action");
700
+ recorder.push(shot2);
701
+ }
702
+ return result;
703
+ }
704
+ };
705
+ return taskWithScreenshot;
706
+ }
707
+ async convertPlanToExecutable(plans, opts) {
708
+ const tasks = [];
709
+ plans.forEach((plan2) => {
710
+ if (plan2.type === "Locate") {
711
+ if (plan2.locate === null || plan2.locate?.id === null || plan2.locate?.id === "null") {
712
+ return;
713
+ }
714
+ const taskFind = {
715
+ type: "Insight",
716
+ subType: "Locate",
717
+ param: plan2.locate ? {
718
+ ...plan2.locate,
719
+ cacheable: opts?.cacheable
720
+ } : void 0,
721
+ thought: plan2.thought,
722
+ locate: plan2.locate,
723
+ executor: async (param, taskContext) => {
724
+ const { task } = taskContext;
725
+ (0, import_utils6.assert)(
726
+ param?.prompt || param?.id || param?.bbox,
727
+ "No prompt or id or position or bbox to locate"
728
+ );
729
+ let insightDump;
730
+ let usage;
731
+ const dumpCollector = (dump) => {
732
+ insightDump = dump;
733
+ usage = dump?.taskInfo?.usage;
734
+ task.log = {
735
+ dump: insightDump
736
+ };
737
+ task.usage = usage;
738
+ };
739
+ this.insight.onceDumpUpdatedFn = dumpCollector;
740
+ const shotTime = Date.now();
741
+ const pageContext = await this.insight.contextRetrieverFn("locate");
742
+ task.pageContext = pageContext;
743
+ const recordItem = {
744
+ type: "screenshot",
745
+ ts: shotTime,
746
+ screenshot: pageContext.screenshotBase64,
747
+ timing: "before locate"
748
+ };
749
+ task.recorder = [recordItem];
750
+ let cacheHitFlag = false;
751
+ const cachePrompt = param.prompt;
752
+ const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
753
+ const xpaths = locateCacheRecord?.cacheContent?.xpaths;
754
+ let elementFromCache = null;
755
+ try {
756
+ if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
757
+ for (let i = 0; i < xpaths.length; i++) {
758
+ const element2 = await this.page.getElementInfoByXpath(
759
+ xpaths[i]
760
+ );
761
+ if (element2?.id) {
762
+ elementFromCache = element2;
763
+ debug("cache hit, prompt: %s", cachePrompt);
764
+ cacheHitFlag = true;
765
+ debug(
766
+ "found a new new element with same xpath, xpath: %s, id: %s",
767
+ xpaths[i],
768
+ element2?.id
769
+ );
770
+ break;
771
+ }
772
+ }
773
+ }
774
+ } catch (error) {
775
+ debug("get element info by xpath error: ", error);
776
+ }
777
+ const startTime = Date.now();
778
+ const element = elementFromCache || // try to match element from cache
779
+ matchElementFromPlan(param, pageContext.tree) || // try to match element from plan
780
+ (await this.insight.locate(param, {
781
+ context: pageContext
782
+ })).element;
783
+ const aiCost = Date.now() - startTime;
784
+ let currentXpaths;
785
+ if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
786
+ const elementXpaths = await this.getElementXpath(
787
+ pageContext,
788
+ element
789
+ );
790
+ if (elementXpaths?.length) {
791
+ currentXpaths = elementXpaths;
792
+ this.taskCache.updateOrAppendCacheRecord(
793
+ {
794
+ type: "locate",
795
+ prompt: cachePrompt,
796
+ xpaths: elementXpaths
797
+ },
798
+ locateCacheRecord
799
+ );
800
+ } else {
801
+ debug(
802
+ "no xpaths found, will not update cache",
803
+ cachePrompt,
804
+ elementXpaths
805
+ );
806
+ }
807
+ }
808
+ if (!element) {
809
+ throw new Error(`Element not found: ${param.prompt}`);
810
+ }
811
+ return {
812
+ output: {
813
+ element
814
+ },
815
+ pageContext,
816
+ cache: {
817
+ hit: cacheHitFlag,
818
+ originalXpaths: xpaths,
819
+ currentXpaths
820
+ },
821
+ aiCost
822
+ };
823
+ }
824
+ };
825
+ tasks.push(taskFind);
826
+ } else if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
827
+ const assertPlan = plan2;
828
+ const taskAssert = {
829
+ type: "Insight",
830
+ subType: "Assert",
831
+ param: assertPlan.param,
832
+ thought: assertPlan.thought,
833
+ locate: assertPlan.locate,
834
+ executor: async (param, taskContext) => {
835
+ const { task } = taskContext;
836
+ let insightDump;
837
+ const dumpCollector = (dump) => {
838
+ insightDump = dump;
839
+ };
840
+ this.insight.onceDumpUpdatedFn = dumpCollector;
841
+ const assertion = await this.insight.assert(
842
+ assertPlan.param.assertion
843
+ );
844
+ if (!assertion.pass) {
845
+ if (plan2.type === "Assert") {
846
+ task.output = assertion;
847
+ task.log = {
848
+ dump: insightDump
849
+ };
850
+ throw new Error(
851
+ assertion.thought || "Assertion failed without reason"
852
+ );
853
+ }
854
+ task.error = assertion.thought;
855
+ }
856
+ return {
857
+ output: assertion,
858
+ log: {
859
+ dump: insightDump
860
+ },
861
+ usage: assertion.usage
862
+ };
863
+ }
864
+ };
865
+ tasks.push(taskAssert);
866
+ } else if (plan2.type === "Input") {
867
+ const taskActionInput = {
868
+ type: "Action",
869
+ subType: "Input",
870
+ param: plan2.param,
871
+ thought: plan2.thought,
872
+ locate: plan2.locate,
873
+ executor: async (taskParam, { element }) => {
874
+ if (element) {
875
+ await this.page.clearInput(element);
876
+ if (!taskParam || !taskParam.value) {
877
+ return;
878
+ }
879
+ await this.page.keyboard.type(taskParam.value);
880
+ } else {
881
+ await this.page.keyboard.type(taskParam.value);
882
+ }
883
+ }
884
+ };
885
+ tasks.push(taskActionInput);
886
+ } else if (plan2.type === "KeyboardPress") {
887
+ const taskActionKeyboardPress = {
888
+ type: "Action",
889
+ subType: "KeyboardPress",
890
+ param: plan2.param,
891
+ thought: plan2.thought,
892
+ locate: plan2.locate,
893
+ executor: async (taskParam) => {
894
+ const keys = getKeyCommands(taskParam.value);
895
+ await this.page.keyboard.press(keys);
896
+ }
897
+ };
898
+ tasks.push(taskActionKeyboardPress);
899
+ } else if (plan2.type === "Tap") {
900
+ const taskActionTap = {
901
+ type: "Action",
902
+ subType: "Tap",
903
+ thought: plan2.thought,
904
+ locate: plan2.locate,
905
+ executor: async (param, { element }) => {
906
+ (0, import_utils6.assert)(element, "Element not found, cannot tap");
907
+ await this.page.mouse.click(element.center[0], element.center[1]);
908
+ }
909
+ };
910
+ tasks.push(taskActionTap);
911
+ } else if (plan2.type === "Drag") {
912
+ const taskActionDrag = {
913
+ type: "Action",
914
+ subType: "Drag",
915
+ param: plan2.param,
916
+ thought: plan2.thought,
917
+ locate: plan2.locate,
918
+ executor: async (taskParam) => {
919
+ (0, import_utils6.assert)(
920
+ taskParam?.start_box && taskParam?.end_box,
921
+ "No start_box or end_box to drag"
922
+ );
923
+ await this.page.mouse.drag(taskParam.start_box, taskParam.end_box);
924
+ }
925
+ };
926
+ tasks.push(taskActionDrag);
927
+ } else if (plan2.type === "Hover") {
928
+ const taskActionHover = {
929
+ type: "Action",
930
+ subType: "Hover",
931
+ thought: plan2.thought,
932
+ locate: plan2.locate,
933
+ executor: async (param, { element }) => {
934
+ (0, import_utils6.assert)(element, "Element not found, cannot hover");
935
+ await this.page.mouse.move(element.center[0], element.center[1]);
936
+ }
937
+ };
938
+ tasks.push(taskActionHover);
939
+ } else if (plan2.type === "Scroll") {
940
+ const taskActionScroll = {
941
+ type: "Action",
942
+ subType: "Scroll",
943
+ param: plan2.param,
944
+ thought: plan2.thought,
945
+ locate: plan2.locate,
946
+ executor: async (taskParam, { element }) => {
947
+ const startingPoint = element ? {
948
+ left: element.center[0],
949
+ top: element.center[1]
950
+ } : void 0;
951
+ const scrollToEventName = taskParam?.scrollType;
952
+ if (scrollToEventName === "untilTop") {
953
+ await this.page.scrollUntilTop(startingPoint);
954
+ } else if (scrollToEventName === "untilBottom") {
955
+ await this.page.scrollUntilBottom(startingPoint);
956
+ } else if (scrollToEventName === "untilRight") {
957
+ await this.page.scrollUntilRight(startingPoint);
958
+ } else if (scrollToEventName === "untilLeft") {
959
+ await this.page.scrollUntilLeft(startingPoint);
960
+ } else if (scrollToEventName === "once" || !scrollToEventName) {
961
+ if (taskParam?.direction === "down" || !taskParam || !taskParam.direction) {
962
+ await this.page.scrollDown(
963
+ taskParam?.distance || void 0,
964
+ startingPoint
965
+ );
966
+ } else if (taskParam.direction === "up") {
967
+ await this.page.scrollUp(
968
+ taskParam.distance || void 0,
969
+ startingPoint
970
+ );
971
+ } else if (taskParam.direction === "left") {
972
+ await this.page.scrollLeft(
973
+ taskParam.distance || void 0,
974
+ startingPoint
975
+ );
976
+ } else if (taskParam.direction === "right") {
977
+ await this.page.scrollRight(
978
+ taskParam.distance || void 0,
979
+ startingPoint
980
+ );
981
+ } else {
982
+ throw new Error(
983
+ `Unknown scroll direction: ${taskParam.direction}`
984
+ );
985
+ }
986
+ await (0, import_utils5.sleep)(500);
987
+ } else {
988
+ throw new Error(
989
+ `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
990
+ taskParam
991
+ )}`
992
+ );
993
+ }
994
+ }
995
+ };
996
+ tasks.push(taskActionScroll);
997
+ } else if (plan2.type === "Sleep") {
998
+ const taskActionSleep = {
999
+ type: "Action",
1000
+ subType: "Sleep",
1001
+ param: plan2.param,
1002
+ thought: plan2.thought,
1003
+ locate: plan2.locate,
1004
+ executor: async (taskParam) => {
1005
+ await (0, import_utils5.sleep)(taskParam?.timeMs || 3e3);
1006
+ }
1007
+ };
1008
+ tasks.push(taskActionSleep);
1009
+ } else if (plan2.type === "Error") {
1010
+ const taskActionError = {
1011
+ type: "Action",
1012
+ subType: "Error",
1013
+ param: plan2.param,
1014
+ thought: plan2.thought || plan2.param?.thought,
1015
+ locate: plan2.locate,
1016
+ executor: async () => {
1017
+ throw new Error(
1018
+ plan2?.thought || plan2.param?.thought || "error without thought"
1019
+ );
1020
+ }
1021
+ };
1022
+ tasks.push(taskActionError);
1023
+ } else if (plan2.type === "ExpectedFalsyCondition") {
1024
+ const taskActionFalsyConditionStatement = {
1025
+ type: "Action",
1026
+ subType: "ExpectedFalsyCondition",
1027
+ param: null,
1028
+ thought: plan2.param?.reason,
1029
+ locate: plan2.locate,
1030
+ executor: async () => {
1031
+ }
1032
+ };
1033
+ tasks.push(taskActionFalsyConditionStatement);
1034
+ } else if (plan2.type === "Finished") {
1035
+ const taskActionFinished = {
1036
+ type: "Action",
1037
+ subType: "Finished",
1038
+ param: null,
1039
+ thought: plan2.thought,
1040
+ locate: plan2.locate,
1041
+ executor: async (param) => {
1042
+ }
1043
+ };
1044
+ tasks.push(taskActionFinished);
1045
+ } else if (plan2.type === "AndroidHomeButton") {
1046
+ const taskActionAndroidHomeButton = {
1047
+ type: "Action",
1048
+ subType: "AndroidHomeButton",
1049
+ param: null,
1050
+ thought: plan2.thought,
1051
+ locate: plan2.locate,
1052
+ executor: async (param) => {
1053
+ (0, import_utils6.assert)(
1054
+ isAndroidPage(this.page),
1055
+ "Cannot use home button on non-Android devices"
1056
+ );
1057
+ await this.page.home();
1058
+ }
1059
+ };
1060
+ tasks.push(taskActionAndroidHomeButton);
1061
+ } else if (plan2.type === "AndroidBackButton") {
1062
+ const taskActionAndroidBackButton = {
1063
+ type: "Action",
1064
+ subType: "AndroidBackButton",
1065
+ param: null,
1066
+ thought: plan2.thought,
1067
+ locate: plan2.locate,
1068
+ executor: async (param) => {
1069
+ (0, import_utils6.assert)(
1070
+ isAndroidPage(this.page),
1071
+ "Cannot use back button on non-Android devices"
1072
+ );
1073
+ await this.page.back();
1074
+ }
1075
+ };
1076
+ tasks.push(taskActionAndroidBackButton);
1077
+ } else if (plan2.type === "AndroidRecentAppsButton") {
1078
+ const taskActionAndroidRecentAppsButton = {
1079
+ type: "Action",
1080
+ subType: "AndroidRecentAppsButton",
1081
+ param: null,
1082
+ thought: plan2.thought,
1083
+ locate: plan2.locate,
1084
+ executor: async (param) => {
1085
+ (0, import_utils6.assert)(
1086
+ isAndroidPage(this.page),
1087
+ "Cannot use recent apps button on non-Android devices"
1088
+ );
1089
+ await this.page.recentApps();
1090
+ }
1091
+ };
1092
+ tasks.push(taskActionAndroidRecentAppsButton);
1093
+ } else {
1094
+ throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
1095
+ }
1096
+ });
1097
+ const wrappedTasks = tasks.map(
1098
+ (task, index) => {
1099
+ if (task.type === "Action") {
1100
+ return this.prependExecutorWithScreenshot(
1101
+ task,
1102
+ index === tasks.length - 1
1103
+ );
1104
+ }
1105
+ return task;
1106
+ }
1107
+ );
1108
+ return {
1109
+ tasks: wrappedTasks
1110
+ };
1111
+ }
1112
+ async setupPlanningContext(executorContext) {
1113
+ const shotTime = Date.now();
1114
+ const pageContext = await this.insight.contextRetrieverFn("locate");
1115
+ const recordItem = {
1116
+ type: "screenshot",
1117
+ ts: shotTime,
1118
+ screenshot: pageContext.screenshotBase64,
1119
+ timing: "before planning"
1120
+ };
1121
+ executorContext.task.recorder = [recordItem];
1122
+ executorContext.task.pageContext = pageContext;
1123
+ return {
1124
+ pageContext
1125
+ };
1126
+ }
1127
+ async loadYamlFlowAsPlanning(userInstruction, yamlString) {
1128
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userInstruction), {
1129
+ onTaskStart: this.onTaskStartCallback
1130
+ });
1131
+ const task = {
1132
+ type: "Planning",
1133
+ subType: "LoadYaml",
1134
+ locate: null,
1135
+ param: {
1136
+ userInstruction
1137
+ },
1138
+ executor: async (param, executorContext) => {
1139
+ await this.setupPlanningContext(executorContext);
1140
+ return {
1141
+ output: {
1142
+ actions: [],
1143
+ more_actions_needed_by_instruction: false,
1144
+ log: "",
1145
+ yamlString
1146
+ },
1147
+ cache: {
1148
+ hit: true
1149
+ }
1150
+ };
1151
+ }
1152
+ };
1153
+ await taskExecutor.append(task);
1154
+ await taskExecutor.flush();
1155
+ return {
1156
+ executor: taskExecutor
1157
+ };
1158
+ }
1159
+ planningTaskFromPrompt(userInstruction, log, actionContext) {
1160
+ const task = {
1161
+ type: "Planning",
1162
+ subType: "Plan",
1163
+ locate: null,
1164
+ param: {
1165
+ userInstruction,
1166
+ log
1167
+ },
1168
+ executor: async (param, executorContext) => {
1169
+ const startTime = Date.now();
1170
+ const { pageContext } = await this.setupPlanningContext(executorContext);
1171
+ const planResult = await (0, import_misoai_core.plan)(param.userInstruction, {
1172
+ context: pageContext,
1173
+ log: param.log,
1174
+ actionContext,
1175
+ pageType: this.page.pageType
1176
+ });
1177
+ const {
1178
+ actions,
1179
+ log: log2,
1180
+ more_actions_needed_by_instruction,
1181
+ error,
1182
+ usage,
1183
+ rawResponse,
1184
+ sleep: sleep3
1185
+ } = planResult;
1186
+ executorContext.task.log = {
1187
+ ...executorContext.task.log || {},
1188
+ rawResponse
1189
+ };
1190
+ executorContext.task.usage = usage;
1191
+ let stopCollecting = false;
1192
+ let bboxCollected = false;
1193
+ let planParsingError = "";
1194
+ const finalActions = (actions || []).reduce(
1195
+ (acc, planningAction) => {
1196
+ if (stopCollecting) {
1197
+ return acc;
1198
+ }
1199
+ if (planningAction.locate) {
1200
+ if (bboxCollected && planningAction.locate.bbox) {
1201
+ delete planningAction.locate.bbox;
1202
+ }
1203
+ if (planningAction.locate.bbox) {
1204
+ bboxCollected = true;
1205
+ }
1206
+ acc.push({
1207
+ type: "Locate",
1208
+ locate: planningAction.locate,
1209
+ param: null,
1210
+ thought: planningAction.locate.prompt
1211
+ });
1212
+ } else if (["Tap", "Hover", "Input"].includes(planningAction.type)) {
1213
+ planParsingError = `invalid planning response: ${JSON.stringify(planningAction)}`;
1214
+ stopCollecting = true;
1215
+ return acc;
1216
+ }
1217
+ acc.push(planningAction);
1218
+ return acc;
1219
+ },
1220
+ []
1221
+ );
1222
+ if (sleep3) {
1223
+ const timeNow = Date.now();
1224
+ const timeRemaining = sleep3 - (timeNow - startTime);
1225
+ if (timeRemaining > 0) {
1226
+ finalActions.push({
1227
+ type: "Sleep",
1228
+ param: {
1229
+ timeMs: timeRemaining
1230
+ },
1231
+ locate: null
1232
+ });
1233
+ }
1234
+ }
1235
+ if (finalActions.length === 0) {
1236
+ (0, import_utils6.assert)(
1237
+ !more_actions_needed_by_instruction || sleep3,
1238
+ error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
1239
+ );
1240
+ }
1241
+ return {
1242
+ output: {
1243
+ actions: finalActions,
1244
+ more_actions_needed_by_instruction,
1245
+ log: log2,
1246
+ yamlFlow: planResult.yamlFlow
1247
+ },
1248
+ cache: {
1249
+ hit: false
1250
+ },
1251
+ pageContext
1252
+ };
1253
+ }
1254
+ };
1255
+ return task;
1256
+ }
1257
+ planningTaskToGoal(userInstruction) {
1258
+ const task = {
1259
+ type: "Planning",
1260
+ subType: "Plan",
1261
+ locate: null,
1262
+ param: {
1263
+ userInstruction
1264
+ },
1265
+ executor: async (param, executorContext) => {
1266
+ const { pageContext } = await this.setupPlanningContext(executorContext);
1267
+ const imagePayload = await (0, import_ai_model2.resizeImageForUiTars)(
1268
+ pageContext.screenshotBase64,
1269
+ pageContext.size
1270
+ );
1271
+ this.appendConversationHistory({
1272
+ role: "user",
1273
+ content: [
1274
+ {
1275
+ type: "image_url",
1276
+ image_url: {
1277
+ url: imagePayload
1278
+ }
1279
+ }
1280
+ ]
1281
+ });
1282
+ const startTime = Date.now();
1283
+ const planResult = await (0, import_ai_model2.vlmPlanning)({
1284
+ userInstruction: param.userInstruction,
1285
+ conversationHistory: this.conversationHistory,
1286
+ size: pageContext.size
1287
+ });
1288
+ const aiCost = Date.now() - startTime;
1289
+ const { actions, action_summary } = planResult;
1290
+ this.appendConversationHistory({
1291
+ role: "assistant",
1292
+ content: action_summary
1293
+ });
1294
+ return {
1295
+ output: {
1296
+ actions,
1297
+ thought: actions[0]?.thought,
1298
+ actionType: actions[0].type,
1299
+ more_actions_needed_by_instruction: true,
1300
+ log: "",
1301
+ yamlFlow: planResult.yamlFlow
1302
+ },
1303
+ cache: {
1304
+ hit: false
1305
+ },
1306
+ aiCost
1307
+ };
1308
+ }
1309
+ };
1310
+ return task;
1311
+ }
1312
+ async runPlans(title, plans, opts) {
1313
+ const taskExecutor = new import_misoai_core.Executor(title, {
1314
+ onTaskStart: this.onTaskStartCallback
1315
+ });
1316
+ const { tasks } = await this.convertPlanToExecutable(plans, opts);
1317
+ await taskExecutor.append(tasks);
1318
+ const result = await taskExecutor.flush();
1319
+ return {
1320
+ output: result,
1321
+ executor: taskExecutor
1322
+ };
1323
+ }
1324
+ async action(userPrompt, actionContext, opts) {
1325
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
1326
+ onTaskStart: this.onTaskStartCallback
1327
+ });
1328
+ let planningTask = this.planningTaskFromPrompt(userPrompt, void 0, actionContext);
1329
+ let replanCount = 0;
1330
+ const logList = [];
1331
+ const yamlFlow = [];
1332
+ while (planningTask) {
1333
+ if (replanCount > replanningCountLimit) {
1334
+ const errorMsg = "Replanning too many times, please split the task into multiple steps";
1335
+ return this.appendErrorPlan(taskExecutor, errorMsg);
1336
+ }
1337
+ await taskExecutor.append(planningTask);
1338
+ const planResult = await taskExecutor.flush();
1339
+ if (taskExecutor.isInErrorState()) {
1340
+ return {
1341
+ output: planResult,
1342
+ executor: taskExecutor
1343
+ };
1344
+ }
1345
+ const plans = planResult.actions || [];
1346
+ yamlFlow.push(...planResult.yamlFlow || []);
1347
+ let executables;
1348
+ try {
1349
+ executables = await this.convertPlanToExecutable(plans, opts);
1350
+ taskExecutor.append(executables.tasks);
1351
+ } catch (error) {
1352
+ return this.appendErrorPlan(
1353
+ taskExecutor,
1354
+ `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
1355
+ plans
1356
+ )}`
1357
+ );
1358
+ }
1359
+ await taskExecutor.flush();
1360
+ if (taskExecutor.isInErrorState()) {
1361
+ return {
1362
+ output: void 0,
1363
+ executor: taskExecutor
1364
+ };
1365
+ }
1366
+ if (planResult?.log) {
1367
+ logList.push(planResult.log);
1368
+ }
1369
+ if (!planResult.more_actions_needed_by_instruction) {
1370
+ planningTask = null;
1371
+ break;
1372
+ }
1373
+ planningTask = this.planningTaskFromPrompt(
1374
+ userPrompt,
1375
+ logList.length > 0 ? `- ${logList.join("\n- ")}` : void 0,
1376
+ actionContext
1377
+ );
1378
+ replanCount++;
1379
+ }
1380
+ return {
1381
+ output: {
1382
+ yamlFlow
1383
+ },
1384
+ executor: taskExecutor
1385
+ };
1386
+ }
1387
+ async actionToGoal(userPrompt, opts) {
1388
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
1389
+ onTaskStart: this.onTaskStartCallback
1390
+ });
1391
+ this.conversationHistory = [];
1392
+ const isCompleted = false;
1393
+ let currentActionNumber = 0;
1394
+ const maxActionNumber = 40;
1395
+ const yamlFlow = [];
1396
+ while (!isCompleted && currentActionNumber < maxActionNumber) {
1397
+ currentActionNumber++;
1398
+ const planningTask = this.planningTaskToGoal(userPrompt);
1399
+ await taskExecutor.append(planningTask);
1400
+ const output = await taskExecutor.flush();
1401
+ if (taskExecutor.isInErrorState()) {
1402
+ return {
1403
+ output: void 0,
1404
+ executor: taskExecutor
1405
+ };
1406
+ }
1407
+ const plans = output.actions;
1408
+ yamlFlow.push(...output.yamlFlow || []);
1409
+ let executables;
1410
+ try {
1411
+ executables = await this.convertPlanToExecutable(plans, opts);
1412
+ taskExecutor.append(executables.tasks);
1413
+ } catch (error) {
1414
+ return this.appendErrorPlan(
1415
+ taskExecutor,
1416
+ `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
1417
+ plans
1418
+ )}`
1419
+ );
1420
+ }
1421
+ await taskExecutor.flush();
1422
+ if (taskExecutor.isInErrorState()) {
1423
+ return {
1424
+ output: void 0,
1425
+ executor: taskExecutor
1426
+ };
1427
+ }
1428
+ if (plans[0].type === "Finished") {
1429
+ break;
1430
+ }
1431
+ }
1432
+ return {
1433
+ output: {
1434
+ yamlFlow
1435
+ },
1436
+ executor: taskExecutor
1437
+ };
1438
+ }
1439
+ async createTypeQueryTask(type, demand) {
1440
+ const taskExecutor = new import_misoai_core.Executor(
1441
+ taskTitleStr(
1442
+ type,
1443
+ typeof demand === "string" ? demand : JSON.stringify(demand)
1444
+ ),
1445
+ {
1446
+ onTaskStart: this.onTaskStartCallback
1447
+ }
1448
+ );
1449
+ const queryTask = {
1450
+ type: "Insight",
1451
+ subType: type,
1452
+ locate: null,
1453
+ param: {
1454
+ dataDemand: demand
1455
+ // for user param presentation in report right sidebar
1456
+ },
1457
+ executor: async (param) => {
1458
+ let insightDump;
1459
+ const dumpCollector = (dump) => {
1460
+ insightDump = dump;
1461
+ };
1462
+ this.insight.onceDumpUpdatedFn = dumpCollector;
1463
+ const ifTypeRestricted = type !== "Query";
1464
+ let demandInput = demand;
1465
+ if (ifTypeRestricted) {
1466
+ demandInput = {
1467
+ result: `${type}, ${demand}`
1468
+ };
1469
+ }
1470
+ const { data, usage } = await this.insight.extract(demandInput);
1471
+ let outputResult = data;
1472
+ if (ifTypeRestricted) {
1473
+ (0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
1474
+ outputResult = data.result;
1475
+ }
1476
+ return {
1477
+ output: outputResult,
1478
+ log: { dump: insightDump },
1479
+ usage
1480
+ };
1481
+ }
1482
+ };
1483
+ await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
1484
+ const output = await taskExecutor.flush();
1485
+ return {
1486
+ output,
1487
+ executor: taskExecutor
1488
+ };
1489
+ }
1490
+ async query(demand) {
1491
+ return this.createTypeQueryTask("Query", demand);
1492
+ }
1493
+ async boolean(prompt) {
1494
+ return this.createTypeQueryTask("Boolean", prompt);
1495
+ }
1496
+ async number(prompt) {
1497
+ return this.createTypeQueryTask("Number", prompt);
1498
+ }
1499
+ async string(prompt) {
1500
+ return this.createTypeQueryTask("String", prompt);
1501
+ }
1502
+ async assert(assertion) {
1503
+ const description = `assert: ${assertion}`;
1504
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
1505
+ onTaskStart: this.onTaskStartCallback
1506
+ });
1507
+ const assertionPlan = {
1508
+ type: "Assert",
1509
+ param: {
1510
+ assertion
1511
+ },
1512
+ locate: null
1513
+ };
1514
+ const { tasks } = await this.convertPlanToExecutable([assertionPlan]);
1515
+ await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
1516
+ const output = await taskExecutor.flush();
1517
+ return {
1518
+ output,
1519
+ executor: taskExecutor
1520
+ };
1521
+ }
1522
+ /**
1523
+ * Append a message to the conversation history
1524
+ * For user messages with images:
1525
+ * - Keep max 4 user image messages in history
1526
+ * - Remove oldest user image message when limit reached
1527
+ * For assistant messages:
1528
+ * - Simply append to history
1529
+ * @param conversationHistory Message to append
1530
+ */
1531
+ appendConversationHistory(conversationHistory) {
1532
+ if (conversationHistory.role === "user") {
1533
+ const userImgItems = this.conversationHistory.filter(
1534
+ (item) => item.role === "user"
1535
+ );
1536
+ if (userImgItems.length >= 4 && conversationHistory.role === "user") {
1537
+ const firstUserImgIndex = this.conversationHistory.findIndex(
1538
+ (item) => item.role === "user"
1539
+ );
1540
+ if (firstUserImgIndex >= 0) {
1541
+ this.conversationHistory.splice(firstUserImgIndex, 1);
1542
+ }
1543
+ }
1544
+ }
1545
+ this.conversationHistory.push(conversationHistory);
1546
+ }
1547
+ async appendErrorPlan(taskExecutor, errorMsg) {
1548
+ const errorPlan = {
1549
+ type: "Error",
1550
+ param: {
1551
+ thought: errorMsg
1552
+ },
1553
+ locate: null
1554
+ };
1555
+ const { tasks } = await this.convertPlanToExecutable([errorPlan]);
1556
+ await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
1557
+ await taskExecutor.flush();
1558
+ return {
1559
+ output: void 0,
1560
+ executor: taskExecutor
1561
+ };
1562
+ }
1563
+ async waitFor(assertion, opt) {
1564
+ const description = `waitFor: ${assertion}`;
1565
+ const taskExecutor = new import_misoai_core.Executor(taskTitleStr("WaitFor", description), {
1566
+ onTaskStart: this.onTaskStartCallback
1567
+ });
1568
+ const { timeoutMs, checkIntervalMs } = opt;
1569
+ (0, import_utils6.assert)(assertion, "No assertion for waitFor");
1570
+ (0, import_utils6.assert)(timeoutMs, "No timeoutMs for waitFor");
1571
+ (0, import_utils6.assert)(checkIntervalMs, "No checkIntervalMs for waitFor");
1572
+ const overallStartTime = Date.now();
1573
+ let startTime = Date.now();
1574
+ let errorThought = "";
1575
+ while (Date.now() - overallStartTime < timeoutMs) {
1576
+ startTime = Date.now();
1577
+ const assertPlan = {
1578
+ type: "AssertWithoutThrow",
1579
+ param: {
1580
+ assertion
1581
+ },
1582
+ locate: null
1583
+ };
1584
+ const { tasks: assertTasks } = await this.convertPlanToExecutable([
1585
+ assertPlan
1586
+ ]);
1587
+ await taskExecutor.append(
1588
+ this.prependExecutorWithScreenshot(assertTasks[0])
1589
+ );
1590
+ const output = await taskExecutor.flush();
1591
+ if (output?.pass) {
1592
+ return {
1593
+ output: void 0,
1594
+ executor: taskExecutor
1595
+ };
1596
+ }
1597
+ errorThought = output?.thought || `unknown error when waiting for assertion: ${assertion}`;
1598
+ const now = Date.now();
1599
+ if (now - startTime < checkIntervalMs) {
1600
+ const timeRemaining = checkIntervalMs - (now - startTime);
1601
+ const sleepPlan = {
1602
+ type: "Sleep",
1603
+ param: {
1604
+ timeMs: timeRemaining
1605
+ },
1606
+ locate: null
1607
+ };
1608
+ const { tasks: sleepTasks } = await this.convertPlanToExecutable([
1609
+ sleepPlan
1610
+ ]);
1611
+ await taskExecutor.append(
1612
+ this.prependExecutorWithScreenshot(sleepTasks[0])
1613
+ );
1614
+ await taskExecutor.flush();
1615
+ }
1616
+ }
1617
+ return this.appendErrorPlan(
1618
+ taskExecutor,
1619
+ `waitFor timeout: ${errorThought}`
1620
+ );
1621
+ }
1622
+ };
1623
+
1624
+ // src/common/plan-builder.ts
1625
+ var import_logger2 = require("misoai-shared/logger");
1626
+ var import_utils8 = require("misoai-shared/utils");
1627
+ var debug2 = (0, import_logger2.getDebug)("plan-builder");
1628
+ function buildPlans(type, locateParam, param) {
1629
+ let returnPlans = [];
1630
+ const locatePlan = locateParam ? {
1631
+ type: "Locate",
1632
+ locate: locateParam,
1633
+ param: locateParam,
1634
+ thought: ""
1635
+ } : null;
1636
+ if (type === "Tap" || type === "Hover") {
1637
+ (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
1638
+ (0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
1639
+ const tapPlan = {
1640
+ type,
1641
+ param: null,
1642
+ thought: "",
1643
+ locate: locateParam
1644
+ };
1645
+ returnPlans = [locatePlan, tapPlan];
1646
+ }
1647
+ if (type === "Input" || type === "KeyboardPress") {
1648
+ if (type === "Input") {
1649
+ (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
1650
+ }
1651
+ (0, import_utils8.assert)(param, `missing param for action "${type}"`);
1652
+ const inputPlan = {
1653
+ type,
1654
+ param,
1655
+ thought: "",
1656
+ locate: locateParam
1657
+ };
1658
+ if (locatePlan) {
1659
+ returnPlans = [locatePlan, inputPlan];
1660
+ } else {
1661
+ returnPlans = [inputPlan];
1662
+ }
1663
+ }
1664
+ if (type === "Scroll") {
1665
+ (0, import_utils8.assert)(param, `missing param for action "${type}"`);
1666
+ const scrollPlan = {
1667
+ type,
1668
+ param,
1669
+ thought: "",
1670
+ locate: locateParam
1671
+ };
1672
+ if (locatePlan) {
1673
+ returnPlans = [locatePlan, scrollPlan];
1674
+ } else {
1675
+ returnPlans = [scrollPlan];
1676
+ }
1677
+ }
1678
+ if (type === "Sleep") {
1679
+ (0, import_utils8.assert)(param, `missing param for action "${type}"`);
1680
+ const sleepPlan = {
1681
+ type,
1682
+ param,
1683
+ thought: "",
1684
+ locate: null
1685
+ };
1686
+ returnPlans = [sleepPlan];
1687
+ }
1688
+ if (type === "Locate") {
1689
+ (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
1690
+ const locatePlan2 = {
1691
+ type,
1692
+ param: locateParam,
1693
+ locate: locateParam,
1694
+ thought: ""
1695
+ };
1696
+ returnPlans = [locatePlan2];
1697
+ }
1698
+ if (returnPlans) {
1699
+ debug2("buildPlans", returnPlans);
1700
+ return returnPlans;
1701
+ }
1702
+ throw new Error(`Not supported type: ${type}`);
1703
+ }
1704
+
1705
+ // src/common/task-cache.ts
1706
+ var import_node_assert = __toESM(require("assert"));
1707
+ var import_node_fs2 = require("fs");
1708
+ var import_node_path2 = require("path");
1709
+ var import_common2 = require("misoai-shared/common");
1710
+ var import_logger3 = require("misoai-shared/logger");
1711
+ var import_utils9 = require("misoai-shared/utils");
1712
+ var import_js_yaml3 = __toESM(require("js-yaml"));
1713
+ var import_semver = __toESM(require("semver"));
1714
+
1715
+ // package.json
1716
+ var version = "1.0.4";
1717
+
1718
+ // src/common/task-cache.ts
1719
+ var debug3 = (0, import_logger3.getDebug)("cache");
1720
+ var lowestSupportedMidsceneVersion = "0.16.10";
1721
+ var cacheFileExt = ".cache.yaml";
1722
+ var TaskCache = class {
1723
+ // Track matched records
1724
+ constructor(cacheId, isCacheResultUsed, cacheFilePath) {
1725
+ this.matchedCacheIndices = /* @__PURE__ */ new Set();
1726
+ (0, import_node_assert.default)(cacheId, "cacheId is required");
1727
+ this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
1728
+ this.cacheFilePath = import_utils9.ifInBrowser ? void 0 : cacheFilePath || (0, import_node_path2.join)((0, import_common2.getMidsceneRunSubDir)("cache"), `${this.cacheId}${cacheFileExt}`);
1729
+ this.isCacheResultUsed = isCacheResultUsed;
1730
+ let cacheContent;
1731
+ if (this.cacheFilePath) {
1732
+ cacheContent = this.loadCacheFromFile();
1733
+ }
1734
+ if (!cacheContent) {
1735
+ cacheContent = {
1736
+ midsceneVersion: version,
1737
+ cacheId: this.cacheId,
1738
+ caches: []
1739
+ };
1740
+ }
1741
+ this.cache = cacheContent;
1742
+ this.cacheOriginalLength = this.cache.caches.length;
1743
+ }
1744
+ matchCache(prompt, type) {
1745
+ for (let i = 0; i < this.cacheOriginalLength; i++) {
1746
+ const item = this.cache.caches[i];
1747
+ const key = `${type}:${prompt}:${i}`;
1748
+ if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
1749
+ this.matchedCacheIndices.add(key);
1750
+ debug3(
1751
+ "cache found and marked as used, type: %s, prompt: %s, index: %d",
1752
+ type,
1753
+ prompt,
1754
+ i
1755
+ );
1756
+ return {
1757
+ cacheContent: item,
1758
+ updateFn: (cb) => {
1759
+ debug3(
1760
+ "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
1761
+ type,
1762
+ prompt,
1763
+ i
1764
+ );
1765
+ cb(item);
1766
+ debug3(
1767
+ "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
1768
+ type,
1769
+ prompt,
1770
+ i
1771
+ );
1772
+ this.flushCacheToFile();
1773
+ }
1774
+ };
1775
+ }
1776
+ }
1777
+ debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
1778
+ return void 0;
1779
+ }
1780
+ matchPlanCache(prompt) {
1781
+ return this.matchCache(prompt, "plan");
1782
+ }
1783
+ matchLocateCache(prompt) {
1784
+ return this.matchCache(prompt, "locate");
1785
+ }
1786
+ appendCache(cache) {
1787
+ debug3("will append cache", cache);
1788
+ this.cache.caches.push(cache);
1789
+ this.flushCacheToFile();
1790
+ }
1791
+ loadCacheFromFile() {
1792
+ const cacheFile = this.cacheFilePath;
1793
+ (0, import_node_assert.default)(cacheFile, "cache file path is required");
1794
+ if (!(0, import_node_fs2.existsSync)(cacheFile)) {
1795
+ debug3("no cache file found, path: %s", cacheFile);
1796
+ return void 0;
1797
+ }
1798
+ const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
1799
+ if ((0, import_node_fs2.existsSync)(jsonTypeCacheFile) && this.isCacheResultUsed) {
1800
+ console.warn(
1801
+ `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
1802
+ );
1803
+ return void 0;
1804
+ }
1805
+ try {
1806
+ const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
1807
+ const jsonData = import_js_yaml3.default.load(data);
1808
+ if (!version) {
1809
+ debug3("no midscene version info, will not read cache from file");
1810
+ return void 0;
1811
+ }
1812
+ if (import_semver.default.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
1813
+ console.warn(
1814
+ `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
1815
+ Please delete the existing cache and rebuild it. Sorry for the inconvenience.
1816
+ cache file: ${cacheFile}`
1817
+ );
1818
+ return void 0;
1819
+ }
1820
+ debug3(
1821
+ "cache loaded from file, path: %s, cache version: %s, record length: %s",
1822
+ cacheFile,
1823
+ jsonData.midsceneVersion,
1824
+ jsonData.caches.length
1825
+ );
1826
+ jsonData.midsceneVersion = version;
1827
+ return jsonData;
1828
+ } catch (err) {
1829
+ debug3(
1830
+ "cache file exists but load failed, path: %s, error: %s",
1831
+ cacheFile,
1832
+ err
1833
+ );
1834
+ return void 0;
1835
+ }
1836
+ }
1837
+ flushCacheToFile() {
1838
+ if (!version) {
1839
+ debug3("no midscene version info, will not write cache to file");
1840
+ return;
1841
+ }
1842
+ if (!this.cacheFilePath) {
1843
+ debug3("no cache file path, will not write cache to file");
1844
+ return;
1845
+ }
1846
+ try {
1847
+ const yamlData = import_js_yaml3.default.dump(this.cache);
1848
+ (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
1849
+ } catch (err) {
1850
+ debug3(
1851
+ "write cache to file failed, path: %s, error: %s",
1852
+ this.cacheFilePath,
1853
+ err
1854
+ );
1855
+ }
1856
+ }
1857
+ updateOrAppendCacheRecord(newRecord, cachedRecord) {
1858
+ if (cachedRecord) {
1859
+ if (newRecord.type === "plan") {
1860
+ cachedRecord.updateFn((cache) => {
1861
+ cache.yamlWorkflow = newRecord.yamlWorkflow;
1862
+ });
1863
+ } else {
1864
+ cachedRecord.updateFn((cache) => {
1865
+ cache.xpaths = newRecord.xpaths;
1866
+ });
1867
+ }
1868
+ } else {
1869
+ this.appendCache(newRecord);
1870
+ }
1871
+ }
1872
+ };
1873
+
1874
+ // src/common/agent.ts
1875
+ var debug4 = (0, import_logger4.getDebug)("web-integration");
1876
+ var distanceOfTwoPoints = (p1, p2) => {
1877
+ const [x1, y1] = p1;
1878
+ const [x2, y2] = p2;
1879
+ return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));
1880
+ };
1881
+ var includedInRect = (point, rect) => {
1882
+ const [x, y] = point;
1883
+ const { left, top, width, height } = rect;
1884
+ return x >= left && x <= left + width && y >= top && y <= top + height;
1885
+ };
1886
+ var PageAgent = class {
1887
+ constructor(page, opts) {
1888
+ /**
1889
+ * If true, the agent will not perform any actions
1890
+ */
1891
+ this.dryMode = false;
1892
+ this.page = page;
1893
+ this.opts = Object.assign(
1894
+ {
1895
+ generateReport: true,
1896
+ autoPrintReportMsg: true,
1897
+ groupName: "Midscene Report",
1898
+ groupDescription: ""
1899
+ },
1900
+ opts || {}
1901
+ );
1902
+ if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
1903
+ this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
1904
+ this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
1905
+ }
1906
+ this.onTaskStartTip = this.opts.onTaskStartTip;
1907
+ this.insight = new import_misoai_core2.Insight(
1908
+ async (action) => {
1909
+ return this.getUIContext(action);
1910
+ }
1911
+ );
1912
+ if (opts?.cacheId && this.page.pageType !== "android") {
1913
+ this.taskCache = new TaskCache(
1914
+ opts.cacheId,
1915
+ (0, import_env2.getAIConfigInBoolean)("MIDSCENE_CACHE")
1916
+ // if we should use cache to match the element
1917
+ );
1918
+ }
1919
+ this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {
1920
+ taskCache: this.taskCache,
1921
+ onTaskStart: this.callbackOnTaskStartTip.bind(this)
1922
+ });
1923
+ this.dump = this.resetDump();
1924
+ this.reportFileName = reportFileName(
1925
+ opts?.testId || this.page.pageType || "web"
1926
+ );
1927
+ }
1928
+ async getUIContext(action) {
1929
+ if (action && (action === "extract" || action === "assert" || action === "captcha")) {
1930
+ return await parseContextFromWebPage(this.page, {
1931
+ ignoreMarker: true
1932
+ });
1933
+ }
1934
+ return await parseContextFromWebPage(this.page, {
1935
+ ignoreMarker: !!(0, import_env2.vlLocateMode)()
1936
+ });
1937
+ }
1938
+ // Helper method to call the insight.captcha method
1939
+ async _callInsightCaptcha(options) {
1940
+ const context = await this.getUIContext();
1941
+ if (this.page.url) {
1942
+ const url = await this.page.url();
1943
+ context.url = url;
1944
+ }
1945
+ return this.insight.captcha(context, options);
1946
+ }
1947
+ async setAIActionContext(prompt) {
1948
+ this.opts.aiActionContext = prompt;
1949
+ }
1950
+ resetDump() {
1951
+ this.dump = {
1952
+ groupName: this.opts.groupName,
1953
+ groupDescription: this.opts.groupDescription,
1954
+ executions: []
1955
+ };
1956
+ return this.dump;
1957
+ }
1958
+ appendExecutionDump(execution) {
1959
+ const currentDump = this.dump;
1960
+ currentDump.executions.push(execution);
1961
+ }
1962
+ dumpDataString() {
1963
+ this.dump.groupName = this.opts.groupName;
1964
+ this.dump.groupDescription = this.opts.groupDescription;
1965
+ return (0, import_utils11.stringifyDumpData)(this.dump);
1966
+ }
1967
+ reportHTMLString() {
1968
+ return (0, import_utils11.reportHTMLContent)(this.dumpDataString());
1969
+ }
1970
+ writeOutActionDumps() {
1971
+ const { generateReport, autoPrintReportMsg } = this.opts;
1972
+ this.reportFile = (0, import_utils11.writeLogFile)({
1973
+ fileName: this.reportFileName,
1974
+ fileExt: import_utils11.groupedActionDumpFileExt,
1975
+ fileContent: this.dumpDataString(),
1976
+ type: "dump",
1977
+ generateReport
1978
+ });
1979
+ debug4("writeOutActionDumps", this.reportFile);
1980
+ if (generateReport && autoPrintReportMsg && this.reportFile) {
1981
+ printReportMsg(this.reportFile);
1982
+ }
1983
+ }
1984
+ async callbackOnTaskStartTip(task) {
1985
+ const param = paramStr(task);
1986
+ const tip = param ? `${typeStr(task)} - ${param}` : typeStr(task);
1987
+ if (this.onTaskStartTip) {
1988
+ await this.onTaskStartTip(tip);
1989
+ }
1990
+ }
1991
+ afterTaskRunning(executor, doNotThrowError = false) {
1992
+ this.appendExecutionDump(executor.dump());
1993
+ this.writeOutActionDumps();
1994
+ if (executor.isInErrorState() && !doNotThrowError) {
1995
+ const errorTask = executor.latestErrorTask();
1996
+ throw new Error(`${errorTask?.error}`);
1997
+ }
1998
+ const lastTask = executor.tasks[executor.tasks.length - 1];
1999
+ const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
2000
+ const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
2001
+ const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
2002
+ const planningTasks = executor.tasks.filter((task) => task.type === "Planning");
2003
+ const insightTasks = executor.tasks.filter((task) => task.type === "Insight");
2004
+ const actionTasks = executor.tasks.filter((task) => task.type === "Action");
2005
+ const planning = planningTasks.length > 0 ? {
2006
+ type: "Planning",
2007
+ description: `Planning for task execution`,
2008
+ steps: planningTasks.map((task) => task.thought || "Planning step")
2009
+ } : void 0;
2010
+ const insight = insightTasks.length > 0 ? {
2011
+ type: "Insight",
2012
+ description: `Insight for task execution`,
2013
+ elements: insightTasks.map((task) => task.thought || "Insight element")
2014
+ } : void 0;
2015
+ const action = actionTasks.length > 0 ? {
2016
+ type: "Action",
2017
+ description: `Action for task execution`,
2018
+ result: lastTask?.output
2019
+ } : void 0;
2020
+ const actionDetails = executor.tasks.map((task) => ({
2021
+ type: task.type,
2022
+ subType: task.subType,
2023
+ status: task.status,
2024
+ thought: task.thought
2025
+ }));
2026
+ const metadata = {
2027
+ status: lastTask?.status,
2028
+ start: lastTask?.timing?.start,
2029
+ end: lastTask?.timing?.end,
2030
+ totalTime: lastTask?.timing?.cost,
2031
+ cache: lastTask?.cache,
2032
+ usage: lastTask?.usage,
2033
+ thought: allThoughts.length > 0 ? allThoughts.join("\n") : lastTask?.thought,
2034
+ locate: allLocates.length > 0 ? allLocates : lastTask?.locate,
2035
+ plan: allPlans.length > 0 ? allPlans : lastTask?.param?.plans,
2036
+ // Add planning, insight, and action information
2037
+ planning,
2038
+ insight,
2039
+ action,
2040
+ actionDetails,
2041
+ // Include raw tasks for debugging
2042
+ tasks: executor.tasks.map((task) => ({
2043
+ type: task.type,
2044
+ subType: task.subType,
2045
+ status: task.status,
2046
+ thought: task.thought,
2047
+ locate: task.locate,
2048
+ timing: task.timing,
2049
+ usage: task.usage,
2050
+ cache: task.cache,
2051
+ error: task.error
2052
+ }))
2053
+ };
2054
+ return metadata;
2055
+ }
2056
+ buildDetailedLocateParam(locatePrompt, opt) {
2057
+ (0, import_utils12.assert)(locatePrompt, "missing locate prompt");
2058
+ if (typeof opt === "object") {
2059
+ const prompt = opt.prompt ?? locatePrompt;
2060
+ const deepThink = opt.deepThink ?? false;
2061
+ const cacheable = opt.cacheable ?? true;
2062
+ return {
2063
+ prompt,
2064
+ deepThink,
2065
+ cacheable
2066
+ };
2067
+ }
2068
+ return {
2069
+ prompt: locatePrompt
2070
+ };
2071
+ }
2072
+ async aiTap(locatePrompt, opt) {
2073
+ const detailedLocateParam = this.buildDetailedLocateParam(
2074
+ locatePrompt,
2075
+ opt
2076
+ );
2077
+ const plans = buildPlans("Tap", detailedLocateParam);
2078
+ const { executor, output } = await this.taskExecutor.runPlans(
2079
+ taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
2080
+ plans,
2081
+ { cacheable: opt?.cacheable }
2082
+ );
2083
+ const metadata = this.afterTaskRunning(executor);
2084
+ return {
2085
+ result: output,
2086
+ metadata
2087
+ };
2088
+ }
2089
+ async aiHover(locatePrompt, opt) {
2090
+ const detailedLocateParam = this.buildDetailedLocateParam(
2091
+ locatePrompt,
2092
+ opt
2093
+ );
2094
+ const plans = buildPlans("Hover", detailedLocateParam);
2095
+ const { executor, output } = await this.taskExecutor.runPlans(
2096
+ taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
2097
+ plans,
2098
+ { cacheable: opt?.cacheable }
2099
+ );
2100
+ const metadata = this.afterTaskRunning(executor);
2101
+ return {
2102
+ result: output,
2103
+ metadata
2104
+ };
2105
+ }
2106
+ async aiInput(value, locatePrompt, opt) {
2107
+ (0, import_utils12.assert)(
2108
+ typeof value === "string",
2109
+ "input value must be a string, use empty string if you want to clear the input"
2110
+ );
2111
+ (0, import_utils12.assert)(locatePrompt, "missing locate prompt for input");
2112
+ const detailedLocateParam = this.buildDetailedLocateParam(
2113
+ locatePrompt,
2114
+ opt
2115
+ );
2116
+ const plans = buildPlans("Input", detailedLocateParam, {
2117
+ value
2118
+ });
2119
+ const { executor, output } = await this.taskExecutor.runPlans(
2120
+ taskTitleStr("Input", locateParamStr(detailedLocateParam)),
2121
+ plans,
2122
+ { cacheable: opt?.cacheable }
2123
+ );
2124
+ const metadata = this.afterTaskRunning(executor);
2125
+ return {
2126
+ result: output,
2127
+ metadata
2128
+ };
2129
+ }
2130
+ async aiKeyboardPress(keyName, locatePrompt, opt) {
2131
+ (0, import_utils12.assert)(keyName, "missing keyName for keyboard press");
2132
+ const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
2133
+ const plans = buildPlans("KeyboardPress", detailedLocateParam, {
2134
+ value: keyName
2135
+ });
2136
+ const { executor, output } = await this.taskExecutor.runPlans(
2137
+ taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
2138
+ plans,
2139
+ { cacheable: opt?.cacheable }
2140
+ );
2141
+ const metadata = this.afterTaskRunning(executor);
2142
+ return {
2143
+ result: output,
2144
+ metadata
2145
+ };
2146
+ }
2147
+ async aiScroll(scrollParam, locatePrompt, opt) {
2148
+ const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
2149
+ const plans = buildPlans("Scroll", detailedLocateParam, scrollParam);
2150
+ const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
2151
+ const { executor, output } = await this.taskExecutor.runPlans(
2152
+ taskTitleStr("Scroll", paramInTitle),
2153
+ plans,
2154
+ { cacheable: opt?.cacheable }
2155
+ );
2156
+ const metadata = this.afterTaskRunning(executor);
2157
+ return {
2158
+ result: output,
2159
+ metadata
2160
+ };
2161
+ }
2162
+ async aiAction(taskPrompt, opt) {
2163
+ try {
2164
+ const aiModel = await import("misoai-core/ai-model");
2165
+ const contextStore = aiModel.getContextStore();
2166
+ const processedPrompt = contextStore.replaceAllReferences(taskPrompt, "action");
2167
+ contextStore.addStep({
2168
+ type: "action",
2169
+ summary: `Action: ${processedPrompt}`,
2170
+ prompt: processedPrompt
2171
+ });
2172
+ taskPrompt = processedPrompt;
2173
+ } catch (error) {
2174
+ debug4("Context store not available:", error);
2175
+ }
2176
+ const cacheable = opt?.cacheable;
2177
+ const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
2178
+ const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
2179
+ if (matchedCache && this.taskCache?.isCacheResultUsed) {
2180
+ const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
2181
+ taskPrompt,
2182
+ matchedCache.cacheContent?.yamlWorkflow
2183
+ );
2184
+ const metadata2 = this.afterTaskRunning(executor2);
2185
+ debug4("matched cache, will call .runYaml to run the action");
2186
+ const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
2187
+ const result = await this.runYaml(yaml5);
2188
+ return {
2189
+ result: result.result,
2190
+ metadata: metadata2
2191
+ };
2192
+ }
2193
+ const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
2194
+ cacheable
2195
+ }));
2196
+ if (this.taskCache && output?.yamlFlow && cacheable !== false) {
2197
+ const yamlContent = {
2198
+ tasks: [
2199
+ {
2200
+ name: taskPrompt,
2201
+ flow: output.yamlFlow
2202
+ }
2203
+ ]
2204
+ };
2205
+ const yamlFlowStr = import_js_yaml4.default.dump(yamlContent);
2206
+ this.taskCache.updateOrAppendCacheRecord(
2207
+ {
2208
+ type: "plan",
2209
+ prompt: taskPrompt,
2210
+ yamlWorkflow: yamlFlowStr
2211
+ },
2212
+ matchedCache
2213
+ );
2214
+ }
2215
+ const metadata = this.afterTaskRunning(executor);
2216
+ return {
2217
+ result: output,
2218
+ metadata
2219
+ };
2220
+ }
2221
+ async aiQuery(demand) {
2222
+ let processedDemand = demand;
2223
+ let storageKey;
2224
+ try {
2225
+ const aiModel = await import("misoai-core/ai-model");
2226
+ const contextStore = aiModel.getContextStore();
2227
+ if (typeof demand === "string") {
2228
+ const storageInstruction = contextStore.parseStorageInstruction(demand);
2229
+ if (storageInstruction) {
2230
+ storageKey = storageInstruction.key;
2231
+ processedDemand = storageInstruction.cleanText;
2232
+ contextStore._pendingAliases = storageInstruction.aliases;
2233
+ } else {
2234
+ const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
2235
+ if (storageMatch) {
2236
+ storageKey = storageMatch[1];
2237
+ processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
2238
+ }
2239
+ }
2240
+ }
2241
+ } catch (error) {
2242
+ debug4("Context store not available:", error);
2243
+ }
2244
+ const { output, executor } = await this.taskExecutor.query(processedDemand);
2245
+ if (storageKey && output) {
2246
+ try {
2247
+ const aiModel = await import("misoai-core/ai-model");
2248
+ const contextStore = aiModel.getContextStore();
2249
+ const pendingAliases = contextStore._pendingAliases;
2250
+ if (pendingAliases) {
2251
+ contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
2252
+ delete contextStore._pendingAliases;
2253
+ } else {
2254
+ contextStore.storeData(storageKey, output);
2255
+ }
2256
+ contextStore.addStep({
2257
+ type: "query",
2258
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
2259
+ data: output,
2260
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2261
+ });
2262
+ } catch (error) {
2263
+ debug4("Failed to store query result:", error);
2264
+ }
2265
+ } else {
2266
+ try {
2267
+ const aiModel = await import("misoai-core/ai-model");
2268
+ const contextStore = aiModel.getContextStore();
2269
+ contextStore.addStep({
2270
+ type: "query",
2271
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
2272
+ data: output,
2273
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2274
+ });
2275
+ } catch (error) {
2276
+ debug4("Failed to add query step:", error);
2277
+ }
2278
+ }
2279
+ const metadata = this.afterTaskRunning(executor);
2280
+ return {
2281
+ result: output,
2282
+ metadata
2283
+ };
2284
+ }
2285
+ async aiBoolean(prompt) {
2286
+ const { output, executor } = await this.taskExecutor.boolean(prompt);
2287
+ const metadata = this.afterTaskRunning(executor);
2288
+ return {
2289
+ result: output,
2290
+ metadata
2291
+ };
2292
+ }
2293
+ async aiNumber(prompt) {
2294
+ const { output, executor } = await this.taskExecutor.number(prompt);
2295
+ const metadata = this.afterTaskRunning(executor);
2296
+ return {
2297
+ result: output,
2298
+ metadata
2299
+ };
2300
+ }
2301
+ async aiString(prompt) {
2302
+ const { output, executor } = await this.taskExecutor.string(prompt);
2303
+ const metadata = this.afterTaskRunning(executor);
2304
+ return {
2305
+ result: output,
2306
+ metadata
2307
+ };
2308
+ }
2309
+ async describeElementAtPoint(center, opt) {
2310
+ const { verifyPrompt = true, retryLimit = 3 } = opt || {};
2311
+ let success = false;
2312
+ let retryCount = 0;
2313
+ let resultPrompt = "";
2314
+ let deepThink = opt?.deepThink || false;
2315
+ let verifyResult;
2316
+ while (!success && retryCount < retryLimit) {
2317
+ if (retryCount >= 2) {
2318
+ deepThink = true;
2319
+ }
2320
+ debug4(
2321
+ "aiDescribe",
2322
+ center,
2323
+ "verifyPrompt",
2324
+ verifyPrompt,
2325
+ "retryCount",
2326
+ retryCount,
2327
+ "deepThink",
2328
+ deepThink
2329
+ );
2330
+ const text = await this.insight.describe(center, { deepThink });
2331
+ debug4("aiDescribe text", text);
2332
+ (0, import_utils12.assert)(text.description, `failed to describe element at [${center}]`);
2333
+ resultPrompt = text.description;
2334
+ verifyResult = await this.verifyLocator(
2335
+ resultPrompt,
2336
+ deepThink ? { deepThink: true } : void 0,
2337
+ center,
2338
+ opt
2339
+ );
2340
+ if (verifyResult.pass) {
2341
+ success = true;
2342
+ } else {
2343
+ retryCount++;
2344
+ }
2345
+ }
2346
+ return {
2347
+ prompt: resultPrompt,
2348
+ deepThink,
2349
+ verifyResult
2350
+ };
2351
+ }
2352
+ async verifyLocator(prompt, locateOpt, expectCenter, verifyLocateOption) {
2353
+ debug4("verifyLocator", prompt, locateOpt, expectCenter, verifyLocateOption);
2354
+ const locateResult = await this.aiLocate(prompt, locateOpt);
2355
+ const { center: verifyCenter, rect: verifyRect } = locateResult.result;
2356
+ const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
2357
+ const included = includedInRect(expectCenter, verifyRect);
2358
+ const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;
2359
+ const verifyResult = {
2360
+ pass,
2361
+ rect: verifyRect,
2362
+ center: verifyCenter,
2363
+ centerDistance: distance
2364
+ };
2365
+ debug4("aiDescribe verifyResult", verifyResult);
2366
+ return verifyResult;
2367
+ }
2368
+ async aiLocate(prompt, opt) {
2369
+ const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
2370
+ const plans = buildPlans("Locate", detailedLocateParam);
2371
+ const { executor, output } = await this.taskExecutor.runPlans(
2372
+ taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
2373
+ plans,
2374
+ { cacheable: opt?.cacheable }
2375
+ );
2376
+ const metadata = this.afterTaskRunning(executor);
2377
+ const { element } = output;
2378
+ const result = {
2379
+ rect: element?.rect,
2380
+ center: element?.center
2381
+ };
2382
+ return {
2383
+ result,
2384
+ metadata
2385
+ };
2386
+ }
2387
+ async aiAssert(assertion, msg, opt) {
2388
+ let processedAssertion = assertion;
2389
+ try {
2390
+ const aiModel = await import("misoai-core/ai-model");
2391
+ const contextStore = aiModel.getContextStore();
2392
+ processedAssertion = contextStore.replaceAllReferences(assertion, "assertion");
2393
+ contextStore.addStep({
2394
+ type: "assertion",
2395
+ summary: `Assertion: ${processedAssertion}`,
2396
+ prompt: processedAssertion
2397
+ });
2398
+ } catch (error) {
2399
+ debug4("Context store not available:", error);
2400
+ }
2401
+ let currentUrl = "";
2402
+ if (this.page.url) {
2403
+ try {
2404
+ currentUrl = await this.page.url();
2405
+ } catch (e) {
2406
+ }
2407
+ }
2408
+ const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
2409
+ const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
2410
+ const metadata = this.afterTaskRunning(executor, true);
2411
+ if (output && opt?.keepRawResponse) {
2412
+ return {
2413
+ result: output,
2414
+ metadata
2415
+ };
2416
+ }
2417
+ if (!output?.pass) {
2418
+ const errMsg = msg || `Assertion failed: ${assertion}`;
2419
+ const reasonMsg = `Reason: ${output?.thought || executor.latestErrorTask()?.error || "(no_reason)"}`;
2420
+ throw new Error(`${errMsg}
2421
+ ${reasonMsg}`);
2422
+ }
2423
+ return {
2424
+ result: true,
2425
+ metadata
2426
+ };
2427
+ }
2428
+ async aiCaptcha(options) {
2429
+ const { deepThink = false, autoDetectComplexity = true } = options || {};
2430
+ let shouldUseDeepThink = deepThink;
2431
+ if (autoDetectComplexity && !deepThink) {
2432
+ const context = await this.getUIContext();
2433
+ const { screenshotBase64 } = context;
2434
+ try {
2435
+ const complexityAnalysisPrompt = `
2436
+ Analyze this screenshot and determine if it contains a complex CAPTCHA that would benefit from deep thinking.
2437
+ A complex CAPTCHA typically has one or more of these characteristics:
2438
+ - Distorted or overlapping text that is hard to read
2439
+ - Multiple images that need to be selected based on a specific criteria
2440
+ - Puzzles that require spatial reasoning
2441
+ - Multiple steps or verification methods
2442
+ - Small or hard-to-distinguish elements
2443
+
2444
+ Return only "complex" or "simple" based on your analysis.
2445
+ `;
2446
+ const complexityMsgs = [
2447
+ { role: "system", content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity." },
2448
+ {
2449
+ role: "user",
2450
+ content: [
2451
+ {
2452
+ type: "image_url",
2453
+ image_url: {
2454
+ url: screenshotBase64,
2455
+ detail: "high"
2456
+ }
2457
+ },
2458
+ {
2459
+ type: "text",
2460
+ text: complexityAnalysisPrompt
2461
+ }
2462
+ ]
2463
+ }
2464
+ ];
2465
+ const complexityResult = await this.insight.aiVendorFn(
2466
+ complexityMsgs,
2467
+ { type: "extract_data" }
2468
+ );
2469
+ const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
2470
+ shouldUseDeepThink = responseText.includes("complex");
2471
+ debug4("CAPTCHA complexity analysis:", responseText, "Using deep think:", shouldUseDeepThink);
2472
+ } catch (error) {
2473
+ debug4("Failed to analyze CAPTCHA complexity:", error);
2474
+ }
2475
+ }
2476
+ const captchaResponse = await this._callInsightCaptcha({
2477
+ deepThink: shouldUseDeepThink
2478
+ });
2479
+ const captchaResult = captchaResponse.content;
2480
+ const usage = captchaResponse.usage;
2481
+ const actualDeepThink = captchaResponse.deepThink || false;
2482
+ if (captchaResult.captchaType === "text") {
2483
+ for (const action of captchaResult.actions) {
2484
+ if (action.type === "click" && action.target) {
2485
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2486
+ } else if (action.type === "input" && action.value) {
2487
+ if (action.target) {
2488
+ await this.aiInput(action.value, action.target, { deepThink: shouldUseDeepThink });
2489
+ }
2490
+ } else if (action.type === "verify" && action.target) {
2491
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2492
+ }
2493
+ }
2494
+ } else if (captchaResult.captchaType === "image") {
2495
+ for (const action of captchaResult.actions) {
2496
+ if (action.type === "click") {
2497
+ if (action.coordinates) {
2498
+ const x = action.coordinates[0];
2499
+ const y = action.coordinates[1];
2500
+ await this.aiTap(`element at coordinates (${x}, ${y})`, { deepThink: shouldUseDeepThink });
2501
+ } else if (action.target) {
2502
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2503
+ }
2504
+ } else if (action.type === "verify" && action.target) {
2505
+ await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
2506
+ }
2507
+ }
2508
+ }
2509
+ await new Promise((resolve2) => setTimeout(resolve2, 3e3));
2510
+ const metadata = {
2511
+ status: "finished",
2512
+ usage,
2513
+ thought: captchaResult.thought
2514
+ };
2515
+ metadata.deepThink = actualDeepThink;
2516
+ if (autoDetectComplexity && !deepThink) {
2517
+ metadata.autoDetectedComplexity = shouldUseDeepThink;
2518
+ }
2519
+ return {
2520
+ result: captchaResult,
2521
+ metadata
2522
+ };
2523
+ }
2524
+ async aiWaitFor(assertion, opt) {
2525
+ const startTime = Date.now();
2526
+ const { executor } = await this.taskExecutor.waitFor(assertion, {
2527
+ timeoutMs: opt?.timeoutMs || 15 * 1e3,
2528
+ checkIntervalMs: opt?.checkIntervalMs || 3 * 1e3,
2529
+ assertion
2530
+ });
2531
+ const metadata = {
2532
+ status: executor.isInErrorState() ? "failed" : "finished",
2533
+ start: startTime,
2534
+ end: Date.now(),
2535
+ totalTime: Date.now() - startTime,
2536
+ thought: executor.latestErrorTask()?.thought,
2537
+ actionDetails: executor.tasks.map((task) => ({
2538
+ type: task.type,
2539
+ subType: task.subType,
2540
+ status: task.status,
2541
+ thought: task.thought
2542
+ }))
2543
+ };
2544
+ this.appendExecutionDump(executor.dump());
2545
+ this.writeOutActionDumps();
2546
+ if (executor.isInErrorState()) {
2547
+ const errorTask = executor.latestErrorTask();
2548
+ throw new Error(`${errorTask?.error}
2549
+ ${errorTask?.errorStack}`);
2550
+ }
2551
+ return {
2552
+ result: true,
2553
+ // Successfully waited
2554
+ metadata
2555
+ };
2556
+ }
2557
+ async ai(taskPrompt, type = "action", options) {
2558
+ if (type === "action") {
2559
+ return this.aiAction(taskPrompt);
2560
+ }
2561
+ if (type === "query") {
2562
+ return this.aiQuery(taskPrompt);
2563
+ }
2564
+ if (type === "assert") {
2565
+ return this.aiAssert(taskPrompt);
2566
+ }
2567
+ if (type === "tap") {
2568
+ return this.aiTap(taskPrompt, options);
2569
+ }
2570
+ if (type === "captcha") {
2571
+ return this.aiCaptcha(options);
2572
+ }
2573
+ throw new Error(
2574
+ `Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'captcha'`
2575
+ );
2576
+ }
2577
+ async runYaml(yamlScriptContent) {
2578
+ const startTime = Date.now();
2579
+ const script = parseYamlScript(yamlScriptContent, "yaml", true);
2580
+ const player = new ScriptPlayer(script, async () => {
2581
+ return { agent: this, freeFn: [] };
2582
+ });
2583
+ await player.run();
2584
+ const endTime = Date.now();
2585
+ const metadata = {
2586
+ status: player.status,
2587
+ start: startTime,
2588
+ end: endTime,
2589
+ totalTime: endTime - startTime,
2590
+ tasks: player.taskStatusList.map((task) => ({
2591
+ type: "yaml-task",
2592
+ subType: task.name,
2593
+ status: task.status,
2594
+ error: task.error?.message
2595
+ }))
2596
+ };
2597
+ if (player.status === "error") {
2598
+ const errors = player.taskStatusList.filter((task) => task.status === "error").map((task) => {
2599
+ return `task - ${task.name}: ${task.error?.message}`;
2600
+ }).join("\n");
2601
+ throw new Error(`Error(s) occurred in running yaml script:
2602
+ ${errors}`);
2603
+ }
2604
+ return {
2605
+ result: player.result,
2606
+ metadata
2607
+ };
2608
+ }
2609
+ async evaluateJavaScript(script) {
2610
+ (0, import_utils12.assert)(
2611
+ this.page.evaluateJavaScript,
2612
+ "evaluateJavaScript is not supported in current agent"
2613
+ );
2614
+ if (this.page.evaluateJavaScript) {
2615
+ return this.page.evaluateJavaScript(script);
2616
+ }
2617
+ throw new Error("evaluateJavaScript is not supported in current agent");
2618
+ }
2619
+ async destroy() {
2620
+ await this.page.destroy();
2621
+ }
2622
+ };
2623
+
2624
+ // src/puppeteer/index.ts
2625
+ var import_logger6 = require("misoai-shared/logger");
2626
+
2627
+ // src/puppeteer/page.ts
2628
+ var import_constants4 = require("misoai-shared/constants");
2629
+
2630
+ // src/puppeteer/base-page.ts
2631
+ var import_utils15 = require("misoai-core/utils");
2632
+ var import_constants3 = require("misoai-shared/constants");
2633
+ var import_extractor2 = require("misoai-shared/extractor");
2634
+ var import_fs = require("misoai-shared/fs");
2635
+ var import_logger5 = require("misoai-shared/logger");
2636
+ var import_utils16 = require("misoai-shared/utils");
2637
+ var debugPage = (0, import_logger5.getDebug)("web:page");
2638
+ var Page = class {
2639
+ constructor(underlyingPage, pageType, opts) {
2640
+ this.everMoved = false;
2641
+ this.underlyingPage = underlyingPage;
2642
+ this.pageType = pageType;
2643
+ this.waitForNavigationTimeout = opts?.waitForNavigationTimeout || import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
2644
+ }
2645
+ async evaluate(pageFunction, arg) {
2646
+ let result;
2647
+ debugPage("evaluate function begin");
2648
+ if (this.pageType === "puppeteer") {
2649
+ result = await this.underlyingPage.evaluate(
2650
+ pageFunction,
2651
+ arg
2652
+ );
2653
+ } else {
2654
+ result = await this.underlyingPage.evaluate(
2655
+ pageFunction,
2656
+ arg
2657
+ );
2658
+ }
2659
+ debugPage("evaluate function end");
2660
+ return result;
2661
+ }
2662
+ async evaluateJavaScript(script) {
2663
+ return this.evaluate(script);
2664
+ }
2665
+ async waitForNavigation() {
2666
+ if (this.pageType === "puppeteer" || this.pageType === "playwright") {
2667
+ debugPage("waitForNavigation begin");
2668
+ debugPage(`waitForNavigation timeout: ${this.waitForNavigationTimeout}`);
2669
+ try {
2670
+ await this.underlyingPage.waitForSelector("html", {
2671
+ timeout: this.waitForNavigationTimeout
2672
+ });
2673
+ } catch (error) {
2674
+ console.warn(
2675
+ "[midscene:warning] Waiting for the navigation has timed out, but Midscene will continue execution. Please check https://midscenejs.com/faq.html#customize-the-network-timeout for more information on customizing the network timeout"
2676
+ );
2677
+ }
2678
+ debugPage("waitForNavigation end");
2679
+ }
2680
+ }
2681
+ // @deprecated
2682
+ async getElementsInfo() {
2683
+ await this.waitForNavigation();
2684
+ debugPage("getElementsInfo begin");
2685
+ const tree = await this.getElementsNodeTree();
2686
+ debugPage("getElementsInfo end");
2687
+ return (0, import_extractor2.treeToList)(tree);
2688
+ }
2689
+ async getXpathsById(id) {
2690
+ const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
2691
+ return this.evaluateJavaScript(
2692
+ `${elementInfosScriptContent}midscene_element_inspector.getXpathsById('${id}')`
2693
+ );
2694
+ }
2695
+ async getElementInfoByXpath(xpath) {
2696
+ const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
2697
+ return this.evaluateJavaScript(
2698
+ `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByXpath('${xpath}')`
2699
+ );
2700
+ }
2701
+ async getElementsNodeTree() {
2702
+ await this.waitForNavigation();
2703
+ const scripts = await (0, import_fs.getExtraReturnLogic)(true);
2704
+ (0, import_utils16.assert)(scripts, "scripts should be set before writing report in browser");
2705
+ const captureElementSnapshot = await this.evaluate(scripts);
2706
+ return captureElementSnapshot;
2707
+ }
2708
+ async size() {
2709
+ if (this.viewportSize)
2710
+ return this.viewportSize;
2711
+ const sizeInfo = await this.evaluate(() => {
2712
+ return {
2713
+ width: document.documentElement.clientWidth,
2714
+ height: document.documentElement.clientHeight,
2715
+ dpr: window.devicePixelRatio
2716
+ };
2717
+ });
2718
+ this.viewportSize = sizeInfo;
2719
+ return sizeInfo;
2720
+ }
2721
+ async screenshotBase64() {
2722
+ const imgType = "jpeg";
2723
+ const quality = 90;
2724
+ await this.waitForNavigation();
2725
+ debugPage("screenshotBase64 begin");
2726
+ let base64;
2727
+ if (this.pageType === "puppeteer") {
2728
+ const result = await this.underlyingPage.screenshot({
2729
+ type: imgType,
2730
+ quality,
2731
+ encoding: "base64"
2732
+ });
2733
+ base64 = `data:image/jpeg;base64,${result}`;
2734
+ } else if (this.pageType === "playwright") {
2735
+ const buffer = await this.underlyingPage.screenshot({
2736
+ type: imgType,
2737
+ quality,
2738
+ timeout: 10 * 1e3
2739
+ });
2740
+ base64 = `data:image/jpeg;base64,${buffer.toString("base64")}`;
2741
+ } else {
2742
+ throw new Error("Unsupported page type for screenshot");
2743
+ }
2744
+ debugPage("screenshotBase64 end");
2745
+ return base64;
2746
+ }
2747
+ async url() {
2748
+ return this.underlyingPage.url();
2749
+ }
2750
+ get mouse() {
2751
+ return {
2752
+ click: async (x, y, options) => {
2753
+ await this.mouse.move(x, y);
2754
+ this.underlyingPage.mouse.click(x, y, {
2755
+ button: options?.button || "left",
2756
+ count: options?.count || 1
2757
+ });
2758
+ },
2759
+ wheel: async (deltaX, deltaY) => {
2760
+ if (this.pageType === "puppeteer") {
2761
+ await this.underlyingPage.mouse.wheel({
2762
+ deltaX,
2763
+ deltaY
2764
+ });
2765
+ } else if (this.pageType === "playwright") {
2766
+ await this.underlyingPage.mouse.wheel(
2767
+ deltaX,
2768
+ deltaY
2769
+ );
2770
+ }
2771
+ },
2772
+ move: async (x, y) => {
2773
+ this.everMoved = true;
2774
+ return this.underlyingPage.mouse.move(x, y);
2775
+ },
2776
+ drag: async (from, to) => {
2777
+ if (this.pageType === "puppeteer") {
2778
+ await this.underlyingPage.mouse.drag(
2779
+ {
2780
+ x: from.x,
2781
+ y: from.y
2782
+ },
2783
+ {
2784
+ x: to.x,
2785
+ y: to.y
2786
+ }
2787
+ );
2788
+ } else if (this.pageType === "playwright") {
2789
+ await this.underlyingPage.mouse.move(
2790
+ from.x,
2791
+ from.y
2792
+ );
2793
+ await this.underlyingPage.mouse.down();
2794
+ await this.underlyingPage.mouse.move(to.x, to.y);
2795
+ await this.underlyingPage.mouse.up();
2796
+ }
2797
+ }
2798
+ };
2799
+ }
2800
+ get keyboard() {
2801
+ return {
2802
+ type: async (text) => this.underlyingPage.keyboard.type(text, { delay: 80 }),
2803
+ press: async (action) => {
2804
+ const keys = Array.isArray(action) ? action : [action];
2805
+ for (const k of keys) {
2806
+ const commands = k.command ? [k.command] : [];
2807
+ await this.underlyingPage.keyboard.down(k.key, { commands });
2808
+ }
2809
+ for (const k of [...keys].reverse()) {
2810
+ await this.underlyingPage.keyboard.up(k.key);
2811
+ }
2812
+ },
2813
+ down: async (key) => {
2814
+ this.underlyingPage.keyboard.down(key);
2815
+ },
2816
+ up: async (key) => {
2817
+ this.underlyingPage.keyboard.up(key);
2818
+ }
2819
+ };
2820
+ }
2821
+ async clearInput(element) {
2822
+ if (!element) {
2823
+ console.warn("No element to clear input");
2824
+ return;
2825
+ }
2826
+ const isMac = process.platform === "darwin";
2827
+ if (isMac) {
2828
+ if (this.pageType === "puppeteer") {
2829
+ await this.mouse.click(element.center[0], element.center[1], {
2830
+ count: 3
2831
+ });
2832
+ } else {
2833
+ await this.mouse.click(element.center[0], element.center[1]);
2834
+ await this.underlyingPage.keyboard.down("Meta");
2835
+ await this.underlyingPage.keyboard.press("a");
2836
+ await this.underlyingPage.keyboard.up("Meta");
2837
+ }
2838
+ } else {
2839
+ await this.mouse.click(element.center[0], element.center[1]);
2840
+ await this.underlyingPage.keyboard.down("Control");
2841
+ await this.underlyingPage.keyboard.press("a");
2842
+ await this.underlyingPage.keyboard.up("Control");
2843
+ }
2844
+ await (0, import_utils15.sleep)(100);
2845
+ await this.keyboard.press([{ key: "Backspace" }]);
2846
+ }
2847
+ async moveToPointBeforeScroll(point) {
2848
+ if (point) {
2849
+ await this.mouse.move(point.left, point.top);
2850
+ } else if (!this.everMoved) {
2851
+ const size = await this.size();
2852
+ const targetX = Math.floor(size.width / 2);
2853
+ const targetY = Math.floor(size.height / 2);
2854
+ await this.mouse.move(targetX, targetY);
2855
+ }
2856
+ }
2857
+ async scrollUntilTop(startingPoint) {
2858
+ await this.moveToPointBeforeScroll(startingPoint);
2859
+ return this.mouse.wheel(0, -9999999);
2860
+ }
2861
+ async scrollUntilBottom(startingPoint) {
2862
+ await this.moveToPointBeforeScroll(startingPoint);
2863
+ return this.mouse.wheel(0, 9999999);
2864
+ }
2865
+ async scrollUntilLeft(startingPoint) {
2866
+ await this.moveToPointBeforeScroll(startingPoint);
2867
+ return this.mouse.wheel(-9999999, 0);
2868
+ }
2869
+ async scrollUntilRight(startingPoint) {
2870
+ await this.moveToPointBeforeScroll(startingPoint);
2871
+ return this.mouse.wheel(9999999, 0);
2872
+ }
2873
+ async scrollUp(distance, startingPoint) {
2874
+ const innerHeight = await this.evaluate(() => window.innerHeight);
2875
+ const scrollDistance = distance || innerHeight * 0.7;
2876
+ await this.moveToPointBeforeScroll(startingPoint);
2877
+ return this.mouse.wheel(0, -scrollDistance);
2878
+ }
2879
+ async scrollDown(distance, startingPoint) {
2880
+ const innerHeight = await this.evaluate(() => window.innerHeight);
2881
+ const scrollDistance = distance || innerHeight * 0.7;
2882
+ await this.moveToPointBeforeScroll(startingPoint);
2883
+ return this.mouse.wheel(0, scrollDistance);
2884
+ }
2885
+ async scrollLeft(distance, startingPoint) {
2886
+ const innerWidth = await this.evaluate(() => window.innerWidth);
2887
+ const scrollDistance = distance || innerWidth * 0.7;
2888
+ await this.moveToPointBeforeScroll(startingPoint);
2889
+ return this.mouse.wheel(-scrollDistance, 0);
2890
+ }
2891
+ async scrollRight(distance, startingPoint) {
2892
+ const innerWidth = await this.evaluate(() => window.innerWidth);
2893
+ const scrollDistance = distance || innerWidth * 0.7;
2894
+ await this.moveToPointBeforeScroll(startingPoint);
2895
+ return this.mouse.wheel(scrollDistance, 0);
2896
+ }
2897
+ async navigate(url) {
2898
+ if (this.pageType === "puppeteer") {
2899
+ await this.underlyingPage.goto(url);
2900
+ } else if (this.pageType === "playwright") {
2901
+ await this.underlyingPage.goto(url);
2902
+ } else {
2903
+ throw new Error("Unsupported page type for navigate");
2904
+ }
2905
+ }
2906
+ async destroy() {
2907
+ }
2908
+ };
2909
+
2910
+ // src/puppeteer/page.ts
2911
+ var WebPage = class extends Page {
2912
+ constructor(page, opts) {
2913
+ super(page, "puppeteer");
2914
+ const {
2915
+ waitForNavigationTimeout = import_constants4.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
2916
+ waitForNetworkIdleTimeout = import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT
2917
+ } = opts ?? {};
2918
+ this.waitForNavigationTimeout = waitForNavigationTimeout;
2919
+ this.waitForNetworkIdleTimeout = waitForNetworkIdleTimeout;
2920
+ }
2921
+ async waitUntilNetworkIdle(options) {
2922
+ await this.underlyingPage.waitForNetworkIdle({
2923
+ idleTime: options?.idleTime || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
2924
+ concurrency: options?.concurrency || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
2925
+ timeout: options?.timeout || this.waitForNetworkIdleTimeout
2926
+ });
2927
+ }
2928
+ };
2929
+
2930
+ // src/puppeteer/index.ts
2931
+ var import_env3 = require("misoai-shared/env");
2932
+ var debug5 = (0, import_logger6.getDebug)("puppeteer:agent");
2933
+ var PuppeteerAgent = class extends PageAgent {
2934
+ constructor(page, opts) {
2935
+ const webPage = new WebPage(page);
2936
+ super(webPage, opts);
2937
+ const { forceSameTabNavigation = true } = opts ?? {};
2938
+ if (forceSameTabNavigation) {
2939
+ forceClosePopup(page, debug5);
2940
+ }
2941
+ }
2942
+ };
2943
+
2944
+ // src/puppeteer/agent-launcher.ts
2945
+ var import_constants5 = require("misoai-shared/constants");
2946
+ var import_puppeteer2 = __toESM(require("puppeteer"));
2947
+ var defaultUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
2948
+ var defaultViewportWidth = 1440;
2949
+ var defaultViewportHeight = 768;
2950
+ var defaultViewportScale = process.platform === "darwin" ? 2 : 1;
2951
+ var defaultWaitForNetworkIdleTimeout = import_constants5.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
2952
+ var launcherDebug = (0, import_logger7.getDebug)("puppeteer:launcher");
2953
+ async function launchPuppeteerPage(target, preference) {
2954
+ (0, import_utils18.assert)(target.url, "url is required");
2955
+ const freeFn = [];
2956
+ const ua = target.userAgent || defaultUA;
2957
+ let width = defaultViewportWidth;
2958
+ let preferMaximizedWindow = true;
2959
+ if (target.viewportWidth) {
2960
+ preferMaximizedWindow = false;
2961
+ (0, import_utils18.assert)(
2962
+ typeof target.viewportWidth === "number",
2963
+ "viewportWidth must be a number"
2964
+ );
2965
+ width = Number.parseInt(target.viewportWidth, 10);
2966
+ (0, import_utils18.assert)(width > 0, `viewportWidth must be greater than 0, but got ${width}`);
2967
+ }
2968
+ let height = defaultViewportHeight;
2969
+ if (target.viewportHeight) {
2970
+ preferMaximizedWindow = false;
2971
+ (0, import_utils18.assert)(
2972
+ typeof target.viewportHeight === "number",
2973
+ "viewportHeight must be a number"
2974
+ );
2975
+ height = Number.parseInt(target.viewportHeight, 10);
2976
+ (0, import_utils18.assert)(
2977
+ height > 0,
2978
+ `viewportHeight must be greater than 0, but got ${height}`
2979
+ );
2980
+ }
2981
+ let dpr = defaultViewportScale;
2982
+ if (target.viewportScale) {
2983
+ preferMaximizedWindow = false;
2984
+ (0, import_utils18.assert)(
2985
+ typeof target.viewportScale === "number",
2986
+ "viewportScale must be a number"
2987
+ );
2988
+ dpr = Number.parseInt(target.viewportScale, 10);
2989
+ (0, import_utils18.assert)(dpr > 0, `viewportScale must be greater than 0, but got ${dpr}`);
2990
+ }
2991
+ const viewportConfig = {
2992
+ width,
2993
+ height,
2994
+ deviceScaleFactor: dpr
2995
+ };
2996
+ const headed = preference?.headed || preference?.keepWindow;
2997
+ preferMaximizedWindow = preferMaximizedWindow && !!headed;
2998
+ if (headed && process.env.CI === "1") {
2999
+ console.warn(
3000
+ "you are probably running headed mode in CI, this will usually fail."
3001
+ );
3002
+ }
3003
+ const isWindows = process.platform === "win32";
3004
+ const args = [
3005
+ ...isWindows ? [] : ["--no-sandbox", "--disable-setuid-sandbox"],
3006
+ "--disable-features=HttpsFirstBalancedModeAutoEnable",
3007
+ "--disable-features=PasswordLeakDetection",
3008
+ "--disable-save-password-bubble",
3009
+ `--user-agent="${ua}"`,
3010
+ preferMaximizedWindow ? "--start-maximized" : `--window-size=${width},${height + 200}`
3011
+ // add 200px for the address bar
3012
+ ];
3013
+ launcherDebug(
3014
+ "launching browser with viewport, headed",
3015
+ headed,
3016
+ "viewport",
3017
+ viewportConfig,
3018
+ "args",
3019
+ args,
3020
+ "preference",
3021
+ preference
3022
+ );
3023
+ const browser = await import_puppeteer2.default.launch({
3024
+ headless: !headed,
3025
+ defaultViewport: viewportConfig,
3026
+ args,
3027
+ acceptInsecureCerts: target.acceptInsecureCerts
3028
+ });
3029
+ freeFn.push({
3030
+ name: "puppeteer_browser",
3031
+ fn: () => {
3032
+ if (!preference?.keepWindow) {
3033
+ if (isWindows) {
3034
+ setTimeout(() => {
3035
+ browser.close();
3036
+ }, 800);
3037
+ } else {
3038
+ browser.close();
3039
+ }
3040
+ }
3041
+ }
3042
+ });
3043
+ const pages = await browser.pages();
3044
+ const page = pages[0];
3045
+ if (target.cookie) {
3046
+ const cookieFileContent = (0, import_node_fs3.readFileSync)(target.cookie, "utf-8");
3047
+ await page.setCookie(...JSON.parse(cookieFileContent));
3048
+ }
3049
+ const waitForNetworkIdleTimeout = typeof target.waitForNetworkIdle?.timeout === "number" ? target.waitForNetworkIdle.timeout : defaultWaitForNetworkIdleTimeout;
3050
+ try {
3051
+ launcherDebug("goto", target.url);
3052
+ await page.goto(target.url);
3053
+ if (waitForNetworkIdleTimeout > 0) {
3054
+ launcherDebug("waitForNetworkIdle", waitForNetworkIdleTimeout);
3055
+ await page.waitForNetworkIdle({
3056
+ timeout: waitForNetworkIdleTimeout
3057
+ });
3058
+ }
3059
+ } catch (e) {
3060
+ if (typeof target.waitForNetworkIdle?.continueOnNetworkIdleError === "boolean" && !target.waitForNetworkIdle?.continueOnNetworkIdleError) {
3061
+ const newError = new Error(`failed to wait for network idle: ${e}`, {
3062
+ cause: e
3063
+ });
3064
+ throw newError;
3065
+ }
3066
+ const newMessage = `failed to wait for network idle after ${waitForNetworkIdleTimeout}ms, but the script will continue.`;
3067
+ console.warn(newMessage);
3068
+ }
3069
+ return { page, freeFn };
3070
+ }
3071
+ async function puppeteerAgentForTarget(target, preference) {
3072
+ const { page, freeFn } = await launchPuppeteerPage(target, preference);
3073
+ const agent = new PuppeteerAgent(page, {
3074
+ autoPrintReportMsg: false,
3075
+ testId: preference?.testId,
3076
+ cacheId: preference?.cacheId,
3077
+ aiActionContext: target.aiActionContext,
3078
+ forceSameTabNavigation: typeof target.forceSameTabNavigation !== "undefined" ? target.forceSameTabNavigation : true
3079
+ // true for default in yaml script
3080
+ });
3081
+ freeFn.push({
3082
+ name: "midscene_puppeteer_agent",
3083
+ fn: () => agent.destroy()
3084
+ });
3085
+ return { agent, freeFn };
3086
+ }
3087
+ // Annotate the CommonJS export names for ESM import in node:
3088
+ 0 && (module.exports = {
3089
+ defaultUA,
3090
+ defaultViewportHeight,
3091
+ defaultViewportScale,
3092
+ defaultViewportWidth,
3093
+ defaultWaitForNetworkIdleTimeout,
3094
+ launchPuppeteerPage,
3095
+ puppeteerAgentForTarget
3096
+ });
3097
+
3098
+ //# sourceMappingURL=puppeteer-agent-launcher.js.map