misoai-web 1.0.3 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/es/agent.js +2574 -0
  2. package/dist/es/agent.js.map +1 -0
  3. package/dist/es/bridge-mode-browser.js +955 -0
  4. package/dist/es/bridge-mode-browser.js.map +1 -0
  5. package/dist/es/bridge-mode.js +2935 -0
  6. package/dist/es/bridge-mode.js.map +1 -0
  7. package/dist/es/chrome-extension.js +3322 -0
  8. package/dist/es/chrome-extension.js.map +1 -0
  9. package/dist/es/index.js +3190 -0
  10. package/dist/es/index.js.map +1 -0
  11. package/dist/es/midscene-playground.js +2910 -0
  12. package/dist/es/midscene-playground.js.map +1 -0
  13. package/dist/es/midscene-server.js +247 -0
  14. package/dist/es/midscene-server.js.map +1 -0
  15. package/dist/es/playground.js +2681 -0
  16. package/dist/es/playground.js.map +1 -0
  17. package/dist/es/playwright-report.js +120 -0
  18. package/dist/es/playwright-report.js.map +1 -0
  19. package/dist/es/playwright.js +3135 -0
  20. package/dist/es/playwright.js.map +1 -0
  21. package/dist/es/puppeteer-agent-launcher.js +3085 -0
  22. package/dist/es/puppeteer-agent-launcher.js.map +1 -0
  23. package/dist/es/puppeteer.js +2932 -0
  24. package/dist/es/puppeteer.js.map +1 -0
  25. package/dist/es/ui-utils.js +106 -0
  26. package/dist/es/ui-utils.js.map +1 -0
  27. package/dist/es/utils.js +197 -0
  28. package/dist/es/utils.js.map +1 -0
  29. package/dist/es/yaml.js +351 -0
  30. package/dist/es/yaml.js.map +1 -0
  31. package/dist/lib/agent.js +2589 -0
  32. package/dist/lib/agent.js.map +1 -0
  33. package/dist/lib/bridge-mode-browser.js +989 -0
  34. package/dist/lib/bridge-mode-browser.js.map +1 -0
  35. package/dist/lib/bridge-mode.js +2955 -0
  36. package/dist/lib/bridge-mode.js.map +1 -0
  37. package/dist/lib/chrome-extension.js +3339 -0
  38. package/dist/lib/chrome-extension.js.map +1 -0
  39. package/dist/lib/index.js +3206 -0
  40. package/dist/lib/index.js.map +1 -0
  41. package/dist/lib/midscene-playground.js +2914 -0
  42. package/dist/lib/midscene-playground.js.map +1 -0
  43. package/dist/lib/midscene-server.js +273 -0
  44. package/dist/lib/midscene-server.js.map +1 -0
  45. package/dist/lib/playground.js +2700 -0
  46. package/dist/lib/playground.js.map +1 -0
  47. package/dist/lib/playwright-report.js +148 -0
  48. package/dist/lib/playwright-report.js.map +1 -0
  49. package/dist/lib/playwright.js +3152 -0
  50. package/dist/lib/playwright.js.map +1 -0
  51. package/dist/lib/puppeteer-agent-launcher.js +3098 -0
  52. package/dist/lib/puppeteer-agent-launcher.js.map +1 -0
  53. package/dist/lib/puppeteer.js +2943 -0
  54. package/dist/lib/puppeteer.js.map +1 -0
  55. package/dist/lib/ui-utils.js +137 -0
  56. package/dist/lib/ui-utils.js.map +1 -0
  57. package/dist/lib/utils.js +235 -0
  58. package/dist/lib/utils.js.map +1 -0
  59. package/dist/lib/yaml.js +390 -0
  60. package/dist/lib/yaml.js.map +1 -0
  61. package/dist/types/agent.d.ts +264 -0
  62. package/dist/types/bridge-mode-browser.d.ts +9 -0
  63. package/dist/types/bridge-mode.d.ts +40 -0
  64. package/dist/types/browser-a1877d18.d.ts +37 -0
  65. package/dist/types/chrome-extension.d.ts +18 -0
  66. package/dist/types/index.d.ts +16 -0
  67. package/dist/types/midscene-playground.d.ts +2 -0
  68. package/dist/types/midscene-server.d.ts +31 -0
  69. package/dist/types/page-663ece08.d.ts +333 -0
  70. package/dist/types/playground.d.ts +17 -0
  71. package/dist/types/playwright-report.d.ts +11 -0
  72. package/dist/types/playwright.d.ts +87 -0
  73. package/dist/types/puppeteer-agent-launcher.d.ts +40 -0
  74. package/dist/types/puppeteer.d.ts +17 -0
  75. package/dist/types/ui-utils.d.ts +14 -0
  76. package/dist/types/utils-badc824e.d.ts +34 -0
  77. package/dist/types/utils.d.ts +8 -0
  78. package/dist/types/yaml.d.ts +15 -0
  79. package/iife-script/htmlElement.js +99 -37
  80. package/iife-script/htmlElementDebug.js +92 -9
  81. package/package.json +2 -2
@@ -0,0 +1,955 @@
1
+ var __accessCheck = (obj, member, msg) => {
2
+ if (!member.has(obj))
3
+ throw TypeError("Cannot " + msg);
4
+ };
5
+ var __privateGet = (obj, member, getter) => {
6
+ __accessCheck(obj, member, "read from private field");
7
+ return getter ? getter.call(obj) : member.get(obj);
8
+ };
9
+ var __privateAdd = (obj, member, value) => {
10
+ if (member.has(obj))
11
+ throw TypeError("Cannot add the same private member more than once");
12
+ member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
13
+ };
14
+ var __privateSet = (obj, member, value, setter) => {
15
+ __accessCheck(obj, member, "write to private field");
16
+ setter ? setter.call(obj, value) : member.set(obj, value);
17
+ return value;
18
+ };
19
+ var __privateMethod = (obj, member, method) => {
20
+ __accessCheck(obj, member, "access private method");
21
+ return method;
22
+ };
23
+
24
+ // src/bridge-mode/page-browser-side.ts
25
+ import { assert as assert4 } from "misoai-shared/utils";
26
+
27
+ // src/common/ui-utils.ts
28
+ var limitOpenNewTabScript = `
29
+ if (!window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__) {
30
+ window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__ = true;
31
+
32
+ // Intercept the window.open method (only once)
33
+ window.open = function(url) {
34
+ console.log('Blocked window.open:', url);
35
+ window.location.href = url;
36
+ return null;
37
+ };
38
+
39
+ // Block all a tag clicks with target="_blank" (only once)
40
+ document.addEventListener('click', function(e) {
41
+ const target = e.target.closest('a');
42
+ if (target && target.target === '_blank') {
43
+ e.preventDefault();
44
+ console.log('Blocked new tab:', target.href);
45
+ window.location.href = target.href;
46
+ target.removeAttribute('target');
47
+ }
48
+ }, true);
49
+ }
50
+ `;
51
+
52
+ // src/chrome-extension/page.ts
53
+ import { treeToList } from "misoai-shared/extractor";
54
+ import { assert as assert2 } from "misoai-shared/utils";
55
+
56
+ // src/chrome-extension/cdpInput.ts
57
+ import {
58
+ _keyDefinitions
59
+ } from "misoai-shared/keyboard-layout";
60
+ import { assert } from "misoai-shared/utils";
61
+ var _pressedKeys, _client, _modifierBit, modifierBit_fn, _keyDescriptionForString, keyDescriptionForString_fn;
62
+ var CdpKeyboard = class {
63
+ constructor(client) {
64
+ __privateAdd(this, _modifierBit);
65
+ __privateAdd(this, _keyDescriptionForString);
66
+ __privateAdd(this, _pressedKeys, /* @__PURE__ */ new Set());
67
+ __privateAdd(this, _client, void 0);
68
+ this._modifiers = 0;
69
+ __privateSet(this, _client, client);
70
+ }
71
+ updateClient(client) {
72
+ __privateSet(this, _client, client);
73
+ }
74
+ async down(key, options = {
75
+ text: void 0,
76
+ commands: []
77
+ }) {
78
+ const description = __privateMethod(this, _keyDescriptionForString, keyDescriptionForString_fn).call(this, key);
79
+ const autoRepeat = __privateGet(this, _pressedKeys).has(description.code);
80
+ __privateGet(this, _pressedKeys).add(description.code);
81
+ this._modifiers |= __privateMethod(this, _modifierBit, modifierBit_fn).call(this, description.key);
82
+ const text = options.text === void 0 ? description.text : options.text;
83
+ await __privateGet(this, _client).send("Input.dispatchKeyEvent", {
84
+ type: text ? "keyDown" : "rawKeyDown",
85
+ modifiers: this._modifiers,
86
+ windowsVirtualKeyCode: description.keyCode,
87
+ code: description.code,
88
+ key: description.key,
89
+ text,
90
+ unmodifiedText: text,
91
+ autoRepeat,
92
+ location: description.location,
93
+ isKeypad: description.location === 3,
94
+ commands: options.commands
95
+ });
96
+ }
97
+ async up(key) {
98
+ const description = __privateMethod(this, _keyDescriptionForString, keyDescriptionForString_fn).call(this, key);
99
+ this._modifiers &= ~__privateMethod(this, _modifierBit, modifierBit_fn).call(this, description.key);
100
+ __privateGet(this, _pressedKeys).delete(description.code);
101
+ await __privateGet(this, _client).send("Input.dispatchKeyEvent", {
102
+ type: "keyUp",
103
+ modifiers: this._modifiers,
104
+ key: description.key,
105
+ windowsVirtualKeyCode: description.keyCode,
106
+ code: description.code,
107
+ location: description.location
108
+ });
109
+ }
110
+ async sendCharacter(char) {
111
+ await __privateGet(this, _client).send("Input.insertText", { text: char });
112
+ }
113
+ charIsKey(char) {
114
+ return !!_keyDefinitions[char];
115
+ }
116
+ async type(text, options = {}) {
117
+ const delay = options.delay || void 0;
118
+ for (const char of text) {
119
+ if (this.charIsKey(char)) {
120
+ await this.press(char, { delay });
121
+ } else {
122
+ if (delay) {
123
+ await new Promise((f) => {
124
+ return setTimeout(f, delay);
125
+ });
126
+ }
127
+ await this.sendCharacter(char);
128
+ }
129
+ }
130
+ }
131
+ async press(key, options = {}) {
132
+ const { delay = null } = options;
133
+ const keys = Array.isArray(key) ? key : [key];
134
+ for (const k of keys) {
135
+ await this.down(k, options);
136
+ }
137
+ if (delay) {
138
+ await new Promise((f) => {
139
+ return setTimeout(f, options.delay);
140
+ });
141
+ }
142
+ for (const k of [...keys].reverse()) {
143
+ await this.up(k);
144
+ }
145
+ }
146
+ };
147
+ _pressedKeys = new WeakMap();
148
+ _client = new WeakMap();
149
+ _modifierBit = new WeakSet();
150
+ modifierBit_fn = function(key) {
151
+ if (key === "Alt") {
152
+ return 1;
153
+ }
154
+ if (key === "Control") {
155
+ return 2;
156
+ }
157
+ if (key === "Meta") {
158
+ return 4;
159
+ }
160
+ if (key === "Shift") {
161
+ return 8;
162
+ }
163
+ return 0;
164
+ };
165
+ _keyDescriptionForString = new WeakSet();
166
+ keyDescriptionForString_fn = function(keyString) {
167
+ const shift = this._modifiers & 8;
168
+ const description = {
169
+ key: "",
170
+ keyCode: 0,
171
+ code: "",
172
+ text: "",
173
+ location: 0
174
+ };
175
+ const definition = _keyDefinitions[keyString];
176
+ assert(definition, `Unknown key: "${keyString}"`);
177
+ if (definition.key) {
178
+ description.key = definition.key;
179
+ }
180
+ if (shift && definition.shiftKey) {
181
+ description.key = definition.shiftKey;
182
+ }
183
+ if (definition.keyCode) {
184
+ description.keyCode = definition.keyCode;
185
+ }
186
+ if (shift && definition.shiftKeyCode) {
187
+ description.keyCode = definition.shiftKeyCode;
188
+ }
189
+ if (definition.code) {
190
+ description.code = definition.code;
191
+ }
192
+ if (definition.location) {
193
+ description.location = definition.location;
194
+ }
195
+ if (description.key.length === 1) {
196
+ description.text = description.key;
197
+ }
198
+ if (definition.text) {
199
+ description.text = definition.text;
200
+ }
201
+ if (shift && definition.shiftText) {
202
+ description.text = definition.shiftText;
203
+ }
204
+ if (this._modifiers & ~8) {
205
+ description.text = "";
206
+ }
207
+ return description;
208
+ };
209
+
210
+ // src/chrome-extension/dynamic-scripts.ts
211
+ import fs from "fs";
212
+ import { ifInBrowser } from "misoai-shared/utils";
213
+ var scriptFileContentCache = null;
214
+ var getHtmlElementScript = async () => {
215
+ const scriptFileToRetrieve = chrome.runtime.getURL("scripts/htmlElement.js");
216
+ if (scriptFileContentCache)
217
+ return scriptFileContentCache;
218
+ if (ifInBrowser) {
219
+ const script = await fetch(scriptFileToRetrieve);
220
+ scriptFileContentCache = await script.text();
221
+ return scriptFileContentCache;
222
+ }
223
+ return fs.readFileSync(scriptFileToRetrieve, "utf8");
224
+ };
225
+ var waterFlowScriptFileContentCache = null;
226
+ var injectWaterFlowAnimation = async () => {
227
+ const waterFlowScriptFileToRetrieve = chrome.runtime.getURL(
228
+ "scripts/water-flow.js"
229
+ );
230
+ if (waterFlowScriptFileContentCache)
231
+ return waterFlowScriptFileContentCache;
232
+ if (ifInBrowser) {
233
+ const script = await fetch(waterFlowScriptFileToRetrieve);
234
+ waterFlowScriptFileContentCache = await script.text();
235
+ return waterFlowScriptFileContentCache;
236
+ }
237
+ return fs.readFileSync(waterFlowScriptFileToRetrieve, "utf8");
238
+ };
239
+ var stopWaterFlowScriptFileContentCache = null;
240
+ var injectStopWaterFlowAnimation = async () => {
241
+ const stopWaterFlowScriptFileToRetrieve = chrome.runtime.getURL(
242
+ "scripts/stop-water-flow.js"
243
+ );
244
+ if (stopWaterFlowScriptFileContentCache)
245
+ return stopWaterFlowScriptFileContentCache;
246
+ if (ifInBrowser) {
247
+ const script = await fetch(stopWaterFlowScriptFileToRetrieve);
248
+ stopWaterFlowScriptFileContentCache = await script.text();
249
+ return stopWaterFlowScriptFileContentCache;
250
+ }
251
+ return fs.readFileSync(stopWaterFlowScriptFileToRetrieve, "utf8");
252
+ };
253
+
254
+ // src/chrome-extension/page.ts
255
+ function sleep(ms) {
256
+ return new Promise((resolve) => setTimeout(resolve, ms));
257
+ }
258
+ var ChromeExtensionProxyPage = class {
259
+ constructor(forceSameTabNavigation) {
260
+ this.pageType = "chrome-extension-proxy";
261
+ this.version = "1.0.4";
262
+ this.activeTabId = null;
263
+ this.tabIdOfDebuggerAttached = null;
264
+ this.attachingDebugger = null;
265
+ this.destroyed = false;
266
+ this.isMobileEmulation = null;
267
+ this.latestMouseX = 100;
268
+ this.latestMouseY = 100;
269
+ this.mouse = {
270
+ click: async (x, y) => {
271
+ await this.mouse.move(x, y);
272
+ if (this.isMobileEmulation === null) {
273
+ const result = await this.sendCommandToDebugger("Runtime.evaluate", {
274
+ expression: `(() => {
275
+ return /Android|iPhone|iPad|iPod|Mobile/i.test(navigator.userAgent);
276
+ })()`,
277
+ returnByValue: true
278
+ });
279
+ this.isMobileEmulation = result?.result?.value;
280
+ }
281
+ if (this.isMobileEmulation) {
282
+ const touchPoints = [{ x: Math.round(x), y: Math.round(y) }];
283
+ await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
284
+ type: "touchStart",
285
+ touchPoints,
286
+ modifiers: 0
287
+ });
288
+ await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
289
+ type: "touchEnd",
290
+ touchPoints: [],
291
+ modifiers: 0
292
+ });
293
+ } else {
294
+ await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
295
+ type: "mousePressed",
296
+ x,
297
+ y,
298
+ button: "left",
299
+ clickCount: 1
300
+ });
301
+ await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
302
+ type: "mouseReleased",
303
+ x,
304
+ y,
305
+ button: "left",
306
+ clickCount: 1
307
+ });
308
+ }
309
+ },
310
+ wheel: async (deltaX, deltaY, startX, startY) => {
311
+ const finalX = startX || this.latestMouseX;
312
+ const finalY = startY || this.latestMouseY;
313
+ await this.showMousePointer(finalX, finalY);
314
+ await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
315
+ type: "mouseWheel",
316
+ x: finalX,
317
+ y: finalY,
318
+ deltaX,
319
+ deltaY
320
+ });
321
+ this.latestMouseX = finalX;
322
+ this.latestMouseY = finalY;
323
+ },
324
+ move: async (x, y) => {
325
+ await this.showMousePointer(x, y);
326
+ await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
327
+ type: "mouseMoved",
328
+ x,
329
+ y
330
+ });
331
+ this.latestMouseX = x;
332
+ this.latestMouseY = y;
333
+ },
334
+ drag: async (from, to) => {
335
+ await this.mouse.move(from.x, from.y);
336
+ await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
337
+ type: "mousePressed",
338
+ x: from.x,
339
+ y: from.y,
340
+ button: "left",
341
+ clickCount: 1
342
+ });
343
+ await this.mouse.move(to.x, to.y);
344
+ await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
345
+ type: "mouseReleased",
346
+ x: to.x,
347
+ y: to.y,
348
+ button: "left",
349
+ clickCount: 1
350
+ });
351
+ }
352
+ };
353
+ this.keyboard = {
354
+ type: async (text) => {
355
+ const cdpKeyboard = new CdpKeyboard({
356
+ send: this.sendCommandToDebugger.bind(this)
357
+ });
358
+ await cdpKeyboard.type(text, { delay: 0 });
359
+ },
360
+ press: async (action) => {
361
+ const cdpKeyboard = new CdpKeyboard({
362
+ send: this.sendCommandToDebugger.bind(this)
363
+ });
364
+ const keys = Array.isArray(action) ? action : [action];
365
+ for (const k of keys) {
366
+ const commands = k.command ? [k.command] : [];
367
+ await cdpKeyboard.down(k.key, { commands });
368
+ }
369
+ for (const k of [...keys].reverse()) {
370
+ await cdpKeyboard.up(k.key);
371
+ }
372
+ }
373
+ };
374
+ this.forceSameTabNavigation = forceSameTabNavigation;
375
+ }
376
+ async setActiveTabId(tabId) {
377
+ if (this.activeTabId) {
378
+ throw new Error(
379
+ `Active tab id is already set, which is ${this.activeTabId}, cannot set it to ${tabId}`
380
+ );
381
+ }
382
+ await chrome.tabs.update(tabId, { active: true });
383
+ this.activeTabId = tabId;
384
+ }
385
+ async getActiveTabId() {
386
+ return this.activeTabId;
387
+ }
388
+ /**
389
+ * Get a list of current tabs
390
+ * @returns {Promise<Array<{id: number, title: string, url: string}>>}
391
+ */
392
+ async getBrowserTabList() {
393
+ const tabs = await chrome.tabs.query({ currentWindow: true });
394
+ return tabs.map((tab) => ({
395
+ id: `${tab.id}`,
396
+ title: tab.title,
397
+ url: tab.url,
398
+ currentActiveTab: tab.active
399
+ })).filter((tab) => tab.id && tab.title && tab.url);
400
+ }
401
+ async getTabIdOrConnectToCurrentTab() {
402
+ if (this.activeTabId) {
403
+ return this.activeTabId;
404
+ }
405
+ const tabId = await chrome.tabs.query({ active: true, currentWindow: true }).then((tabs) => tabs[0]?.id);
406
+ this.activeTabId = tabId || 0;
407
+ return this.activeTabId;
408
+ }
409
+ async attachDebugger() {
410
+ assert2(!this.destroyed, "Page is destroyed");
411
+ if (this.attachingDebugger) {
412
+ await this.attachingDebugger;
413
+ return;
414
+ }
415
+ this.attachingDebugger = (async () => {
416
+ const url = await this.url();
417
+ let error = null;
418
+ if (url.startsWith("chrome://")) {
419
+ throw new Error(
420
+ "Cannot attach debugger to chrome:// pages, please use Midscene in a normal page with http://, https:// or file://"
421
+ );
422
+ }
423
+ try {
424
+ const currentTabId = await this.getTabIdOrConnectToCurrentTab();
425
+ if (this.tabIdOfDebuggerAttached === currentTabId) {
426
+ return;
427
+ }
428
+ if (this.tabIdOfDebuggerAttached && this.tabIdOfDebuggerAttached !== currentTabId) {
429
+ console.log(
430
+ "detach the previous tab",
431
+ this.tabIdOfDebuggerAttached,
432
+ "->",
433
+ currentTabId
434
+ );
435
+ try {
436
+ await this.detachDebugger(this.tabIdOfDebuggerAttached);
437
+ } catch (error2) {
438
+ console.error("Failed to detach debugger", error2);
439
+ }
440
+ }
441
+ console.log("attaching debugger", currentTabId);
442
+ await chrome.debugger.attach({ tabId: currentTabId }, "1.3");
443
+ await sleep(500);
444
+ this.tabIdOfDebuggerAttached = currentTabId;
445
+ await this.enableWaterFlowAnimation();
446
+ } catch (e) {
447
+ console.error("Failed to attach debugger", e);
448
+ error = e;
449
+ } finally {
450
+ this.attachingDebugger = null;
451
+ }
452
+ if (error) {
453
+ throw error;
454
+ }
455
+ })();
456
+ await this.attachingDebugger;
457
+ }
458
+ async showMousePointer(x, y) {
459
+ const pointerScript = `(() => {
460
+ if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
461
+ window.midsceneWaterFlowAnimation.enable();
462
+ window.midsceneWaterFlowAnimation.showMousePointer(${x}, ${y});
463
+ } else {
464
+ console.log('midsceneWaterFlowAnimation is not defined');
465
+ }
466
+ })()`;
467
+ await this.sendCommandToDebugger("Runtime.evaluate", {
468
+ expression: `${pointerScript}`
469
+ });
470
+ }
471
+ async hideMousePointer() {
472
+ await this.sendCommandToDebugger("Runtime.evaluate", {
473
+ expression: `(() => {
474
+ if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
475
+ window.midsceneWaterFlowAnimation.hideMousePointer();
476
+ }
477
+ })()`
478
+ });
479
+ }
480
+ async detachDebugger(tabId) {
481
+ const tabIdToDetach = tabId || this.tabIdOfDebuggerAttached;
482
+ console.log("detaching debugger", tabIdToDetach);
483
+ if (!tabIdToDetach) {
484
+ console.warn("No tab id to detach");
485
+ return;
486
+ }
487
+ try {
488
+ await this.disableWaterFlowAnimation(tabIdToDetach);
489
+ await sleep(200);
490
+ } catch (error) {
491
+ console.warn("Failed to disable water flow animation", error);
492
+ }
493
+ try {
494
+ await chrome.debugger.detach({ tabId: tabIdToDetach });
495
+ } catch (error) {
496
+ console.warn("Failed to detach debugger", error);
497
+ }
498
+ this.tabIdOfDebuggerAttached = null;
499
+ }
500
+ async enableWaterFlowAnimation() {
501
+ if (this.forceSameTabNavigation) {
502
+ await chrome.debugger.sendCommand(
503
+ { tabId: this.tabIdOfDebuggerAttached },
504
+ "Runtime.evaluate",
505
+ {
506
+ expression: limitOpenNewTabScript
507
+ }
508
+ );
509
+ }
510
+ const script = await injectWaterFlowAnimation();
511
+ await chrome.debugger.sendCommand(
512
+ { tabId: this.tabIdOfDebuggerAttached },
513
+ "Runtime.evaluate",
514
+ {
515
+ expression: script
516
+ }
517
+ );
518
+ }
519
+ async disableWaterFlowAnimation(tabId) {
520
+ const script = await injectStopWaterFlowAnimation();
521
+ await chrome.debugger.sendCommand({ tabId }, "Runtime.evaluate", {
522
+ expression: script
523
+ });
524
+ }
525
+ async sendCommandToDebugger(command, params) {
526
+ await this.attachDebugger();
527
+ assert2(this.tabIdOfDebuggerAttached, "Debugger is not attached");
528
+ this.enableWaterFlowAnimation();
529
+ return await chrome.debugger.sendCommand(
530
+ { tabId: this.tabIdOfDebuggerAttached },
531
+ command,
532
+ params
533
+ );
534
+ }
535
+ async getPageContentByCDP() {
536
+ const script = await getHtmlElementScript();
537
+ await this.sendCommandToDebugger("Runtime.evaluate", {
538
+ expression: script
539
+ });
540
+ const expression = () => {
541
+ window.midscene_element_inspector.setNodeHashCacheListOnWindow();
542
+ return {
543
+ tree: window.midscene_element_inspector.webExtractNodeTree(),
544
+ size: {
545
+ width: document.documentElement.clientWidth,
546
+ height: document.documentElement.clientHeight,
547
+ dpr: window.devicePixelRatio
548
+ }
549
+ };
550
+ };
551
+ const returnValue = await this.sendCommandToDebugger("Runtime.evaluate", {
552
+ expression: `(${expression.toString()})()`,
553
+ returnByValue: true
554
+ });
555
+ if (!returnValue.result.value) {
556
+ const errorDescription = returnValue.exceptionDetails?.exception?.description || "";
557
+ if (!errorDescription) {
558
+ console.error("returnValue from cdp", returnValue);
559
+ }
560
+ throw new Error(
561
+ `Failed to get page content from page, error: ${errorDescription}`
562
+ );
563
+ }
564
+ return returnValue.result.value;
565
+ }
566
+ async evaluateJavaScript(script) {
567
+ return this.sendCommandToDebugger("Runtime.evaluate", {
568
+ expression: script
569
+ });
570
+ }
571
+ // current implementation is wait until domReadyState is complete
572
+ async waitUntilNetworkIdle() {
573
+ const timeout = 1e4;
574
+ const startTime = Date.now();
575
+ let lastReadyState = "";
576
+ while (Date.now() - startTime < timeout) {
577
+ const result = await this.sendCommandToDebugger("Runtime.evaluate", {
578
+ expression: "document.readyState"
579
+ });
580
+ lastReadyState = result.result.value;
581
+ if (lastReadyState === "complete") {
582
+ await new Promise((resolve) => setTimeout(resolve, 300));
583
+ return;
584
+ }
585
+ await new Promise((resolve) => setTimeout(resolve, 300));
586
+ }
587
+ throw new Error(
588
+ `Failed to wait until network idle, last readyState: ${lastReadyState}`
589
+ );
590
+ }
591
+ async getElementsInfo() {
592
+ const tree = await this.getElementsNodeTree();
593
+ return treeToList(tree);
594
+ }
595
+ async getXpathsById(id) {
596
+ const script = await getHtmlElementScript();
597
+ await this.sendCommandToDebugger("Runtime.evaluate", {
598
+ expression: script
599
+ });
600
+ const result = await this.sendCommandToDebugger("Runtime.evaluate", {
601
+ expression: `window.midscene_element_inspector.getXpathsById('${id}')`,
602
+ returnByValue: true
603
+ });
604
+ return result.result.value;
605
+ }
606
+ async getElementInfoByXpath(xpath) {
607
+ const script = await getHtmlElementScript();
608
+ await this.sendCommandToDebugger("Runtime.evaluate", {
609
+ expression: script
610
+ });
611
+ const result = await this.sendCommandToDebugger("Runtime.evaluate", {
612
+ expression: `window.midscene_element_inspector.getElementInfoByXpath('${xpath}')`,
613
+ returnByValue: true
614
+ });
615
+ return result.result.value;
616
+ }
617
+ async getElementsNodeTree() {
618
+ await this.hideMousePointer();
619
+ const content = await this.getPageContentByCDP();
620
+ if (content?.size) {
621
+ this.viewportSize = content.size;
622
+ }
623
+ return content?.tree || { node: null, children: [] };
624
+ }
625
+ async size() {
626
+ if (this.viewportSize)
627
+ return this.viewportSize;
628
+ const content = await this.getPageContentByCDP();
629
+ return content.size;
630
+ }
631
+ async screenshotBase64() {
632
+ await this.hideMousePointer();
633
+ const base64 = await this.sendCommandToDebugger("Page.captureScreenshot", {
634
+ format: "jpeg",
635
+ quality: 90
636
+ });
637
+ return `data:image/jpeg;base64,${base64.data}`;
638
+ }
639
+ async url() {
640
+ const tabId = await this.getTabIdOrConnectToCurrentTab();
641
+ const url = await chrome.tabs.get(tabId).then((tab) => tab.url);
642
+ return url || "";
643
+ }
644
+ async scrollUntilTop(startingPoint) {
645
+ if (startingPoint) {
646
+ await this.mouse.move(startingPoint.left, startingPoint.top);
647
+ }
648
+ return this.mouse.wheel(0, -9999999);
649
+ }
650
+ async scrollUntilBottom(startingPoint) {
651
+ if (startingPoint) {
652
+ await this.mouse.move(startingPoint.left, startingPoint.top);
653
+ }
654
+ return this.mouse.wheel(0, 9999999);
655
+ }
656
+ async scrollUntilLeft(startingPoint) {
657
+ if (startingPoint) {
658
+ await this.mouse.move(startingPoint.left, startingPoint.top);
659
+ }
660
+ return this.mouse.wheel(-9999999, 0);
661
+ }
662
+ async scrollUntilRight(startingPoint) {
663
+ if (startingPoint) {
664
+ await this.mouse.move(startingPoint.left, startingPoint.top);
665
+ }
666
+ return this.mouse.wheel(9999999, 0);
667
+ }
668
+ async scrollUp(distance, startingPoint) {
669
+ const { height } = await this.size();
670
+ const scrollDistance = distance || height * 0.7;
671
+ return this.mouse.wheel(
672
+ 0,
673
+ -scrollDistance,
674
+ startingPoint?.left,
675
+ startingPoint?.top
676
+ );
677
+ }
678
+ async scrollDown(distance, startingPoint) {
679
+ const { height } = await this.size();
680
+ const scrollDistance = distance || height * 0.7;
681
+ return this.mouse.wheel(
682
+ 0,
683
+ scrollDistance,
684
+ startingPoint?.left,
685
+ startingPoint?.top
686
+ );
687
+ }
688
+ async scrollLeft(distance, startingPoint) {
689
+ const { width } = await this.size();
690
+ const scrollDistance = distance || width * 0.7;
691
+ return this.mouse.wheel(
692
+ -scrollDistance,
693
+ 0,
694
+ startingPoint?.left,
695
+ startingPoint?.top
696
+ );
697
+ }
698
+ async scrollRight(distance, startingPoint) {
699
+ const { width } = await this.size();
700
+ const scrollDistance = distance || width * 0.7;
701
+ return this.mouse.wheel(
702
+ scrollDistance,
703
+ 0,
704
+ startingPoint?.left,
705
+ startingPoint?.top
706
+ );
707
+ }
708
+ async clearInput(element) {
709
+ if (!element) {
710
+ console.warn("No element to clear input");
711
+ return;
712
+ }
713
+ await this.mouse.click(element.center[0], element.center[1]);
714
+ await this.sendCommandToDebugger("Input.dispatchKeyEvent", {
715
+ type: "keyDown",
716
+ commands: ["selectAll"]
717
+ });
718
+ await this.sendCommandToDebugger("Input.dispatchKeyEvent", {
719
+ type: "keyUp",
720
+ commands: ["selectAll"]
721
+ });
722
+ await sleep(100);
723
+ await this.keyboard.press({
724
+ key: "Backspace"
725
+ });
726
+ }
727
+ async destroy() {
728
+ this.activeTabId = null;
729
+ await this.detachDebugger();
730
+ this.destroyed = true;
731
+ }
732
+ };
733
+
734
+ // src/bridge-mode/common.ts
735
+ var DefaultBridgeServerPort = 3766;
736
+ var DefaultLocalEndpoint = `http://127.0.0.1:${DefaultBridgeServerPort}`;
737
+
738
+ // src/bridge-mode/io-client.ts
739
+ import { assert as assert3 } from "misoai-shared/utils";
740
+ import { io as ClientIO } from "socket.io-client";
741
+ var BridgeClient = class {
742
+ constructor(endpoint, onBridgeCall, onDisconnect) {
743
+ this.endpoint = endpoint;
744
+ this.onBridgeCall = onBridgeCall;
745
+ this.onDisconnect = onDisconnect;
746
+ this.socket = null;
747
+ this.serverVersion = null;
748
+ }
749
+ async connect() {
750
+ return new Promise((resolve, reject) => {
751
+ this.socket = ClientIO(this.endpoint, {
752
+ reconnection: false,
753
+ query: {
754
+ version: "1.0.4"
755
+ }
756
+ });
757
+ const timeout = setTimeout(() => {
758
+ try {
759
+ this.socket?.offAny();
760
+ this.socket?.close();
761
+ } catch (e) {
762
+ console.warn("got error when offing socket", e);
763
+ }
764
+ this.socket = null;
765
+ reject(new Error("failed to connect to bridge server after timeout"));
766
+ }, 1 * 1e3);
767
+ this.socket.on("disconnect", (reason) => {
768
+ this.socket = null;
769
+ this.onDisconnect?.();
770
+ });
771
+ this.socket.on("connect_error", (e) => {
772
+ console.error("bridge-connect-error", e);
773
+ reject(new Error(e || "bridge connect error"));
774
+ });
775
+ this.socket.on(
776
+ "bridge-connected" /* Connected */,
777
+ (payload) => {
778
+ clearTimeout(timeout);
779
+ this.serverVersion = payload?.version || "unknown";
780
+ resolve(this.socket);
781
+ }
782
+ );
783
+ this.socket.on("bridge-refused" /* Refused */, (e) => {
784
+ console.error("bridge-refused", e);
785
+ try {
786
+ this.socket?.disconnect();
787
+ } catch (e2) {
788
+ }
789
+ reject(new Error(e || "bridge refused"));
790
+ });
791
+ this.socket.on("bridge-call" /* Call */, (call) => {
792
+ const id = call.id;
793
+ assert3(typeof id !== "undefined", "call id is required");
794
+ (async () => {
795
+ let response;
796
+ try {
797
+ response = await this.onBridgeCall(call.method, call.args);
798
+ } catch (e) {
799
+ const errorContent = `Error from bridge client when calling, method: ${call.method}, args: ${call.args}, error: ${e?.message || e}
800
+ ${e?.stack || ""}`;
801
+ console.error(errorContent);
802
+ return this.socket?.emit("bridge-call-response" /* CallResponse */, {
803
+ id,
804
+ error: errorContent
805
+ });
806
+ }
807
+ this.socket?.emit("bridge-call-response" /* CallResponse */, {
808
+ id,
809
+ response
810
+ });
811
+ })();
812
+ });
813
+ });
814
+ }
815
+ disconnect() {
816
+ this.socket?.disconnect();
817
+ this.socket = null;
818
+ }
819
+ };
820
+
821
+ // src/bridge-mode/page-browser-side.ts
822
+ var ExtensionBridgePageBrowserSide = class extends ChromeExtensionProxyPage {
823
+ constructor(onDisconnect = () => {
824
+ }, onLogMessage = () => {
825
+ }, forceSameTabNavigation = true) {
826
+ super(forceSameTabNavigation);
827
+ this.onDisconnect = onDisconnect;
828
+ this.onLogMessage = onLogMessage;
829
+ this.bridgeClient = null;
830
+ this.newlyCreatedTabIds = [];
831
+ }
832
+ async setupBridgeClient() {
833
+ this.bridgeClient = new BridgeClient(
834
+ `ws://localhost:${DefaultBridgeServerPort}`,
835
+ async (method, args) => {
836
+ console.log("bridge call from cli side", method, args);
837
+ if (method === "connectNewTabWithUrl" /* ConnectNewTabWithUrl */) {
838
+ return this.connectNewTabWithUrl.apply(
839
+ this,
840
+ args
841
+ );
842
+ }
843
+ if (method === "getBrowserTabList" /* GetBrowserTabList */) {
844
+ return this.getBrowserTabList.apply(this, args);
845
+ }
846
+ if (method === "setActiveTabId" /* SetActiveTabId */) {
847
+ return this.setActiveTabId.apply(this, args);
848
+ }
849
+ if (method === "connectCurrentTab" /* ConnectCurrentTab */) {
850
+ return this.connectCurrentTab.apply(this, args);
851
+ }
852
+ if (method === "bridge-update-agent-status" /* UpdateAgentStatus */) {
853
+ return this.onLogMessage(args[0], "status");
854
+ }
855
+ const tabId = await this.getActiveTabId();
856
+ if (!tabId || tabId === 0) {
857
+ throw new Error("no tab is connected");
858
+ }
859
+ if (method.startsWith("mouse." /* PREFIX */)) {
860
+ const actionName = method.split(".")[1];
861
+ if (actionName === "drag") {
862
+ return this.mouse[actionName].apply(this.mouse, args);
863
+ }
864
+ return this.mouse[actionName].apply(this.mouse, args);
865
+ }
866
+ if (method.startsWith("keyboard." /* PREFIX */)) {
867
+ const actionName = method.split(".")[1];
868
+ if (actionName === "press") {
869
+ return this.keyboard[actionName].apply(this.keyboard, args);
870
+ }
871
+ return this.keyboard[actionName].apply(this.keyboard, args);
872
+ }
873
+ try {
874
+ const result = await this[method](
875
+ ...args
876
+ );
877
+ return result;
878
+ } catch (e) {
879
+ const errorMessage = e instanceof Error ? e.message : "Unknown error";
880
+ console.error("error calling method", method, args, e);
881
+ this.onLogMessage(
882
+ `Error calling method: ${method}, ${errorMessage}`,
883
+ "log"
884
+ );
885
+ throw new Error(errorMessage, { cause: e });
886
+ }
887
+ },
888
+ // on disconnect
889
+ () => {
890
+ return this.destroy();
891
+ }
892
+ );
893
+ await this.bridgeClient.connect();
894
+ this.onLogMessage(
895
+ `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"1.0.4"}`,
896
+ "log"
897
+ );
898
+ }
899
+ async connect() {
900
+ return await this.setupBridgeClient();
901
+ }
902
+ async connectNewTabWithUrl(url, options = {
903
+ forceSameTabNavigation: true
904
+ }) {
905
+ const tab = await chrome.tabs.create({ url });
906
+ const tabId = tab.id;
907
+ assert4(tabId, "failed to get tabId after creating a new tab");
908
+ this.onLogMessage(`Creating new tab: ${url}`, "log");
909
+ this.newlyCreatedTabIds.push(tabId);
910
+ if (options?.forceSameTabNavigation) {
911
+ this.forceSameTabNavigation = true;
912
+ }
913
+ await this.setActiveTabId(tabId);
914
+ }
915
+ async connectCurrentTab(options = {
916
+ forceSameTabNavigation: true
917
+ }) {
918
+ const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
919
+ const tabId = tabs[0]?.id;
920
+ assert4(tabId, "failed to get tabId");
921
+ this.onLogMessage(`Connected to current tab: ${tabs[0]?.url}`, "log");
922
+ if (options?.forceSameTabNavigation) {
923
+ this.forceSameTabNavigation = true;
924
+ }
925
+ await this.setActiveTabId(tabId);
926
+ }
927
+ async setDestroyOptions(options) {
928
+ this.destroyOptions = options;
929
+ }
930
+ async destroy() {
931
+ if (this.destroyOptions?.closeTab && this.newlyCreatedTabIds.length > 0) {
932
+ this.onLogMessage("Closing all newly created tabs by bridge...", "log");
933
+ for (const tabId of this.newlyCreatedTabIds) {
934
+ await chrome.tabs.remove(tabId);
935
+ }
936
+ this.newlyCreatedTabIds = [];
937
+ }
938
+ await super.destroy();
939
+ if (this.bridgeClient) {
940
+ this.bridgeClient.disconnect();
941
+ this.bridgeClient = null;
942
+ this.onDisconnect();
943
+ }
944
+ }
945
+ };
946
+ export {
947
+ ExtensionBridgePageBrowserSide
948
+ };
949
+ /**
950
+ * @license
951
+ * Copyright 2017 Google Inc.
952
+ * SPDX-License-Identifier: Apache-2.0
953
+ */
954
+
955
+ //# sourceMappingURL=bridge-mode-browser.js.map