misoai-web 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -8
- package/bin/midscene-playground +2 -2
- package/package.json +23 -24
- package/dist/es/agent.js +0 -2451
- package/dist/es/agent.js.map +0 -1
- package/dist/es/bridge-mode-browser.js +0 -908
- package/dist/es/bridge-mode-browser.js.map +0 -1
- package/dist/es/bridge-mode.js +0 -2812
- package/dist/es/bridge-mode.js.map +0 -1
- package/dist/es/chrome-extension.js +0 -3152
- package/dist/es/chrome-extension.js.map +0 -1
- package/dist/es/index.js +0 -3052
- package/dist/es/index.js.map +0 -1
- package/dist/es/midscene-playground.js +0 -2781
- package/dist/es/midscene-playground.js.map +0 -1
- package/dist/es/midscene-server.js +0 -247
- package/dist/es/midscene-server.js.map +0 -1
- package/dist/es/playground.js +0 -2552
- package/dist/es/playground.js.map +0 -1
- package/dist/es/playwright-report.js +0 -120
- package/dist/es/playwright-report.js.map +0 -1
- package/dist/es/playwright.js +0 -2997
- package/dist/es/playwright.js.map +0 -1
- package/dist/es/puppeteer-agent-launcher.js +0 -2947
- package/dist/es/puppeteer-agent-launcher.js.map +0 -1
- package/dist/es/puppeteer.js +0 -2794
- package/dist/es/puppeteer.js.map +0 -1
- package/dist/es/ui-utils.js +0 -106
- package/dist/es/ui-utils.js.map +0 -1
- package/dist/es/utils.js +0 -197
- package/dist/es/utils.js.map +0 -1
- package/dist/es/yaml.js +0 -333
- package/dist/es/yaml.js.map +0 -1
- package/dist/lib/agent.js +0 -2466
- package/dist/lib/agent.js.map +0 -1
- package/dist/lib/bridge-mode-browser.js +0 -942
- package/dist/lib/bridge-mode-browser.js.map +0 -1
- package/dist/lib/bridge-mode.js +0 -2832
- package/dist/lib/bridge-mode.js.map +0 -1
- package/dist/lib/chrome-extension.js +0 -3169
- package/dist/lib/chrome-extension.js.map +0 -1
- package/dist/lib/index.js +0 -3071
- package/dist/lib/index.js.map +0 -1
- package/dist/lib/midscene-playground.js +0 -2785
- package/dist/lib/midscene-playground.js.map +0 -1
- package/dist/lib/midscene-server.js +0 -273
- package/dist/lib/midscene-server.js.map +0 -1
- package/dist/lib/playground.js +0 -2571
- package/dist/lib/playground.js.map +0 -1
- package/dist/lib/playwright-report.js +0 -148
- package/dist/lib/playwright-report.js.map +0 -1
- package/dist/lib/playwright.js +0 -3017
- package/dist/lib/playwright.js.map +0 -1
- package/dist/lib/puppeteer-agent-launcher.js +0 -2963
- package/dist/lib/puppeteer-agent-launcher.js.map +0 -1
- package/dist/lib/puppeteer.js +0 -2808
- package/dist/lib/puppeteer.js.map +0 -1
- package/dist/lib/ui-utils.js +0 -137
- package/dist/lib/ui-utils.js.map +0 -1
- package/dist/lib/utils.js +0 -235
- package/dist/lib/utils.js.map +0 -1
- package/dist/lib/yaml.js +0 -372
- package/dist/lib/yaml.js.map +0 -1
- package/dist/types/agent.d.ts +0 -254
- package/dist/types/bridge-mode-browser.d.ts +0 -9
- package/dist/types/bridge-mode.d.ts +0 -40
- package/dist/types/browser-d447695b.d.ts +0 -37
- package/dist/types/chrome-extension.d.ts +0 -18
- package/dist/types/index.d.ts +0 -16
- package/dist/types/midscene-playground.d.ts +0 -2
- package/dist/types/midscene-server.d.ts +0 -31
- package/dist/types/page-b8ada1f3.d.ts +0 -322
- package/dist/types/playground.d.ts +0 -17
- package/dist/types/playwright-report.d.ts +0 -11
- package/dist/types/playwright.d.ts +0 -87
- package/dist/types/puppeteer-agent-launcher.d.ts +0 -40
- package/dist/types/puppeteer.d.ts +0 -17
- package/dist/types/ui-utils.d.ts +0 -14
- package/dist/types/utils-badc824e.d.ts +0 -34
- package/dist/types/utils.d.ts +0 -8
- package/dist/types/yaml.d.ts +0 -15
@@ -1,908 +0,0 @@
|
|
1
|
-
var __accessCheck = (obj, member, msg) => {
|
2
|
-
if (!member.has(obj))
|
3
|
-
throw TypeError("Cannot " + msg);
|
4
|
-
};
|
5
|
-
var __privateGet = (obj, member, getter) => {
|
6
|
-
__accessCheck(obj, member, "read from private field");
|
7
|
-
return getter ? getter.call(obj) : member.get(obj);
|
8
|
-
};
|
9
|
-
var __privateAdd = (obj, member, value) => {
|
10
|
-
if (member.has(obj))
|
11
|
-
throw TypeError("Cannot add the same private member more than once");
|
12
|
-
member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
|
13
|
-
};
|
14
|
-
var __privateSet = (obj, member, value, setter) => {
|
15
|
-
__accessCheck(obj, member, "write to private field");
|
16
|
-
setter ? setter.call(obj, value) : member.set(obj, value);
|
17
|
-
return value;
|
18
|
-
};
|
19
|
-
var __privateMethod = (obj, member, method) => {
|
20
|
-
__accessCheck(obj, member, "access private method");
|
21
|
-
return method;
|
22
|
-
};
|
23
|
-
|
24
|
-
// src/bridge-mode/page-browser-side.ts
|
25
|
-
import { assert as assert4 } from "misoai-shared/utils";
|
26
|
-
|
27
|
-
// src/common/ui-utils.ts
|
28
|
-
var limitOpenNewTabScript = `
|
29
|
-
if (!window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__) {
|
30
|
-
window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__ = true;
|
31
|
-
|
32
|
-
// Intercept the window.open method (only once)
|
33
|
-
window.open = function(url) {
|
34
|
-
console.log('Blocked window.open:', url);
|
35
|
-
window.location.href = url;
|
36
|
-
return null;
|
37
|
-
};
|
38
|
-
|
39
|
-
// Block all a tag clicks with target="_blank" (only once)
|
40
|
-
document.addEventListener('click', function(e) {
|
41
|
-
const target = e.target.closest('a');
|
42
|
-
if (target && target.target === '_blank') {
|
43
|
-
e.preventDefault();
|
44
|
-
console.log('Blocked new tab:', target.href);
|
45
|
-
window.location.href = target.href;
|
46
|
-
target.removeAttribute('target');
|
47
|
-
}
|
48
|
-
}, true);
|
49
|
-
}
|
50
|
-
`;
|
51
|
-
|
52
|
-
// src/chrome-extension/page.ts
|
53
|
-
import { treeToList } from "misoai-shared/extractor";
|
54
|
-
import { assert as assert2 } from "misoai-shared/utils";
|
55
|
-
|
56
|
-
// src/chrome-extension/cdpInput.ts
|
57
|
-
import {
|
58
|
-
_keyDefinitions
|
59
|
-
} from "misoai-shared/keyboard-layout";
|
60
|
-
import { assert } from "misoai-shared/utils";
|
61
|
-
var _pressedKeys, _client, _modifierBit, modifierBit_fn, _keyDescriptionForString, keyDescriptionForString_fn;
|
62
|
-
var CdpKeyboard = class {
|
63
|
-
constructor(client) {
|
64
|
-
__privateAdd(this, _modifierBit);
|
65
|
-
__privateAdd(this, _keyDescriptionForString);
|
66
|
-
__privateAdd(this, _pressedKeys, /* @__PURE__ */ new Set());
|
67
|
-
__privateAdd(this, _client, void 0);
|
68
|
-
this._modifiers = 0;
|
69
|
-
__privateSet(this, _client, client);
|
70
|
-
}
|
71
|
-
updateClient(client) {
|
72
|
-
__privateSet(this, _client, client);
|
73
|
-
}
|
74
|
-
async down(key, options = {
|
75
|
-
text: void 0,
|
76
|
-
commands: []
|
77
|
-
}) {
|
78
|
-
const description = __privateMethod(this, _keyDescriptionForString, keyDescriptionForString_fn).call(this, key);
|
79
|
-
const autoRepeat = __privateGet(this, _pressedKeys).has(description.code);
|
80
|
-
__privateGet(this, _pressedKeys).add(description.code);
|
81
|
-
this._modifiers |= __privateMethod(this, _modifierBit, modifierBit_fn).call(this, description.key);
|
82
|
-
const text = options.text === void 0 ? description.text : options.text;
|
83
|
-
await __privateGet(this, _client).send("Input.dispatchKeyEvent", {
|
84
|
-
type: text ? "keyDown" : "rawKeyDown",
|
85
|
-
modifiers: this._modifiers,
|
86
|
-
windowsVirtualKeyCode: description.keyCode,
|
87
|
-
code: description.code,
|
88
|
-
key: description.key,
|
89
|
-
text,
|
90
|
-
unmodifiedText: text,
|
91
|
-
autoRepeat,
|
92
|
-
location: description.location,
|
93
|
-
isKeypad: description.location === 3,
|
94
|
-
commands: options.commands
|
95
|
-
});
|
96
|
-
}
|
97
|
-
async up(key) {
|
98
|
-
const description = __privateMethod(this, _keyDescriptionForString, keyDescriptionForString_fn).call(this, key);
|
99
|
-
this._modifiers &= ~__privateMethod(this, _modifierBit, modifierBit_fn).call(this, description.key);
|
100
|
-
__privateGet(this, _pressedKeys).delete(description.code);
|
101
|
-
await __privateGet(this, _client).send("Input.dispatchKeyEvent", {
|
102
|
-
type: "keyUp",
|
103
|
-
modifiers: this._modifiers,
|
104
|
-
key: description.key,
|
105
|
-
windowsVirtualKeyCode: description.keyCode,
|
106
|
-
code: description.code,
|
107
|
-
location: description.location
|
108
|
-
});
|
109
|
-
}
|
110
|
-
async sendCharacter(char) {
|
111
|
-
await __privateGet(this, _client).send("Input.insertText", { text: char });
|
112
|
-
}
|
113
|
-
charIsKey(char) {
|
114
|
-
return !!_keyDefinitions[char];
|
115
|
-
}
|
116
|
-
async type(text, options = {}) {
|
117
|
-
const delay = options.delay || void 0;
|
118
|
-
for (const char of text) {
|
119
|
-
if (this.charIsKey(char)) {
|
120
|
-
await this.press(char, { delay });
|
121
|
-
} else {
|
122
|
-
if (delay) {
|
123
|
-
await new Promise((f) => {
|
124
|
-
return setTimeout(f, delay);
|
125
|
-
});
|
126
|
-
}
|
127
|
-
await this.sendCharacter(char);
|
128
|
-
}
|
129
|
-
}
|
130
|
-
}
|
131
|
-
async press(key, options = {}) {
|
132
|
-
const { delay = null } = options;
|
133
|
-
const keys = Array.isArray(key) ? key : [key];
|
134
|
-
for (const k of keys) {
|
135
|
-
await this.down(k, options);
|
136
|
-
}
|
137
|
-
if (delay) {
|
138
|
-
await new Promise((f) => {
|
139
|
-
return setTimeout(f, options.delay);
|
140
|
-
});
|
141
|
-
}
|
142
|
-
for (const k of [...keys].reverse()) {
|
143
|
-
await this.up(k);
|
144
|
-
}
|
145
|
-
}
|
146
|
-
};
|
147
|
-
_pressedKeys = new WeakMap();
|
148
|
-
_client = new WeakMap();
|
149
|
-
_modifierBit = new WeakSet();
|
150
|
-
modifierBit_fn = function(key) {
|
151
|
-
if (key === "Alt") {
|
152
|
-
return 1;
|
153
|
-
}
|
154
|
-
if (key === "Control") {
|
155
|
-
return 2;
|
156
|
-
}
|
157
|
-
if (key === "Meta") {
|
158
|
-
return 4;
|
159
|
-
}
|
160
|
-
if (key === "Shift") {
|
161
|
-
return 8;
|
162
|
-
}
|
163
|
-
return 0;
|
164
|
-
};
|
165
|
-
_keyDescriptionForString = new WeakSet();
|
166
|
-
keyDescriptionForString_fn = function(keyString) {
|
167
|
-
const shift = this._modifiers & 8;
|
168
|
-
const description = {
|
169
|
-
key: "",
|
170
|
-
keyCode: 0,
|
171
|
-
code: "",
|
172
|
-
text: "",
|
173
|
-
location: 0
|
174
|
-
};
|
175
|
-
const definition = _keyDefinitions[keyString];
|
176
|
-
assert(definition, `Unknown key: "${keyString}"`);
|
177
|
-
if (definition.key) {
|
178
|
-
description.key = definition.key;
|
179
|
-
}
|
180
|
-
if (shift && definition.shiftKey) {
|
181
|
-
description.key = definition.shiftKey;
|
182
|
-
}
|
183
|
-
if (definition.keyCode) {
|
184
|
-
description.keyCode = definition.keyCode;
|
185
|
-
}
|
186
|
-
if (shift && definition.shiftKeyCode) {
|
187
|
-
description.keyCode = definition.shiftKeyCode;
|
188
|
-
}
|
189
|
-
if (definition.code) {
|
190
|
-
description.code = definition.code;
|
191
|
-
}
|
192
|
-
if (definition.location) {
|
193
|
-
description.location = definition.location;
|
194
|
-
}
|
195
|
-
if (description.key.length === 1) {
|
196
|
-
description.text = description.key;
|
197
|
-
}
|
198
|
-
if (definition.text) {
|
199
|
-
description.text = definition.text;
|
200
|
-
}
|
201
|
-
if (shift && definition.shiftText) {
|
202
|
-
description.text = definition.shiftText;
|
203
|
-
}
|
204
|
-
if (this._modifiers & ~8) {
|
205
|
-
description.text = "";
|
206
|
-
}
|
207
|
-
return description;
|
208
|
-
};
|
209
|
-
|
210
|
-
// src/chrome-extension/dynamic-scripts.ts
|
211
|
-
import fs from "fs";
|
212
|
-
import { ifInBrowser } from "misoai-shared/utils";
|
213
|
-
var scriptFileContentCache = null;
|
214
|
-
var getHtmlElementScript = async () => {
|
215
|
-
const scriptFileToRetrieve = chrome.runtime.getURL("scripts/htmlElement.js");
|
216
|
-
if (scriptFileContentCache)
|
217
|
-
return scriptFileContentCache;
|
218
|
-
if (ifInBrowser) {
|
219
|
-
const script = await fetch(scriptFileToRetrieve);
|
220
|
-
scriptFileContentCache = await script.text();
|
221
|
-
return scriptFileContentCache;
|
222
|
-
}
|
223
|
-
return fs.readFileSync(scriptFileToRetrieve, "utf8");
|
224
|
-
};
|
225
|
-
var waterFlowScriptFileContentCache = null;
|
226
|
-
var injectWaterFlowAnimation = async () => {
|
227
|
-
const waterFlowScriptFileToRetrieve = chrome.runtime.getURL(
|
228
|
-
"scripts/water-flow.js"
|
229
|
-
);
|
230
|
-
if (waterFlowScriptFileContentCache)
|
231
|
-
return waterFlowScriptFileContentCache;
|
232
|
-
if (ifInBrowser) {
|
233
|
-
const script = await fetch(waterFlowScriptFileToRetrieve);
|
234
|
-
waterFlowScriptFileContentCache = await script.text();
|
235
|
-
return waterFlowScriptFileContentCache;
|
236
|
-
}
|
237
|
-
return fs.readFileSync(waterFlowScriptFileToRetrieve, "utf8");
|
238
|
-
};
|
239
|
-
var stopWaterFlowScriptFileContentCache = null;
|
240
|
-
var injectStopWaterFlowAnimation = async () => {
|
241
|
-
const stopWaterFlowScriptFileToRetrieve = chrome.runtime.getURL(
|
242
|
-
"scripts/stop-water-flow.js"
|
243
|
-
);
|
244
|
-
if (stopWaterFlowScriptFileContentCache)
|
245
|
-
return stopWaterFlowScriptFileContentCache;
|
246
|
-
if (ifInBrowser) {
|
247
|
-
const script = await fetch(stopWaterFlowScriptFileToRetrieve);
|
248
|
-
stopWaterFlowScriptFileContentCache = await script.text();
|
249
|
-
return stopWaterFlowScriptFileContentCache;
|
250
|
-
}
|
251
|
-
return fs.readFileSync(stopWaterFlowScriptFileToRetrieve, "utf8");
|
252
|
-
};
|
253
|
-
|
254
|
-
// src/chrome-extension/page.ts
|
255
|
-
function sleep(ms) {
|
256
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
257
|
-
}
|
258
|
-
var ChromeExtensionProxyPage = class {
|
259
|
-
constructor(forceSameTabNavigation) {
|
260
|
-
this.pageType = "chrome-extension-proxy";
|
261
|
-
this.version = "1.0.0";
|
262
|
-
this.activeTabId = null;
|
263
|
-
this.tabIdOfDebuggerAttached = null;
|
264
|
-
this.attachingDebugger = null;
|
265
|
-
this.destroyed = false;
|
266
|
-
this.latestMouseX = 100;
|
267
|
-
this.latestMouseY = 100;
|
268
|
-
this.mouse = {
|
269
|
-
click: async (x, y) => {
|
270
|
-
await this.mouse.move(x, y);
|
271
|
-
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
272
|
-
type: "mousePressed",
|
273
|
-
x,
|
274
|
-
y,
|
275
|
-
button: "left",
|
276
|
-
clickCount: 1
|
277
|
-
});
|
278
|
-
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
279
|
-
type: "mouseReleased",
|
280
|
-
x,
|
281
|
-
y,
|
282
|
-
button: "left",
|
283
|
-
clickCount: 1
|
284
|
-
});
|
285
|
-
},
|
286
|
-
wheel: async (deltaX, deltaY, startX, startY) => {
|
287
|
-
const finalX = startX || this.latestMouseX;
|
288
|
-
const finalY = startY || this.latestMouseY;
|
289
|
-
await this.showMousePointer(finalX, finalY);
|
290
|
-
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
291
|
-
type: "mouseWheel",
|
292
|
-
x: finalX,
|
293
|
-
y: finalY,
|
294
|
-
deltaX,
|
295
|
-
deltaY
|
296
|
-
});
|
297
|
-
this.latestMouseX = finalX;
|
298
|
-
this.latestMouseY = finalY;
|
299
|
-
},
|
300
|
-
move: async (x, y) => {
|
301
|
-
await this.showMousePointer(x, y);
|
302
|
-
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
303
|
-
type: "mouseMoved",
|
304
|
-
x,
|
305
|
-
y
|
306
|
-
});
|
307
|
-
this.latestMouseX = x;
|
308
|
-
this.latestMouseY = y;
|
309
|
-
},
|
310
|
-
drag: async (from, to) => {
|
311
|
-
await this.mouse.move(from.x, from.y);
|
312
|
-
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
313
|
-
type: "mousePressed",
|
314
|
-
x: from.x,
|
315
|
-
y: from.y,
|
316
|
-
button: "left",
|
317
|
-
clickCount: 1
|
318
|
-
});
|
319
|
-
await this.mouse.move(to.x, to.y);
|
320
|
-
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
321
|
-
type: "mouseReleased",
|
322
|
-
x: to.x,
|
323
|
-
y: to.y,
|
324
|
-
button: "left",
|
325
|
-
clickCount: 1
|
326
|
-
});
|
327
|
-
}
|
328
|
-
};
|
329
|
-
this.keyboard = {
|
330
|
-
type: async (text) => {
|
331
|
-
const cdpKeyboard = new CdpKeyboard({
|
332
|
-
send: this.sendCommandToDebugger.bind(this)
|
333
|
-
});
|
334
|
-
await cdpKeyboard.type(text, { delay: 0 });
|
335
|
-
},
|
336
|
-
press: async (action) => {
|
337
|
-
const cdpKeyboard = new CdpKeyboard({
|
338
|
-
send: this.sendCommandToDebugger.bind(this)
|
339
|
-
});
|
340
|
-
const keys = Array.isArray(action) ? action : [action];
|
341
|
-
for (const k of keys) {
|
342
|
-
const commands = k.command ? [k.command] : [];
|
343
|
-
await cdpKeyboard.down(k.key, { commands });
|
344
|
-
}
|
345
|
-
for (const k of [...keys].reverse()) {
|
346
|
-
await cdpKeyboard.up(k.key);
|
347
|
-
}
|
348
|
-
}
|
349
|
-
};
|
350
|
-
this.forceSameTabNavigation = forceSameTabNavigation;
|
351
|
-
}
|
352
|
-
async setActiveTabId(tabId) {
|
353
|
-
if (this.activeTabId) {
|
354
|
-
throw new Error(
|
355
|
-
`Active tab id is already set, which is ${this.activeTabId}, cannot set it to ${tabId}`
|
356
|
-
);
|
357
|
-
}
|
358
|
-
await chrome.tabs.update(tabId, { active: true });
|
359
|
-
this.activeTabId = tabId;
|
360
|
-
}
|
361
|
-
async getActiveTabId() {
|
362
|
-
return this.activeTabId;
|
363
|
-
}
|
364
|
-
/**
|
365
|
-
* Get a list of current tabs
|
366
|
-
* @returns {Promise<Array<{id: number, title: string, url: string}>>}
|
367
|
-
*/
|
368
|
-
async getBrowserTabList() {
|
369
|
-
const tabs = await chrome.tabs.query({ currentWindow: true });
|
370
|
-
return tabs.map((tab) => ({
|
371
|
-
id: `${tab.id}`,
|
372
|
-
title: tab.title,
|
373
|
-
url: tab.url,
|
374
|
-
currentActiveTab: tab.active
|
375
|
-
})).filter((tab) => tab.id && tab.title && tab.url);
|
376
|
-
}
|
377
|
-
async getTabIdOrConnectToCurrentTab() {
|
378
|
-
if (this.activeTabId) {
|
379
|
-
return this.activeTabId;
|
380
|
-
}
|
381
|
-
const tabId = await chrome.tabs.query({ active: true, currentWindow: true }).then((tabs) => tabs[0]?.id);
|
382
|
-
this.activeTabId = tabId || 0;
|
383
|
-
return this.activeTabId;
|
384
|
-
}
|
385
|
-
async attachDebugger() {
|
386
|
-
assert2(!this.destroyed, "Page is destroyed");
|
387
|
-
if (this.attachingDebugger) {
|
388
|
-
await this.attachingDebugger;
|
389
|
-
return;
|
390
|
-
}
|
391
|
-
this.attachingDebugger = (async () => {
|
392
|
-
const url = await this.url();
|
393
|
-
let error = null;
|
394
|
-
if (url.startsWith("chrome://")) {
|
395
|
-
throw new Error(
|
396
|
-
"Cannot attach debugger to chrome:// pages, please use Midscene in a normal page with http://, https:// or file://"
|
397
|
-
);
|
398
|
-
}
|
399
|
-
try {
|
400
|
-
const currentTabId = await this.getTabIdOrConnectToCurrentTab();
|
401
|
-
if (this.tabIdOfDebuggerAttached === currentTabId) {
|
402
|
-
return;
|
403
|
-
}
|
404
|
-
if (this.tabIdOfDebuggerAttached && this.tabIdOfDebuggerAttached !== currentTabId) {
|
405
|
-
console.log(
|
406
|
-
"detach the previous tab",
|
407
|
-
this.tabIdOfDebuggerAttached,
|
408
|
-
"->",
|
409
|
-
currentTabId
|
410
|
-
);
|
411
|
-
try {
|
412
|
-
await this.detachDebugger(this.tabIdOfDebuggerAttached);
|
413
|
-
} catch (error2) {
|
414
|
-
console.error("Failed to detach debugger", error2);
|
415
|
-
}
|
416
|
-
}
|
417
|
-
console.log("attaching debugger", currentTabId);
|
418
|
-
await chrome.debugger.attach({ tabId: currentTabId }, "1.3");
|
419
|
-
await sleep(500);
|
420
|
-
this.tabIdOfDebuggerAttached = currentTabId;
|
421
|
-
await this.enableWaterFlowAnimation();
|
422
|
-
} catch (e) {
|
423
|
-
console.error("Failed to attach debugger", e);
|
424
|
-
error = e;
|
425
|
-
} finally {
|
426
|
-
this.attachingDebugger = null;
|
427
|
-
}
|
428
|
-
if (error) {
|
429
|
-
throw error;
|
430
|
-
}
|
431
|
-
})();
|
432
|
-
await this.attachingDebugger;
|
433
|
-
}
|
434
|
-
async showMousePointer(x, y) {
|
435
|
-
const pointerScript = `(() => {
|
436
|
-
if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
|
437
|
-
window.midsceneWaterFlowAnimation.enable();
|
438
|
-
window.midsceneWaterFlowAnimation.showMousePointer(${x}, ${y});
|
439
|
-
} else {
|
440
|
-
console.log('midsceneWaterFlowAnimation is not defined');
|
441
|
-
}
|
442
|
-
})()`;
|
443
|
-
await this.sendCommandToDebugger("Runtime.evaluate", {
|
444
|
-
expression: `${pointerScript}`
|
445
|
-
});
|
446
|
-
}
|
447
|
-
async hideMousePointer() {
|
448
|
-
await this.sendCommandToDebugger("Runtime.evaluate", {
|
449
|
-
expression: `(() => {
|
450
|
-
if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
|
451
|
-
window.midsceneWaterFlowAnimation.hideMousePointer();
|
452
|
-
}
|
453
|
-
})()`
|
454
|
-
});
|
455
|
-
}
|
456
|
-
async detachDebugger(tabId) {
|
457
|
-
const tabIdToDetach = tabId || this.tabIdOfDebuggerAttached;
|
458
|
-
console.log("detaching debugger", tabIdToDetach);
|
459
|
-
if (!tabIdToDetach) {
|
460
|
-
console.warn("No tab id to detach");
|
461
|
-
return;
|
462
|
-
}
|
463
|
-
try {
|
464
|
-
await this.disableWaterFlowAnimation(tabIdToDetach);
|
465
|
-
await sleep(200);
|
466
|
-
} catch (error) {
|
467
|
-
console.warn("Failed to disable water flow animation", error);
|
468
|
-
}
|
469
|
-
try {
|
470
|
-
await chrome.debugger.detach({ tabId: tabIdToDetach });
|
471
|
-
} catch (error) {
|
472
|
-
console.warn("Failed to detach debugger", error);
|
473
|
-
}
|
474
|
-
this.tabIdOfDebuggerAttached = null;
|
475
|
-
}
|
476
|
-
async enableWaterFlowAnimation() {
|
477
|
-
if (this.forceSameTabNavigation) {
|
478
|
-
await chrome.debugger.sendCommand(
|
479
|
-
{ tabId: this.tabIdOfDebuggerAttached },
|
480
|
-
"Runtime.evaluate",
|
481
|
-
{
|
482
|
-
expression: limitOpenNewTabScript
|
483
|
-
}
|
484
|
-
);
|
485
|
-
}
|
486
|
-
const script = await injectWaterFlowAnimation();
|
487
|
-
await chrome.debugger.sendCommand(
|
488
|
-
{ tabId: this.tabIdOfDebuggerAttached },
|
489
|
-
"Runtime.evaluate",
|
490
|
-
{
|
491
|
-
expression: script
|
492
|
-
}
|
493
|
-
);
|
494
|
-
}
|
495
|
-
async disableWaterFlowAnimation(tabId) {
|
496
|
-
const script = await injectStopWaterFlowAnimation();
|
497
|
-
await chrome.debugger.sendCommand({ tabId }, "Runtime.evaluate", {
|
498
|
-
expression: script
|
499
|
-
});
|
500
|
-
}
|
501
|
-
async sendCommandToDebugger(command, params) {
|
502
|
-
await this.attachDebugger();
|
503
|
-
assert2(this.tabIdOfDebuggerAttached, "Debugger is not attached");
|
504
|
-
this.enableWaterFlowAnimation();
|
505
|
-
return await chrome.debugger.sendCommand(
|
506
|
-
{ tabId: this.tabIdOfDebuggerAttached },
|
507
|
-
command,
|
508
|
-
params
|
509
|
-
);
|
510
|
-
}
|
511
|
-
async getPageContentByCDP() {
|
512
|
-
const script = await getHtmlElementScript();
|
513
|
-
await this.sendCommandToDebugger("Runtime.evaluate", {
|
514
|
-
expression: script
|
515
|
-
});
|
516
|
-
const expression = () => {
|
517
|
-
return {
|
518
|
-
tree: window.midscene_element_inspector.webExtractNodeTree(),
|
519
|
-
size: {
|
520
|
-
width: document.documentElement.clientWidth,
|
521
|
-
height: document.documentElement.clientHeight,
|
522
|
-
dpr: window.devicePixelRatio
|
523
|
-
}
|
524
|
-
};
|
525
|
-
};
|
526
|
-
const returnValue = await this.sendCommandToDebugger("Runtime.evaluate", {
|
527
|
-
expression: `(${expression.toString()})()`,
|
528
|
-
returnByValue: true
|
529
|
-
});
|
530
|
-
if (!returnValue.result.value) {
|
531
|
-
const errorDescription = returnValue.exceptionDetails?.exception?.description || "";
|
532
|
-
if (!errorDescription) {
|
533
|
-
console.error("returnValue from cdp", returnValue);
|
534
|
-
}
|
535
|
-
throw new Error(
|
536
|
-
`Failed to get page content from page, error: ${errorDescription}`
|
537
|
-
);
|
538
|
-
}
|
539
|
-
return returnValue.result.value;
|
540
|
-
}
|
541
|
-
async evaluateJavaScript(script) {
|
542
|
-
return this.sendCommandToDebugger("Runtime.evaluate", {
|
543
|
-
expression: script
|
544
|
-
});
|
545
|
-
}
|
546
|
-
// current implementation is wait until domReadyState is complete
|
547
|
-
async waitUntilNetworkIdle() {
|
548
|
-
const timeout = 1e4;
|
549
|
-
const startTime = Date.now();
|
550
|
-
let lastReadyState = "";
|
551
|
-
while (Date.now() - startTime < timeout) {
|
552
|
-
const result = await this.sendCommandToDebugger("Runtime.evaluate", {
|
553
|
-
expression: "document.readyState"
|
554
|
-
});
|
555
|
-
lastReadyState = result.result.value;
|
556
|
-
if (lastReadyState === "complete") {
|
557
|
-
await new Promise((resolve) => setTimeout(resolve, 300));
|
558
|
-
return;
|
559
|
-
}
|
560
|
-
await new Promise((resolve) => setTimeout(resolve, 300));
|
561
|
-
}
|
562
|
-
throw new Error(
|
563
|
-
`Failed to wait until network idle, last readyState: ${lastReadyState}`
|
564
|
-
);
|
565
|
-
}
|
566
|
-
async getElementsInfo() {
|
567
|
-
const tree = await this.getElementsNodeTree();
|
568
|
-
return treeToList(tree);
|
569
|
-
}
|
570
|
-
async getElementsNodeTree() {
|
571
|
-
await this.hideMousePointer();
|
572
|
-
const content = await this.getPageContentByCDP();
|
573
|
-
if (content?.size) {
|
574
|
-
this.viewportSize = content.size;
|
575
|
-
}
|
576
|
-
return content?.tree || { node: null, children: [] };
|
577
|
-
}
|
578
|
-
async size() {
|
579
|
-
if (this.viewportSize)
|
580
|
-
return this.viewportSize;
|
581
|
-
const content = await this.getPageContentByCDP();
|
582
|
-
return content.size;
|
583
|
-
}
|
584
|
-
async screenshotBase64() {
|
585
|
-
await this.hideMousePointer();
|
586
|
-
const base64 = await this.sendCommandToDebugger("Page.captureScreenshot", {
|
587
|
-
format: "jpeg",
|
588
|
-
quality: 90
|
589
|
-
});
|
590
|
-
return `data:image/jpeg;base64,${base64.data}`;
|
591
|
-
}
|
592
|
-
async url() {
|
593
|
-
const tabId = await this.getTabIdOrConnectToCurrentTab();
|
594
|
-
const url = await chrome.tabs.get(tabId).then((tab) => tab.url);
|
595
|
-
return url || "";
|
596
|
-
}
|
597
|
-
async scrollUntilTop(startingPoint) {
|
598
|
-
if (startingPoint) {
|
599
|
-
await this.mouse.move(startingPoint.left, startingPoint.top);
|
600
|
-
}
|
601
|
-
return this.mouse.wheel(0, -9999999);
|
602
|
-
}
|
603
|
-
async scrollUntilBottom(startingPoint) {
|
604
|
-
if (startingPoint) {
|
605
|
-
await this.mouse.move(startingPoint.left, startingPoint.top);
|
606
|
-
}
|
607
|
-
return this.mouse.wheel(0, 9999999);
|
608
|
-
}
|
609
|
-
async scrollUntilLeft(startingPoint) {
|
610
|
-
if (startingPoint) {
|
611
|
-
await this.mouse.move(startingPoint.left, startingPoint.top);
|
612
|
-
}
|
613
|
-
return this.mouse.wheel(-9999999, 0);
|
614
|
-
}
|
615
|
-
async scrollUntilRight(startingPoint) {
|
616
|
-
if (startingPoint) {
|
617
|
-
await this.mouse.move(startingPoint.left, startingPoint.top);
|
618
|
-
}
|
619
|
-
return this.mouse.wheel(9999999, 0);
|
620
|
-
}
|
621
|
-
async scrollUp(distance, startingPoint) {
|
622
|
-
const { height } = await this.size();
|
623
|
-
const scrollDistance = distance || height * 0.7;
|
624
|
-
return this.mouse.wheel(
|
625
|
-
0,
|
626
|
-
-scrollDistance,
|
627
|
-
startingPoint?.left,
|
628
|
-
startingPoint?.top
|
629
|
-
);
|
630
|
-
}
|
631
|
-
async scrollDown(distance, startingPoint) {
|
632
|
-
const { height } = await this.size();
|
633
|
-
const scrollDistance = distance || height * 0.7;
|
634
|
-
return this.mouse.wheel(
|
635
|
-
0,
|
636
|
-
scrollDistance,
|
637
|
-
startingPoint?.left,
|
638
|
-
startingPoint?.top
|
639
|
-
);
|
640
|
-
}
|
641
|
-
async scrollLeft(distance, startingPoint) {
|
642
|
-
const { width } = await this.size();
|
643
|
-
const scrollDistance = distance || width * 0.7;
|
644
|
-
return this.mouse.wheel(
|
645
|
-
-scrollDistance,
|
646
|
-
0,
|
647
|
-
startingPoint?.left,
|
648
|
-
startingPoint?.top
|
649
|
-
);
|
650
|
-
}
|
651
|
-
async scrollRight(distance, startingPoint) {
|
652
|
-
const { width } = await this.size();
|
653
|
-
const scrollDistance = distance || width * 0.7;
|
654
|
-
return this.mouse.wheel(
|
655
|
-
scrollDistance,
|
656
|
-
0,
|
657
|
-
startingPoint?.left,
|
658
|
-
startingPoint?.top
|
659
|
-
);
|
660
|
-
}
|
661
|
-
async clearInput(element) {
|
662
|
-
if (!element) {
|
663
|
-
console.warn("No element to clear input");
|
664
|
-
return;
|
665
|
-
}
|
666
|
-
await this.mouse.click(element.center[0], element.center[1]);
|
667
|
-
await this.sendCommandToDebugger("Input.dispatchKeyEvent", {
|
668
|
-
type: "keyDown",
|
669
|
-
commands: ["selectAll"]
|
670
|
-
});
|
671
|
-
await this.sendCommandToDebugger("Input.dispatchKeyEvent", {
|
672
|
-
type: "keyUp",
|
673
|
-
commands: ["selectAll"]
|
674
|
-
});
|
675
|
-
await sleep(100);
|
676
|
-
await this.keyboard.press({
|
677
|
-
key: "Backspace"
|
678
|
-
});
|
679
|
-
}
|
680
|
-
async destroy() {
|
681
|
-
this.activeTabId = null;
|
682
|
-
await this.detachDebugger();
|
683
|
-
this.destroyed = true;
|
684
|
-
}
|
685
|
-
};
|
686
|
-
|
687
|
-
// src/bridge-mode/common.ts
|
688
|
-
var DefaultBridgeServerPort = 3766;
|
689
|
-
var DefaultLocalEndpoint = `http://127.0.0.1:${DefaultBridgeServerPort}`;
|
690
|
-
|
691
|
-
// src/bridge-mode/io-client.ts
|
692
|
-
import { assert as assert3 } from "misoai-shared/utils";
|
693
|
-
import { io as ClientIO } from "socket.io-client";
|
694
|
-
var BridgeClient = class {
|
695
|
-
constructor(endpoint, onBridgeCall, onDisconnect) {
|
696
|
-
this.endpoint = endpoint;
|
697
|
-
this.onBridgeCall = onBridgeCall;
|
698
|
-
this.onDisconnect = onDisconnect;
|
699
|
-
this.socket = null;
|
700
|
-
this.serverVersion = null;
|
701
|
-
}
|
702
|
-
async connect() {
|
703
|
-
return new Promise((resolve, reject) => {
|
704
|
-
this.socket = ClientIO(this.endpoint, {
|
705
|
-
reconnection: false,
|
706
|
-
query: {
|
707
|
-
version: "1.0.0"
|
708
|
-
}
|
709
|
-
});
|
710
|
-
const timeout = setTimeout(() => {
|
711
|
-
try {
|
712
|
-
this.socket?.offAny();
|
713
|
-
this.socket?.close();
|
714
|
-
} catch (e) {
|
715
|
-
console.warn("got error when offing socket", e);
|
716
|
-
}
|
717
|
-
this.socket = null;
|
718
|
-
reject(new Error("failed to connect to bridge server after timeout"));
|
719
|
-
}, 1 * 1e3);
|
720
|
-
this.socket.on("disconnect", (reason) => {
|
721
|
-
this.socket = null;
|
722
|
-
this.onDisconnect?.();
|
723
|
-
});
|
724
|
-
this.socket.on("connect_error", (e) => {
|
725
|
-
console.error("bridge-connect-error", e);
|
726
|
-
reject(new Error(e || "bridge connect error"));
|
727
|
-
});
|
728
|
-
this.socket.on(
|
729
|
-
"bridge-connected" /* Connected */,
|
730
|
-
(payload) => {
|
731
|
-
clearTimeout(timeout);
|
732
|
-
this.serverVersion = payload?.version || "unknown";
|
733
|
-
resolve(this.socket);
|
734
|
-
}
|
735
|
-
);
|
736
|
-
this.socket.on("bridge-refused" /* Refused */, (e) => {
|
737
|
-
console.error("bridge-refused", e);
|
738
|
-
try {
|
739
|
-
this.socket?.disconnect();
|
740
|
-
} catch (e2) {
|
741
|
-
}
|
742
|
-
reject(new Error(e || "bridge refused"));
|
743
|
-
});
|
744
|
-
this.socket.on("bridge-call" /* Call */, (call) => {
|
745
|
-
const id = call.id;
|
746
|
-
assert3(typeof id !== "undefined", "call id is required");
|
747
|
-
(async () => {
|
748
|
-
let response;
|
749
|
-
try {
|
750
|
-
response = await this.onBridgeCall(call.method, call.args);
|
751
|
-
} catch (e) {
|
752
|
-
const errorContent = `Error from bridge client when calling, method: ${call.method}, args: ${call.args}, error: ${e?.message || e}
|
753
|
-
${e?.stack || ""}`;
|
754
|
-
console.error(errorContent);
|
755
|
-
return this.socket?.emit("bridge-call-response" /* CallResponse */, {
|
756
|
-
id,
|
757
|
-
error: errorContent
|
758
|
-
});
|
759
|
-
}
|
760
|
-
this.socket?.emit("bridge-call-response" /* CallResponse */, {
|
761
|
-
id,
|
762
|
-
response
|
763
|
-
});
|
764
|
-
})();
|
765
|
-
});
|
766
|
-
});
|
767
|
-
}
|
768
|
-
disconnect() {
|
769
|
-
this.socket?.disconnect();
|
770
|
-
this.socket = null;
|
771
|
-
}
|
772
|
-
};
|
773
|
-
|
774
|
-
// src/bridge-mode/page-browser-side.ts
|
775
|
-
var ExtensionBridgePageBrowserSide = class extends ChromeExtensionProxyPage {
|
776
|
-
constructor(onDisconnect = () => {
|
777
|
-
}, onLogMessage = () => {
|
778
|
-
}, forceSameTabNavigation = true) {
|
779
|
-
super(forceSameTabNavigation);
|
780
|
-
this.onDisconnect = onDisconnect;
|
781
|
-
this.onLogMessage = onLogMessage;
|
782
|
-
this.bridgeClient = null;
|
783
|
-
this.newlyCreatedTabIds = [];
|
784
|
-
}
|
785
|
-
async setupBridgeClient() {
|
786
|
-
this.bridgeClient = new BridgeClient(
|
787
|
-
`ws://localhost:${DefaultBridgeServerPort}`,
|
788
|
-
async (method, args) => {
|
789
|
-
console.log("bridge call from cli side", method, args);
|
790
|
-
if (method === "connectNewTabWithUrl" /* ConnectNewTabWithUrl */) {
|
791
|
-
return this.connectNewTabWithUrl.apply(
|
792
|
-
this,
|
793
|
-
args
|
794
|
-
);
|
795
|
-
}
|
796
|
-
if (method === "getBrowserTabList" /* GetBrowserTabList */) {
|
797
|
-
return this.getBrowserTabList.apply(this, args);
|
798
|
-
}
|
799
|
-
if (method === "setActiveTabId" /* SetActiveTabId */) {
|
800
|
-
return this.setActiveTabId.apply(this, args);
|
801
|
-
}
|
802
|
-
if (method === "connectCurrentTab" /* ConnectCurrentTab */) {
|
803
|
-
return this.connectCurrentTab.apply(this, args);
|
804
|
-
}
|
805
|
-
if (method === "bridge-update-agent-status" /* UpdateAgentStatus */) {
|
806
|
-
return this.onLogMessage(args[0], "status");
|
807
|
-
}
|
808
|
-
const tabId = await this.getActiveTabId();
|
809
|
-
if (!tabId || tabId === 0) {
|
810
|
-
throw new Error("no tab is connected");
|
811
|
-
}
|
812
|
-
if (method.startsWith("mouse." /* PREFIX */)) {
|
813
|
-
const actionName = method.split(".")[1];
|
814
|
-
if (actionName === "drag") {
|
815
|
-
return this.mouse[actionName].apply(this.mouse, args);
|
816
|
-
}
|
817
|
-
return this.mouse[actionName].apply(this.mouse, args);
|
818
|
-
}
|
819
|
-
if (method.startsWith("keyboard." /* PREFIX */)) {
|
820
|
-
const actionName = method.split(".")[1];
|
821
|
-
if (actionName === "press") {
|
822
|
-
return this.keyboard[actionName].apply(this.keyboard, args);
|
823
|
-
}
|
824
|
-
return this.keyboard[actionName].apply(this.keyboard, args);
|
825
|
-
}
|
826
|
-
try {
|
827
|
-
const result = await this[method](
|
828
|
-
...args
|
829
|
-
);
|
830
|
-
return result;
|
831
|
-
} catch (e) {
|
832
|
-
const errorMessage = e instanceof Error ? e.message : "Unknown error";
|
833
|
-
console.error("error calling method", method, args, e);
|
834
|
-
this.onLogMessage(
|
835
|
-
`Error calling method: ${method}, ${errorMessage}`,
|
836
|
-
"log"
|
837
|
-
);
|
838
|
-
throw new Error(errorMessage, { cause: e });
|
839
|
-
}
|
840
|
-
},
|
841
|
-
// on disconnect
|
842
|
-
() => {
|
843
|
-
return this.destroy();
|
844
|
-
}
|
845
|
-
);
|
846
|
-
await this.bridgeClient.connect();
|
847
|
-
this.onLogMessage(
|
848
|
-
`Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"1.0.0"}`,
|
849
|
-
"log"
|
850
|
-
);
|
851
|
-
}
|
852
|
-
async connect() {
|
853
|
-
return await this.setupBridgeClient();
|
854
|
-
}
|
855
|
-
async connectNewTabWithUrl(url, options = {
|
856
|
-
forceSameTabNavigation: true
|
857
|
-
}) {
|
858
|
-
const tab = await chrome.tabs.create({ url });
|
859
|
-
const tabId = tab.id;
|
860
|
-
assert4(tabId, "failed to get tabId after creating a new tab");
|
861
|
-
this.onLogMessage(`Creating new tab: ${url}`, "log");
|
862
|
-
this.newlyCreatedTabIds.push(tabId);
|
863
|
-
if (options?.forceSameTabNavigation) {
|
864
|
-
this.forceSameTabNavigation = true;
|
865
|
-
}
|
866
|
-
await this.setActiveTabId(tabId);
|
867
|
-
}
|
868
|
-
async connectCurrentTab(options = {
|
869
|
-
forceSameTabNavigation: true
|
870
|
-
}) {
|
871
|
-
const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
|
872
|
-
const tabId = tabs[0]?.id;
|
873
|
-
assert4(tabId, "failed to get tabId");
|
874
|
-
this.onLogMessage(`Connected to current tab: ${tabs[0]?.url}`, "log");
|
875
|
-
if (options?.forceSameTabNavigation) {
|
876
|
-
this.forceSameTabNavigation = true;
|
877
|
-
}
|
878
|
-
await this.setActiveTabId(tabId);
|
879
|
-
}
|
880
|
-
async setDestroyOptions(options) {
|
881
|
-
this.destroyOptions = options;
|
882
|
-
}
|
883
|
-
async destroy() {
|
884
|
-
if (this.destroyOptions?.closeTab && this.newlyCreatedTabIds.length > 0) {
|
885
|
-
this.onLogMessage("Closing all newly created tabs by bridge...", "log");
|
886
|
-
for (const tabId of this.newlyCreatedTabIds) {
|
887
|
-
await chrome.tabs.remove(tabId);
|
888
|
-
}
|
889
|
-
this.newlyCreatedTabIds = [];
|
890
|
-
}
|
891
|
-
await super.destroy();
|
892
|
-
if (this.bridgeClient) {
|
893
|
-
this.bridgeClient.disconnect();
|
894
|
-
this.bridgeClient = null;
|
895
|
-
this.onDisconnect();
|
896
|
-
}
|
897
|
-
}
|
898
|
-
};
|
899
|
-
export {
|
900
|
-
ExtensionBridgePageBrowserSide
|
901
|
-
};
|
902
|
-
/**
|
903
|
-
* @license
|
904
|
-
* Copyright 2017 Google Inc.
|
905
|
-
* SPDX-License-Identifier: Apache-2.0
|
906
|
-
*/
|
907
|
-
|
908
|
-
//# sourceMappingURL=bridge-mode-browser.js.map
|