@glittering0717/ai-browser 0.0.0-stage → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +784 -2
- package/bin/ai-browser.cjs +78 -0
- package/examples/AGENT_INSTRUCTIONS.md +166 -0
- package/examples/mcp-config.json +12 -0
- package/llms-install.md +158 -0
- package/package.json +78 -4
- package/scripts/electron_resolve.cjs +241 -0
- package/scripts/start.cjs +60 -0
- package/src/main/axExtractor.js +436 -0
- package/src/main/canvas_hook_source.js +654 -0
- package/src/main/canvas_monitor.js +269 -0
- package/src/main/capability_probe.js +415 -0
- package/src/main/download_monitor.js +137 -0
- package/src/main/index.js +359 -0
- package/src/main/mcp_server.js +408 -0
- package/src/main/mcp_tools.js +289 -0
- package/src/main/mcp_ws.js +178 -0
- package/src/main/network_monitor.js +805 -0
- package/src/main/page_manager.js +2743 -0
- package/src/main/snapshot_diff.js +371 -0
- package/src/main/ws_server.js +440 -0
- package/src/preload/actions.cjs +190 -0
- package/src/preload/bridge.cjs +75 -0
- package/src/preload/extractor.cjs +936 -0
- package/src/preload/watcher.cjs +241 -0
- package/src/renderer/tab_bar.html +115 -0
- package/src/shared/config.js +32 -0
- package/src/shared/deadline.js +89 -0
- package/src/shared/guards.js +17 -0
- package/src/shared/protocol.js +116 -0
- package/src/shared/value_contract.cjs +132 -0
- package/src/shared/version.js +5 -0
- package/src/shared/watchdog.js +57 -0
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
// main/ws_server.js — WebSocket JSON-RPC server v2 (multi-tab)
|
|
2
|
+
import { WebSocketServer } from 'ws';
|
|
3
|
+
import { evaluateGuardError } from '../shared/guards.js';
|
|
4
|
+
import { withDeadline, actionDeadlineMs } from '../shared/deadline.js';
|
|
5
|
+
import { config } from '../shared/config.js';
|
|
6
|
+
|
|
7
|
+
// Each startWSServer() owns a private WebSocketServer. Keeping it local (not a
|
|
8
|
+
// module global) means multiple servers can coexist — required by tests that
|
|
9
|
+
// spin up independent servers on ephemeral ports without one close() tearing
|
|
10
|
+
// down another's.
|
|
11
|
+
// Helper: search tree by field value
|
|
12
|
+
function findInTree(node, field, value) {
|
|
13
|
+
if (!node) return null;
|
|
14
|
+
if (node[field] && node[field].indexOf && node[field].indexOf(value) >= 0) return node;
|
|
15
|
+
if (node.children) for (const c of node.children) { const f = findInTree(c, field, value); if (f) return f; }
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
// wait{button_enabled} 的目标解析。
|
|
19
|
+
//
|
|
20
|
+
// 老实现直接 findInTree(tree,'label',target) —— 按 **label 子串** 匹配。可整个 API
|
|
21
|
+
// 里 target 一律指元素 id(get_tree 给的那个),agent 拿 id 来等就永远等不到:
|
|
22
|
+
// 实测传 'lateBtn'(元素 id)8s 超时后 satisfied=false,而传按钮文案 3ms 就满足
|
|
23
|
+
// (e2e/audit_ws.cjs AU-E1/AU-E2)。agent 会误判成"按钮一直禁用"。
|
|
24
|
+
//
|
|
25
|
+
// 现在先按 id 精确匹配,命中不了再退化成 label 子串(老用法仍然成立)。
|
|
26
|
+
// 顺序不能反:label 是子串匹配,'go' 会命中任何文案里带 go 的节点。
|
|
27
|
+
function findWaitTarget(tree, target) {
|
|
28
|
+
if (!target) return null;
|
|
29
|
+
const byId = findInTree(tree, 'id', String(target));
|
|
30
|
+
if (byId && byId.id === String(target)) return byId;
|
|
31
|
+
const any = findInTree(tree, 'id', String(target));
|
|
32
|
+
if (any) return any;
|
|
33
|
+
return findInTree(tree, 'label', String(target));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function startWSServer(pageManager, port = config.wsPort, onQuit = null) {
|
|
37
|
+
const wss = new WebSocketServer({ port });
|
|
38
|
+
|
|
39
|
+
wss.on('connection', (ws, _req) => {
|
|
40
|
+
const sessionId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
41
|
+
const subscriptions = new Set();
|
|
42
|
+
|
|
43
|
+
pageManager.registerClient(sessionId, ws);
|
|
44
|
+
|
|
45
|
+
ws.on('message', async (raw) => {
|
|
46
|
+
let msg;
|
|
47
|
+
try {
|
|
48
|
+
msg = JSON.parse(raw.toString());
|
|
49
|
+
} catch {
|
|
50
|
+
ws.send(JSON.stringify({ jsonrpc: '2.0', error: { code: -32700, message: 'Parse error' } }));
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Notification (no id)
|
|
55
|
+
if (msg.id === undefined || msg.id === null) {
|
|
56
|
+
if (msg.method === 'ui.subscribe') {
|
|
57
|
+
const events = msg.params?.events || [];
|
|
58
|
+
events.forEach(ev => subscriptions.add(ev));
|
|
59
|
+
pageManager.addSubscription(sessionId, events);
|
|
60
|
+
}
|
|
61
|
+
if (msg.method === 'ui.unsubscribe') {
|
|
62
|
+
const events = msg.params?.events || [];
|
|
63
|
+
events.forEach(ev => subscriptions.delete(ev));
|
|
64
|
+
pageManager.removeSubscription(sessionId, events);
|
|
65
|
+
}
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const { id, method, params = {} } = msg;
|
|
70
|
+
const tabId = params.tab; // optional tab routing
|
|
71
|
+
const send = (payload) => {
|
|
72
|
+
if (ws.readyState === 1) ws.send(JSON.stringify(payload));
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
try {
|
|
76
|
+
switch (method) {
|
|
77
|
+
case 'ui.get_tree': {
|
|
78
|
+
// P0 probe: params.ax routes to the AX read layer (getFullAXTree);
|
|
79
|
+
// default path (preload extractor) is unchanged.
|
|
80
|
+
const result = params.ax
|
|
81
|
+
? await pageManager.getTreeViaAx(tabId, {
|
|
82
|
+
subset: params.subset,
|
|
83
|
+
mode: params.mode,
|
|
84
|
+
focusedOnly: params.focusedOnly,
|
|
85
|
+
})
|
|
86
|
+
: await pageManager.getTree(params.focusedOnly, tabId);
|
|
87
|
+
send({ jsonrpc: '2.0', id, result });
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
case 'ui.act': {
|
|
91
|
+
// RPC 边界上的保底:这一层也套 deadline,与 page_manager 内部那一层
|
|
92
|
+
// 同源(都用 actionDeadlineMs 算预算)。为什么两层都要:
|
|
93
|
+
// - page_manager 那层保护所有内部调用方(capability_probe、直接 API);
|
|
94
|
+
// - 这一层保护"以后新增的页面路径忘了套",并且**可以用 mock PageManager
|
|
95
|
+
// 做单测**(真实 PageManager 依赖 Electron,vitest 里起不来)。
|
|
96
|
+
// 同源意味着两层不会打架:预算一致,谁先到期谁报,语义相同。
|
|
97
|
+
const budget = actionDeadlineMs(params.action, params.params || {});
|
|
98
|
+
try {
|
|
99
|
+
const result = await withDeadline(
|
|
100
|
+
pageManager.executeAction(params.action, params.target, params.params || {}, tabId),
|
|
101
|
+
budget, `action:${params.action}`
|
|
102
|
+
);
|
|
103
|
+
send({ jsonrpc: '2.0', id, result });
|
|
104
|
+
} catch (e) {
|
|
105
|
+
if (e && e.__deadline) {
|
|
106
|
+
send({
|
|
107
|
+
jsonrpc: '2.0', id,
|
|
108
|
+
result: {
|
|
109
|
+
success: false,
|
|
110
|
+
error: `${e.message} — the page's main thread is likely blocked; the action may have partially applied`,
|
|
111
|
+
timed_out: true,
|
|
112
|
+
deadline_ms: budget,
|
|
113
|
+
action: params.action,
|
|
114
|
+
},
|
|
115
|
+
});
|
|
116
|
+
} else {
|
|
117
|
+
throw e;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
case 'ui.navigate': {
|
|
123
|
+
// navigate 现在返回 {ok, error?}:成功形状与旧的 {ok:true} 一致,
|
|
124
|
+
// 失败时多一个原因。此前只有布尔,agent 拿到 ok:false 无从判断
|
|
125
|
+
// 该重试还是换 URL;而"页面永不 did-finish-load"是会真实发生的。
|
|
126
|
+
const r = await pageManager.navigate(params.url, tabId);
|
|
127
|
+
send({ jsonrpc: '2.0', id, result: r && typeof r === 'object' ? r : { ok: !!r } });
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
case 'ui.repaint': {
|
|
131
|
+
// Force the compositor to redraw a tab's BrowserView. Needed because
|
|
132
|
+
// "the window shows a stale page" has NO observable state to detect:
|
|
133
|
+
// the DOM tree / evaluate already report the NEW page while the
|
|
134
|
+
// screen shows the OLD one. So the agent's only handle on it is an
|
|
135
|
+
// explicit "this window looks wrong, redraw it".
|
|
136
|
+
const r = pageManager.repaint(tabId);
|
|
137
|
+
send({ jsonrpc: '2.0', id, result: r });
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
case 'ui.snapshot': {
|
|
141
|
+
const r = await pageManager.snapshot({ js: params.js, label: params.label, tab: tabId });
|
|
142
|
+
send({ jsonrpc: '2.0', id, result: r });
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
case 'ui.diff': {
|
|
146
|
+
const r = await pageManager.diff({
|
|
147
|
+
snapshot: params.snapshot,
|
|
148
|
+
js: params.js,
|
|
149
|
+
rearm: params.rearm,
|
|
150
|
+
forget: params.forget,
|
|
151
|
+
tab: tabId,
|
|
152
|
+
});
|
|
153
|
+
send({ jsonrpc: '2.0', id, result: r });
|
|
154
|
+
break;
|
|
155
|
+
}
|
|
156
|
+
case 'ui.capabilities': {
|
|
157
|
+
// "How should I operate THIS page?" — one call answers it: is the
|
|
158
|
+
// graph DOM or canvas, is the body in an iframe, does the app hang a
|
|
159
|
+
// model off window, which JSON endpoints exist, and therefore
|
|
160
|
+
// whether to go structured-data / semantic-tree / pointer-gestures.
|
|
161
|
+
const r = await pageManager.capabilities({
|
|
162
|
+
tab: tabId,
|
|
163
|
+
include_network: params.include_network,
|
|
164
|
+
network_limit: params.network_limit,
|
|
165
|
+
});
|
|
166
|
+
send({ jsonrpc: '2.0', id, result: r });
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
case 'ui.evaluate': {
|
|
170
|
+
const js = String(params.js ?? '');
|
|
171
|
+
// Guard rails (shared single source) — enforce length + reject
|
|
172
|
+
// obvious Node-exfil patterns. The MCP layer applies the same guard;
|
|
173
|
+
// a direct ws client bypasses MCP, so enforce the same contract here.
|
|
174
|
+
const guardErr = evaluateGuardError(js);
|
|
175
|
+
if (guardErr) {
|
|
176
|
+
send({ jsonrpc: '2.0', id, error: { code: -32602, message: guardErr } });
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
const value = await pageManager.evaluate(js, tabId);
|
|
180
|
+
send({ jsonrpc: '2.0', id, result: { value } });
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
// === Tab management ===
|
|
184
|
+
case 'ui.new_tab': {
|
|
185
|
+
const newTabId = pageManager.newTab(params.url || null);
|
|
186
|
+
send({ jsonrpc: '2.0', id, result: { tab: newTabId } });
|
|
187
|
+
break;
|
|
188
|
+
}
|
|
189
|
+
case 'ui.close_tab': {
|
|
190
|
+
const ok = pageManager.closeTab(params.tab);
|
|
191
|
+
send({ jsonrpc: '2.0', id, result: { ok } });
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
case 'ui.list_tabs': {
|
|
195
|
+
const tabs = pageManager.listTabs();
|
|
196
|
+
send({ jsonrpc: '2.0', id, result: { tabs } });
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
case 'ui.set_active_tab': {
|
|
200
|
+
const ok = pageManager.setActive(params.tab);
|
|
201
|
+
send({ jsonrpc: '2.0', id, result: { ok } });
|
|
202
|
+
break;
|
|
203
|
+
}
|
|
204
|
+
// === Subscribe/unsubscribe (with response) ===
|
|
205
|
+
case 'ui.subscribe': {
|
|
206
|
+
const events = params.events || [];
|
|
207
|
+
events.forEach(ev => subscriptions.add(ev));
|
|
208
|
+
pageManager.addSubscription(sessionId, events);
|
|
209
|
+
send({ jsonrpc: '2.0', id, result: { ok: true } });
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
case 'ui.unsubscribe': {
|
|
213
|
+
const events = params.events || [];
|
|
214
|
+
events.forEach(ev => subscriptions.delete(ev));
|
|
215
|
+
pageManager.removeSubscription(sessionId, events);
|
|
216
|
+
send({ jsonrpc: '2.0', id, result: { ok: true } });
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
// === ui.canvas_* — canvas 绘制调用记录(语义层,非像素) ===
|
|
220
|
+
case 'ui.canvas_configure': {
|
|
221
|
+
// canvasConfigure 是 async(可能触发 hook 安装与按要求的重载),必须 await
|
|
222
|
+
const result = await pageManager.canvasConfigure(tabId, {
|
|
223
|
+
mode: params.mode,
|
|
224
|
+
clear: params.clear === true,
|
|
225
|
+
reload: params.reload === true,
|
|
226
|
+
});
|
|
227
|
+
send({ jsonrpc: '2.0', id, result });
|
|
228
|
+
break;
|
|
229
|
+
}
|
|
230
|
+
case 'ui.canvas_list': {
|
|
231
|
+
const result = pageManager.canvasList(tabId);
|
|
232
|
+
send({ jsonrpc: '2.0', id, result });
|
|
233
|
+
break;
|
|
234
|
+
}
|
|
235
|
+
case 'ui.canvas_read': {
|
|
236
|
+
const result = pageManager.canvasRead(tabId, {
|
|
237
|
+
canvasId: params.canvas_id,
|
|
238
|
+
view: params.view,
|
|
239
|
+
sinceSeq: params.since_seq,
|
|
240
|
+
limit: params.limit,
|
|
241
|
+
});
|
|
242
|
+
send({ jsonrpc: '2.0', id, result });
|
|
243
|
+
break;
|
|
244
|
+
}
|
|
245
|
+
case 'ui.canvas_capture': {
|
|
246
|
+
// 视觉兜底:默认不该被调用。只在语义层读不懂(WebGL / 纯像素 canvas)
|
|
247
|
+
// 时由调用方显式触发;会消耗调用方的视觉 token。
|
|
248
|
+
const result = await pageManager.canvasCapture(tabId, params.canvas_id);
|
|
249
|
+
send({ jsonrpc: '2.0', id, result: result || { error: 'canvas not found or capture failed' } });
|
|
250
|
+
break;
|
|
251
|
+
}
|
|
252
|
+
case 'ui.get_focused': {
|
|
253
|
+
const result = await pageManager.getFocused(tabId);
|
|
254
|
+
send({ jsonrpc: '2.0', id, result });
|
|
255
|
+
break;
|
|
256
|
+
}
|
|
257
|
+
case 'ui.peek': {
|
|
258
|
+
// A (plan §9.5): safely reveal a folded/hoverable affordance and
|
|
259
|
+
// return the AX diff. Non-committing (hover only).
|
|
260
|
+
const result = await pageManager.peek(tabId, params.target, params);
|
|
261
|
+
send({ jsonrpc: '2.0', id, result });
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
// === ui.wait — poll until condition or timeout ===
|
|
265
|
+
case 'ui.wait': {
|
|
266
|
+
// params: { condition: 'button_enabled'|'modal_appeared'|'text_contains', target, text, timeout_ms }
|
|
267
|
+
const timeoutMs = params.timeout_ms || 10000;
|
|
268
|
+
const pollMs = 500;
|
|
269
|
+
const startTime = Date.now();
|
|
270
|
+
let satisfied = false;
|
|
271
|
+
|
|
272
|
+
const poll = async () => {
|
|
273
|
+
while (Date.now() - startTime < timeoutMs) {
|
|
274
|
+
const result = await pageManager.getTree(false, tabId);
|
|
275
|
+
const tree = result?.tree;
|
|
276
|
+
const ctx = result?.context;
|
|
277
|
+
|
|
278
|
+
if (params.condition === 'button_enabled' && params.target) {
|
|
279
|
+
// Find button by id first (what get_tree hands the agent), label as fallback
|
|
280
|
+
const btn = findWaitTarget(tree, params.target);
|
|
281
|
+
if (btn && (!btn.states || btn.states.indexOf('disabled') < 0)) {
|
|
282
|
+
satisfied = true; break;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
if (params.condition === 'modal_appeared') {
|
|
286
|
+
if (ctx?.modals && ctx.modals.length > 0) { satisfied = true; break; }
|
|
287
|
+
}
|
|
288
|
+
if (params.condition === 'text_contains' && params.text) {
|
|
289
|
+
const bodyText = await pageManager.evaluate('document.body.innerText', tabId);
|
|
290
|
+
if (bodyText && bodyText.indexOf(params.text) >= 0) { satisfied = true; break; }
|
|
291
|
+
}
|
|
292
|
+
if (params.condition === 'url_contains' && params.text) {
|
|
293
|
+
const url = await pageManager.evaluate('location.href', tabId);
|
|
294
|
+
if (url && url.indexOf(params.text) >= 0) { satisfied = true; break; }
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
await new Promise(r => setTimeout(r, pollMs));
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
await poll();
|
|
302
|
+
send({ jsonrpc: '2.0', id, result: { satisfied, elapsed_ms: Date.now() - startTime } });
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
// === ui.scroll — scroll page or element ===
|
|
306
|
+
case 'ui.scroll': {
|
|
307
|
+
const direction = params.direction || 'down';
|
|
308
|
+
const amount = Math.max(-10000, Math.min(10000, Number(params.amount) || 500));
|
|
309
|
+
// Pass target/direction/amount as JSON-encoded args — never
|
|
310
|
+
// interpolate params.target into JS source, otherwise a malicious
|
|
311
|
+
// caller can inject `target = "']); fetch('...') //"`.
|
|
312
|
+
const target = typeof params.target === 'string' ? params.target : null;
|
|
313
|
+
const js = `(function(t, d, a){
|
|
314
|
+
if (t) {
|
|
315
|
+
var all = document.querySelectorAll('[data-ai-id]');
|
|
316
|
+
for (var i = 0; i < all.length; i++) {
|
|
317
|
+
if (all[i].getAttribute('data-ai-id') === t) {
|
|
318
|
+
all[i].scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
319
|
+
return { ok: true };
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
return { ok: false, error: 'target not found' };
|
|
323
|
+
}
|
|
324
|
+
window.scrollBy(0, d === 'down' ? a : -a);
|
|
325
|
+
return { ok: true };
|
|
326
|
+
})(${JSON.stringify(target)}, ${JSON.stringify(direction)}, ${amount})`;
|
|
327
|
+
const result = await pageManager.evaluate(js, tabId).catch(() => ({ ok: false }));
|
|
328
|
+
send({ jsonrpc: '2.0', id, result: result || { ok: true } });
|
|
329
|
+
break;
|
|
330
|
+
}
|
|
331
|
+
// === 网络抓包(可查询的 Network 面板)===
|
|
332
|
+
// 列表/详情/清理/配置。错误码见 network_monitor.NETWORK_ERROR_CODES:
|
|
333
|
+
// 不存在 / body 被淘汰 / body 拿不到 / 敏感头未开,各自独立,不静默 null。
|
|
334
|
+
case 'ui.network_list': {
|
|
335
|
+
const result = pageManager.networkList({
|
|
336
|
+
tab: tabId,
|
|
337
|
+
method: params.method,
|
|
338
|
+
url_contains: params.url_contains,
|
|
339
|
+
status: params.status,
|
|
340
|
+
resource_type: params.resource_type,
|
|
341
|
+
started_after: params.started_after,
|
|
342
|
+
started_before: params.started_before,
|
|
343
|
+
state: params.state,
|
|
344
|
+
limit: params.limit,
|
|
345
|
+
before_seq: params.before_seq,
|
|
346
|
+
});
|
|
347
|
+
send({ jsonrpc: '2.0', id, result });
|
|
348
|
+
break;
|
|
349
|
+
}
|
|
350
|
+
case 'ui.network_get': {
|
|
351
|
+
try {
|
|
352
|
+
const result = await pageManager.networkGet(String(params.network_id || ''), {
|
|
353
|
+
include_request_headers: params.include_request_headers === true,
|
|
354
|
+
include_request_body: params.include_request_body === true,
|
|
355
|
+
include_response_headers: params.include_response_headers === true,
|
|
356
|
+
include_response_body: params.include_response_body === true,
|
|
357
|
+
include_sensitive_headers: params.include_sensitive_headers === true,
|
|
358
|
+
request_body_offset: params.request_body_offset,
|
|
359
|
+
response_body_offset: params.response_body_offset,
|
|
360
|
+
body_limit: params.body_limit,
|
|
361
|
+
});
|
|
362
|
+
send({ jsonrpc: '2.0', id, result });
|
|
363
|
+
} catch (e) {
|
|
364
|
+
if (e && e.rpcCode) send({ jsonrpc: '2.0', id, error: { code: e.rpcCode, message: e.message, data: e.data || undefined } });
|
|
365
|
+
else send({ jsonrpc: '2.0', id, error: { code: -32603, message: e.message } });
|
|
366
|
+
}
|
|
367
|
+
break;
|
|
368
|
+
}
|
|
369
|
+
case 'ui.network_clear': {
|
|
370
|
+
const result = pageManager.networkClear(tabId);
|
|
371
|
+
send({ jsonrpc: '2.0', id, result });
|
|
372
|
+
break;
|
|
373
|
+
}
|
|
374
|
+
case 'ui.network_configure': {
|
|
375
|
+
const result = pageManager.networkConfigure(tabId, {
|
|
376
|
+
enabled: params.enabled,
|
|
377
|
+
capture_bodies: params.capture_bodies,
|
|
378
|
+
});
|
|
379
|
+
send({ jsonrpc: '2.0', id, result });
|
|
380
|
+
break;
|
|
381
|
+
}
|
|
382
|
+
// === ui.network_body — DEPRECATED,保留一个版本 ===
|
|
383
|
+
// 按 url_pattern 选最近完成的匹配项(内部转 network_get),响应仍为
|
|
384
|
+
// 旧形状 {body:string|null}。下一主版本删除。
|
|
385
|
+
case 'ui.network_body': {
|
|
386
|
+
const body = await pageManager.getNetworkBody(params.url_pattern || '', tabId);
|
|
387
|
+
send({
|
|
388
|
+
jsonrpc: '2.0', id,
|
|
389
|
+
result: {
|
|
390
|
+
body: body,
|
|
391
|
+
deprecated: true,
|
|
392
|
+
replacement: 'ui.network_list + ui.network_get',
|
|
393
|
+
warning: 'ui.network_body is deprecated and keeps only the first 5000 chars of the newest finished match — use ui.network_list/ui.network_get for method, headers, POST body and paging',
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
break;
|
|
397
|
+
}
|
|
398
|
+
// === ui.quit — let MCP clients shut down the browser gracefully ===
|
|
399
|
+
// The agent owns the browser lifecycle; without this it has no way
|
|
400
|
+
// to release the process after finishing a task.
|
|
401
|
+
case 'ui.downloads': {
|
|
402
|
+
// 下载此前**完全没人处理**:Electron 默认取消,agent 点"导出"只看到
|
|
403
|
+
// success,然后什么都没有,连错误都没有。落盘路径可预期
|
|
404
|
+
// (<userData>/downloads/<tabId>/),这里就是让 agent 问到"文件在哪儿"。
|
|
405
|
+
const r = pageManager.downloads({ tab: tabId });
|
|
406
|
+
send({ jsonrpc: '2.0', id, result: r });
|
|
407
|
+
break;
|
|
408
|
+
}
|
|
409
|
+
case 'ui.quit': {
|
|
410
|
+
send({ jsonrpc: '2.0', id, result: { ok: true } });
|
|
411
|
+
// Defer so the response actually flushes before the WS closes.
|
|
412
|
+
if (onQuit) setTimeout(() => { try { onQuit(); } catch (e) {} }, 200);
|
|
413
|
+
break;
|
|
414
|
+
}
|
|
415
|
+
default:
|
|
416
|
+
send({ jsonrpc: '2.0', id, error: { code: -32601, message: 'Method not found' } });
|
|
417
|
+
break;
|
|
418
|
+
}
|
|
419
|
+
} catch (e) {
|
|
420
|
+
send({ jsonrpc: '2.0', id, error: { code: -32603, message: e.message } });
|
|
421
|
+
}
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
ws.on('close', () => {
|
|
425
|
+
pageManager.unregisterClient(sessionId);
|
|
426
|
+
});
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
console.log('AI Browser WS server listening on ws://localhost:' + port);
|
|
430
|
+
return {
|
|
431
|
+
// Actual bound port (for tests that pass port 0 to grab an ephemeral one).
|
|
432
|
+
port: wss.address().port,
|
|
433
|
+
close: () => {
|
|
434
|
+
for (const client of wss.clients) {
|
|
435
|
+
try { client.terminate(); } catch (e) {}
|
|
436
|
+
}
|
|
437
|
+
wss.close();
|
|
438
|
+
}
|
|
439
|
+
};
|
|
440
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// preload/actions.cjs — Action execution (CommonJS)
|
|
2
|
+
// Split out of bridge.js v6.9. Pure DOM functions, no Electron dependency.
|
|
3
|
+
// Exports: executeAction()
|
|
4
|
+
|
|
5
|
+
// ============================================================
|
|
6
|
+
// 3. EXECUTE ACTION — click, type, setContent, select, etc.
|
|
7
|
+
// ============================================================
|
|
8
|
+
|
|
9
|
+
function executeAction(elementId, action, params) {
|
|
10
|
+
params = params || {};
|
|
11
|
+
var el = document.querySelector('[data-ai-id="' + elementId + '"]');
|
|
12
|
+
if (!el) return { success: false, error: "Target not found for ref: " + elementId };
|
|
13
|
+
|
|
14
|
+
try {
|
|
15
|
+
switch (action) {
|
|
16
|
+
case "click":
|
|
17
|
+
el.focus();
|
|
18
|
+
el.dispatchEvent(new MouseEvent("mousedown", { bubbles: true, cancelable: true, button: 0 }));
|
|
19
|
+
el.dispatchEvent(new MouseEvent("mouseup", { bubbles: true, cancelable: true, button: 0 }));
|
|
20
|
+
// Single trusted dispatch: el.click() fires exactly one click event AND
|
|
21
|
+
// runs the default behavior on checkbox/radio/submit/details. A manual
|
|
22
|
+
// dispatchEvent(click) would fire a second click, double-triggering
|
|
23
|
+
// non-idempotent handlers and toggling checkbox/radio back to original.
|
|
24
|
+
el.click();
|
|
25
|
+
return { success: true };
|
|
26
|
+
|
|
27
|
+
case "type":
|
|
28
|
+
return doType(el, params.text || "");
|
|
29
|
+
|
|
30
|
+
case "setContent":
|
|
31
|
+
return doSetContent(el, params.text || "", params.format || "text");
|
|
32
|
+
|
|
33
|
+
case "clear":
|
|
34
|
+
if (el.value !== undefined) { el.value = ""; el.dispatchEvent(new Event("input", { bubbles: true })); }
|
|
35
|
+
if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") el.innerHTML = "";
|
|
36
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
37
|
+
return { success: true };
|
|
38
|
+
|
|
39
|
+
case "select":
|
|
40
|
+
if (el.tagName.toLowerCase() !== "select") return { success: false, error: "Not a select element" };
|
|
41
|
+
el.value = params.value || "";
|
|
42
|
+
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
43
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
44
|
+
return { success: true };
|
|
45
|
+
|
|
46
|
+
case "focus":
|
|
47
|
+
el.focus();
|
|
48
|
+
return { success: true };
|
|
49
|
+
|
|
50
|
+
case "hover":
|
|
51
|
+
el.dispatchEvent(new MouseEvent("mouseenter", { bubbles: true }));
|
|
52
|
+
el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
|
53
|
+
return { success: true };
|
|
54
|
+
|
|
55
|
+
case "scroll_to":
|
|
56
|
+
el.scrollIntoView({ behavior: "instant", block: "center" });
|
|
57
|
+
return { success: true };
|
|
58
|
+
|
|
59
|
+
case "submit":
|
|
60
|
+
// Submit parent form
|
|
61
|
+
var form = el.closest("form");
|
|
62
|
+
if (form) { form.dispatchEvent(new Event("submit", { bubbles: true, cancelable: true })); return { success: true }; }
|
|
63
|
+
return { success: false, error: "No parent form" };
|
|
64
|
+
|
|
65
|
+
case "toggle":
|
|
66
|
+
if (el.tagName.toLowerCase() === "details") { el.open = !el.open; return { success: true, state: el.open }; }
|
|
67
|
+
if (el.getAttribute("role") === "switch" || el.type === "checkbox") { el.click(); return { success: true }; }
|
|
68
|
+
return { success: false, error: "Cannot toggle" };
|
|
69
|
+
|
|
70
|
+
default:
|
|
71
|
+
return { success: false, error: "Unknown action: " + action };
|
|
72
|
+
}
|
|
73
|
+
} catch(e) {
|
|
74
|
+
return { success: false, error: e.message };
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Native value setter for Vue/React inputs
|
|
79
|
+
function doType(el, text) {
|
|
80
|
+
var tag = el.tagName.toLowerCase();
|
|
81
|
+
if (tag !== "input" && tag !== "textarea" && el.contentEditable !== "true" && el.getAttribute("contenteditable") !== "true") {
|
|
82
|
+
// Try contenteditable
|
|
83
|
+
if (el.querySelector && el.querySelector("[contenteditable=true]")) {
|
|
84
|
+
el = el.querySelector("[contenteditable=true]");
|
|
85
|
+
} else {
|
|
86
|
+
return { success: false, error: "Not a text input" };
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") {
|
|
91
|
+
// contenteditable — use execCommand
|
|
92
|
+
el.focus();
|
|
93
|
+
document.execCommand("selectAll", false, null);
|
|
94
|
+
document.execCommand("insertText", false, text);
|
|
95
|
+
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
96
|
+
return { success: true };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Standard input/textarea — native setter
|
|
100
|
+
var nset = Object.getOwnPropertyDescriptor(
|
|
101
|
+
tag === "textarea" ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype, "value"
|
|
102
|
+
).set;
|
|
103
|
+
nset.call(el, text);
|
|
104
|
+
el.dispatchEvent(new Event("input", { bubbles: true, composed: true }));
|
|
105
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
106
|
+
el.dispatchEvent(new Event("blur", { bubbles: true }));
|
|
107
|
+
return { success: true, value: el.value.slice(0, 40) };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// setContent — write to any editor type including Draft.js/ProseMirror
|
|
111
|
+
function doSetContent(el, text, format) {
|
|
112
|
+
var paras = text.split("\n");
|
|
113
|
+
var isEmptyPara = false;
|
|
114
|
+
|
|
115
|
+
// 1. Try contenteditable — use execCommand paragraph by paragraph
|
|
116
|
+
var editor = null;
|
|
117
|
+
if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") {
|
|
118
|
+
editor = el;
|
|
119
|
+
} else {
|
|
120
|
+
// Look inside for Draft.js, ProseMirror, CodeMirror editors
|
|
121
|
+
editor = el.querySelector(".public-DraftEditor-content") ||
|
|
122
|
+
el.querySelector("[class*=ProseMirror]") ||
|
|
123
|
+
el.querySelector("[contenteditable=true]") ||
|
|
124
|
+
el.querySelector("[class*=DraftEditor]");
|
|
125
|
+
if (!editor) {
|
|
126
|
+
// Try CodeMirror instance
|
|
127
|
+
var cm = el.querySelector("[class*=CodeMirror]");
|
|
128
|
+
if (cm && cm.CodeMirror) { cm.CodeMirror.setValue(text); return { success: true }; }
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (!editor) {
|
|
133
|
+
// Fallback: textarea
|
|
134
|
+
var ta = el.querySelector("textarea") || (el.tagName.toLowerCase() === "textarea" ? el : null);
|
|
135
|
+
if (ta) {
|
|
136
|
+
var nset = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value").set;
|
|
137
|
+
nset.call(ta, text);
|
|
138
|
+
ta.dispatchEvent(new Event("input", { bubbles: true }));
|
|
139
|
+
ta.dispatchEvent(new Event("change", { bubbles: true }));
|
|
140
|
+
return { success: true, chars: ta.value.length };
|
|
141
|
+
}
|
|
142
|
+
return { success: false, error: "No editable element found" };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Clear editor first
|
|
146
|
+
editor.focus();
|
|
147
|
+
document.execCommand("selectAll", false, null);
|
|
148
|
+
document.execCommand("delete", false, null);
|
|
149
|
+
|
|
150
|
+
// Insert paragraph by paragraph with Enter key events
|
|
151
|
+
for (var pi = 0; pi < paras.length; pi++) {
|
|
152
|
+
var para = paras[pi];
|
|
153
|
+
|
|
154
|
+
// Position cursor at end
|
|
155
|
+
var sel = window.getSelection();
|
|
156
|
+
var range = document.createRange();
|
|
157
|
+
range.selectNodeContents(editor);
|
|
158
|
+
range.collapse(false);
|
|
159
|
+
sel.removeAllRanges();
|
|
160
|
+
sel.addRange(range);
|
|
161
|
+
|
|
162
|
+
if (para.length > 0) {
|
|
163
|
+
document.execCommand("insertText", false, para);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Insert Enter after each paragraph (except last)
|
|
167
|
+
if (pi < paras.length - 1) {
|
|
168
|
+
editor.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", code: "Enter", keyCode: 13, which: 13, bubbles: true }));
|
|
169
|
+
editor.dispatchEvent(new KeyboardEvent("keypress", { key: "Enter", code: "Enter", keyCode: 13, which: 13, bubbles: true }));
|
|
170
|
+
editor.dispatchEvent(new KeyboardEvent("keyup", { key: "Enter", code: "Enter", keyCode: 13, which: 13, bubbles: true }));
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// Small delay between paragraphs
|
|
174
|
+
var sync = new Date().getTime();
|
|
175
|
+
while (new Date().getTime() - sync < 10) {} // 10ms pause for React sync
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
editor.dispatchEvent(new Event("input", { bubbles: true, composed: true }));
|
|
179
|
+
editor.dispatchEvent(new Event("change", { bubbles: true }));
|
|
180
|
+
|
|
181
|
+
// Verify placeholder disappeared
|
|
182
|
+
var innerText = editor.innerText || "";
|
|
183
|
+
if (innerText.indexOf("请输入正文") >= 0) {
|
|
184
|
+
return { success: false, error: "Editor did not register input — placeholder persists" };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return { success: true, chars: innerText.length, blocks: editor.querySelectorAll ? editor.querySelectorAll("div, p").length : 0 };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
module.exports = { executeAction };
|