@glittering0717/ai-browser 0.0.0-stage → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +784 -2
- package/bin/ai-browser.cjs +78 -0
- package/examples/AGENT_INSTRUCTIONS.md +166 -0
- package/examples/mcp-config.json +12 -0
- package/llms-install.md +158 -0
- package/package.json +78 -4
- package/scripts/electron_resolve.cjs +241 -0
- package/scripts/start.cjs +60 -0
- package/src/main/axExtractor.js +436 -0
- package/src/main/canvas_hook_source.js +654 -0
- package/src/main/canvas_monitor.js +269 -0
- package/src/main/capability_probe.js +415 -0
- package/src/main/download_monitor.js +137 -0
- package/src/main/index.js +359 -0
- package/src/main/mcp_server.js +408 -0
- package/src/main/mcp_tools.js +289 -0
- package/src/main/mcp_ws.js +178 -0
- package/src/main/network_monitor.js +805 -0
- package/src/main/page_manager.js +2743 -0
- package/src/main/snapshot_diff.js +371 -0
- package/src/main/ws_server.js +440 -0
- package/src/preload/actions.cjs +190 -0
- package/src/preload/bridge.cjs +75 -0
- package/src/preload/extractor.cjs +936 -0
- package/src/preload/watcher.cjs +241 -0
- package/src/renderer/tab_bar.html +115 -0
- package/src/shared/config.js +32 -0
- package/src/shared/deadline.js +89 -0
- package/src/shared/guards.js +17 -0
- package/src/shared/protocol.js +116 -0
- package/src/shared/value_contract.cjs +132 -0
- package/src/shared/version.js +5 -0
- package/src/shared/watchdog.js +57 -0
|
@@ -0,0 +1,2743 @@
|
|
|
1
|
+
// main/page_manager.js — Multi-tab page lifecycle manager v2
|
|
2
|
+
// One Electron process, one WS server, multiple tabs.
|
|
3
|
+
// Agent routes via tab ID. Tab 0 is default.
|
|
4
|
+
import { ipcMain, BrowserView } from 'electron';
|
|
5
|
+
import { config } from '../shared/config.js';
|
|
6
|
+
import axExtractor from './axExtractor.js';
|
|
7
|
+
import { NetworkMonitor, MAX_POST_DATA_SIZE } from './network_monitor.js';
|
|
8
|
+
import { CanvasMonitor, isCanvasHookEnabled } from './canvas_monitor.js';
|
|
9
|
+
import { buildCanvasHookSource } from './canvas_hook_source.js';
|
|
10
|
+
import { diffValues, previewOf, safeByteLength, DEFAULT_SNAPSHOT_JS, DIFF_MAX_ENTRIES, evictOldest } from './snapshot_diff.js';
|
|
11
|
+
import { withDeadline, actionDeadlineMs } from '../shared/deadline.js';
|
|
12
|
+
import { DownloadMonitor } from './download_monitor.js';
|
|
13
|
+
import { probeCapabilities } from './capability_probe.js';
|
|
14
|
+
import valueContract from '../shared/value_contract.cjs';
|
|
15
|
+
|
|
16
|
+
const { expandValueFetchTree } = valueContract;
|
|
17
|
+
|
|
18
|
+
// canvas 绘制调用记录:默认开。关闭开关 AI_BROWSER_CANVAS_HOOK=0。
|
|
19
|
+
const CANVAS_BRIDGE_KEY = '__aiCanvasBridge';
|
|
20
|
+
|
|
21
|
+
// 网络抓包默认开:只在有订阅时才启用会导致首屏/导航阶段的请求全丢(POST body、
|
|
22
|
+
// 请求头都拿不到)。关闭开关:AI_BROWSER_NETWORK_CAPTURE=0。
|
|
23
|
+
const NETWORK_CAPTURE_DEFAULT = !(process.env.AI_BROWSER_NETWORK_CAPTURE === '0' || process.env.AI_BROWSER_NETWORK_CAPTURE === 'false');
|
|
24
|
+
|
|
25
|
+
// 具名按键 → CDP Input.dispatchKeyEvent 参数。agent 用自然名字('Delete'、
|
|
26
|
+
// 'Escape'、'ArrowRight'、'a')而不是让人去查 Windows 虚拟键码。
|
|
27
|
+
// vk 用 Windows 虚拟键码(Chromium 在 macOS 上也按它解释)。
|
|
28
|
+
const NAMED_KEYS = {
|
|
29
|
+
Enter: { key: 'Enter', code: 'Enter', vk: 13 },
|
|
30
|
+
NumpadEnter: { key: 'Enter', code: 'NumpadEnter', vk: 13 },
|
|
31
|
+
Tab: { key: 'Tab', code: 'Tab', vk: 9 },
|
|
32
|
+
Escape: { key: 'Escape', code: 'Escape', vk: 27 },
|
|
33
|
+
Esc: { key: 'Escape', code: 'Escape', vk: 27 },
|
|
34
|
+
Backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
|
|
35
|
+
Delete: { key: 'Delete', code: 'Delete', vk: 46 },
|
|
36
|
+
Space: { key: ' ', code: 'Space', vk: 32, text: ' ' },
|
|
37
|
+
ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
|
|
38
|
+
ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
|
|
39
|
+
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
|
|
40
|
+
ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
|
|
41
|
+
Home: { key: 'Home', code: 'Home', vk: 36 },
|
|
42
|
+
End: { key: 'End', code: 'End', vk: 35 },
|
|
43
|
+
PageUp: { key: 'PageUp', code: 'PageUp', vk: 33 },
|
|
44
|
+
PageDown: { key: 'PageDown', code: 'PageDown', vk: 34 },
|
|
45
|
+
Minus: { key: '-', code: 'Minus', vk: 189 },
|
|
46
|
+
Equal: { key: '=', code: 'Equal', vk: 187 },
|
|
47
|
+
'0': { key: '0', code: 'Digit0', vk: 48 },
|
|
48
|
+
'1': { key: '1', code: 'Digit1', vk: 49 },
|
|
49
|
+
'2': { key: '2', code: 'Digit2', vk: 50 },
|
|
50
|
+
'3': { key: '3', code: 'Digit3', vk: 51 },
|
|
51
|
+
'4': { key: '4', code: 'Digit4', vk: 52 },
|
|
52
|
+
'5': { key: '5', code: 'Digit5', vk: 53 },
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
// 修饰键 → CDP modifiers 位掩码(Alt=1, Ctrl=2, Meta=4, Shift=8)。
|
|
56
|
+
function modifierMask(modifiers) {
|
|
57
|
+
let m = 0;
|
|
58
|
+
for (const k of modifiers || []) {
|
|
59
|
+
const s = String(k).toLowerCase();
|
|
60
|
+
if (s === 'alt' || s === 'option') m |= 1;
|
|
61
|
+
else if (s === 'control' || s === 'ctrl') m |= 2;
|
|
62
|
+
else if (s === 'meta' || s === 'cmd' || s === 'command') m |= 4;
|
|
63
|
+
else if (s === 'shift') m |= 8;
|
|
64
|
+
}
|
|
65
|
+
return m;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* 解析一个按键名到 _cdpKey 需要的参数。支持具名键、单字符(字母/数字/符号)、
|
|
70
|
+
* 以及 'Shift+Delete' / 'Meta+a' / 'Control+ArrowRight' 这类组合写法。
|
|
71
|
+
* 返回 null 表示无法识别 —— 调用方报错,不静默发一个错误按键。
|
|
72
|
+
*/
|
|
73
|
+
function keySpec(name, extraModifiers) {
|
|
74
|
+
const raw = String(name == null ? '' : name);
|
|
75
|
+
const parts = raw.split('+').map((p) => p.trim()).filter(Boolean);
|
|
76
|
+
const baseName = parts.length ? parts[parts.length - 1] : raw;
|
|
77
|
+
const combos = parts.slice(0, -1);
|
|
78
|
+
let mod = modifierMask([...(extraModifiers || []), ...combos]);
|
|
79
|
+
|
|
80
|
+
let spec = NAMED_KEYS[baseName] || null;
|
|
81
|
+
if (!spec && /^[a-zA-Z]$/.test(baseName)) {
|
|
82
|
+
const up = baseName.toUpperCase();
|
|
83
|
+
// key 用调用方给的大小写('a' → 'a'),code 永远指物理键('KeyA')。
|
|
84
|
+
// 曾经写成 `key: baseName !== up ? up : baseName` —— 对任何小写字母
|
|
85
|
+
// baseName 都 !== 其大写,于是 'a' 被发成 'A',页面 keydown 里读到的全是大写。
|
|
86
|
+
spec = { key: baseName, code: 'Key' + up, vk: up.charCodeAt(0) };
|
|
87
|
+
if (baseName === up) mod |= 8; // 大写字母本身意味着 Shift 按下
|
|
88
|
+
}
|
|
89
|
+
if (!spec && baseName.length === 1) {
|
|
90
|
+
spec = { key: baseName, code: '', vk: baseName.toUpperCase().charCodeAt(0), text: baseName };
|
|
91
|
+
}
|
|
92
|
+
if (!spec) return null;
|
|
93
|
+
// 真实浏览器在 Shift 按下时,字母键报的是大写 key。对齐它,否则页面里的
|
|
94
|
+
// 快捷键判断(e.key === 'A')会失灵。
|
|
95
|
+
let key = spec.key;
|
|
96
|
+
if ((mod & 8) && /^[a-z]$/.test(key)) key = key.toUpperCase();
|
|
97
|
+
return { ...spec, key, mod };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
class PageManager {
|
|
101
|
+
constructor(browserWindow) {
|
|
102
|
+
this.window = browserWindow;
|
|
103
|
+
this.tabs = new Map(); // tabId -> BrowserView
|
|
104
|
+
this.activeTab = null;
|
|
105
|
+
this.tabCounter = 0;
|
|
106
|
+
this.subscriptions = new Map();
|
|
107
|
+
this._pendingRequests = new Map();
|
|
108
|
+
this._requestId = 0;
|
|
109
|
+
this._wsClients = new Map();
|
|
110
|
+
this._openedTabs = new Map(); // sourceTabId -> { id, url } (new tab opened by window.open)
|
|
111
|
+
this._axDiffCache = new Map(); // tabId -> lite interactive snapshot (get_tree {mode:'diff'})
|
|
112
|
+
this._revertTimers = new Map(); // tabId -> 挂起的 ui.peek 自动回撤定时器(新 peek 会取消它)
|
|
113
|
+
this._repaintTimers = new Map();// tabId -> 正在排队/已调度的重绘(合并同一批导航事件)
|
|
114
|
+
this._snapshots = new Map(); // snapshotId -> { tabId, js, value, at }(ui.snapshot 存基线)
|
|
115
|
+
this._snapshotSeq = 0;
|
|
116
|
+
this.networkMonitor = new NetworkMonitor({}, { captureEnabled: NETWORK_CAPTURE_DEFAULT });
|
|
117
|
+
this.canvasMonitor = new CanvasMonitor();
|
|
118
|
+
this._canvasBindings = new Map(); // bindingName -> tabId(hook 回传时据此归 tab)
|
|
119
|
+
// 下载:此前 src/ 里没有任何 will-download 处理器,Electron 默认**取消**下载 ——
|
|
120
|
+
// agent 点"导出 CSV"拿到 success,然后什么都没有,连错误都没有。
|
|
121
|
+
this.downloadMonitor = new DownloadMonitor(null, {
|
|
122
|
+
broadcast: (type, data) => this._broadcast(type, data),
|
|
123
|
+
});
|
|
124
|
+
this.downloadMonitor._resolveTab = (wc) => this._tabIdOfWebContents(wc);
|
|
125
|
+
this._setupPermissions();
|
|
126
|
+
this._setupIPC();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* 权限请求白名单。
|
|
131
|
+
*
|
|
132
|
+
* 不装这个处理器时,Electron 对每个请求都**静默拒绝**。后果与下载一样隐蔽:
|
|
133
|
+
* 站点要 pointerLock / clipboard 写入才能继续,agent 完全看不出是被权限挡住。
|
|
134
|
+
*
|
|
135
|
+
* 默认只放行"不碰用户隐私、且 agent 确实需要"的几项:
|
|
136
|
+
* pointerLock 画布类应用常用,无隐私面
|
|
137
|
+
* clipboard-sanitized-write "复制"按钮要它能用(写入是 sanititized 的,读不放)
|
|
138
|
+
* fullscreen 演示/看图常用,无隐私面
|
|
139
|
+
* localStorage/sessionStorage 等存储类(Chromium 已默认允许,列出来只为显式)
|
|
140
|
+
* 摄像头/麦克风/定位**不放**:那是隐私面,应由人主动决定。
|
|
141
|
+
* 覆盖:AI_BROWSER_PERMISSIONS="midi,geolocation"(追加)或 "*"(全放,慎用)。
|
|
142
|
+
*
|
|
143
|
+
* 拒绝也广播一条事件:否则 agent 无法区分"站点没这功能"和"被权限挡住"。
|
|
144
|
+
*/
|
|
145
|
+
_setupPermissions() {
|
|
146
|
+
const DEFAULT_ALLOW = ['pointerLock', 'clipboard-sanitized-write', 'fullscreen'];
|
|
147
|
+
const raw = process.env.AI_BROWSER_PERMISSIONS;
|
|
148
|
+
const allow = raw === '*' ? null : (raw ? raw.split(',').map((s) => s.trim()).filter(Boolean) : DEFAULT_ALLOW);
|
|
149
|
+
const allowed = (p) => (allow === null ? true : allow.indexOf(p) >= 0);
|
|
150
|
+
this._permissionAllow = allow;
|
|
151
|
+
this._onPermission = (webContents, permission, callback) => {
|
|
152
|
+
if (allowed(permission)) return callback(true);
|
|
153
|
+
const tid = this._tabIdOfWebContents(webContents);
|
|
154
|
+
this._broadcast('permission_denied', {
|
|
155
|
+
tab: tid,
|
|
156
|
+
permission,
|
|
157
|
+
hint: 'grant it with AI_BROWSER_PERMISSIONS="' + permission + '" or "*", or ask the user to allow it in the window',
|
|
158
|
+
});
|
|
159
|
+
callback(false);
|
|
160
|
+
};
|
|
161
|
+
this._installPermissionHandler();
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
_installPermissionHandler() {
|
|
165
|
+
try {
|
|
166
|
+
// 延迟到 app ready 之后由 index.js 调 installSessionHooks(session);
|
|
167
|
+
// 这里只存回调,避免在模块加载期碰 Electron session(vitest 起不来)。
|
|
168
|
+
if (this._session) this._session.setPermissionRequestHandler(this._onPermission);
|
|
169
|
+
} catch (_) { /* 无 session(单测/非 Electron 环境) */ }
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
installSessionHooks(session) {
|
|
173
|
+
this._session = session;
|
|
174
|
+
this.downloadMonitor.session = session;
|
|
175
|
+
this.downloadMonitor.install();
|
|
176
|
+
this._installPermissionHandler();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** webContents → tabId。找不到返回 undefined(调用方各自决定怎么兜底)。 */
|
|
180
|
+
_tabIdOfWebContents(webContents) {
|
|
181
|
+
if (!webContents) return undefined;
|
|
182
|
+
for (const [tid, view] of this.tabs) {
|
|
183
|
+
if (view && view.webContents === webContents) return tid;
|
|
184
|
+
}
|
|
185
|
+
return undefined;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
static _instance = null;
|
|
189
|
+
static getInstance(browserWindow) {
|
|
190
|
+
if (!PageManager._instance) PageManager._instance = new PageManager(browserWindow);
|
|
191
|
+
return PageManager._instance;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
_setupIPC() {
|
|
195
|
+
const self = this;
|
|
196
|
+
// Match by request id — robust under concurrent requests / multi-tab.
|
|
197
|
+
// FIFO broke when getTree/executeAction/evaluate raced: the first response
|
|
198
|
+
// resolved whichever pending promise happened to be first in the map,
|
|
199
|
+
// not the one that actually triggered it.
|
|
200
|
+
ipcMain.on('ai:tree', (_event, payload) => {
|
|
201
|
+
const id = payload && payload.id;
|
|
202
|
+
const pending = (id !== undefined) ? self._pendingRequests.get(id) : null;
|
|
203
|
+
if (pending) {
|
|
204
|
+
self._pendingRequests.delete(id);
|
|
205
|
+
pending.resolve(payload);
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
ipcMain.on('ai:action_result', (_event, result) => {
|
|
210
|
+
const id = result && result.id;
|
|
211
|
+
const pending = (id !== undefined) ? self._pendingRequests.get(id) : null;
|
|
212
|
+
if (pending) {
|
|
213
|
+
self._pendingRequests.delete(id);
|
|
214
|
+
// Strip routing id from the payload before resolving.
|
|
215
|
+
const { id: _drop, ...clean } = result;
|
|
216
|
+
pending.resolve(clean);
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
ipcMain.on('ai:evaluate_result', (_event, result) => {
|
|
221
|
+
const id = result && result.id;
|
|
222
|
+
const pending = (id !== undefined) ? self._pendingRequests.get(id) : null;
|
|
223
|
+
if (pending) {
|
|
224
|
+
self._pendingRequests.delete(id);
|
|
225
|
+
if (result.error) pending.reject(new Error(result.error));
|
|
226
|
+
else pending.resolve(result.value);
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
ipcMain.on('ai:diff', (_event, changes) => {
|
|
231
|
+
this._broadcast('dom_change', { changes });
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
ipcMain.on('ai:event', (_event, payload) => {
|
|
235
|
+
// Tag events with active tab
|
|
236
|
+
this._broadcast(payload.event, payload.data || {});
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
_broadcast(eventType, data) {
|
|
241
|
+
for (const [sessionId, eventTypes] of this.subscriptions) {
|
|
242
|
+
if (eventTypes.has(eventType) || eventTypes.has('*')) {
|
|
243
|
+
const ws = this._wsClients.get(sessionId);
|
|
244
|
+
if (ws && ws.readyState === 1) {
|
|
245
|
+
ws.send(JSON.stringify({ jsonrpc: '2.0', method: eventType, params: data }));
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
registerClient(sessionId, ws) {
|
|
252
|
+
this._wsClients.set(sessionId, ws);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
unregisterClient(sessionId) {
|
|
256
|
+
this._wsClients.delete(sessionId);
|
|
257
|
+
this.subscriptions.delete(sessionId);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
_layoutAllViews(bounds) {
|
|
261
|
+
for (const [id, view] of this.tabs) {
|
|
262
|
+
view.setBounds({ x: 0, y: 36, width: bounds.width, height: bounds.height - 36 });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// === Tab management ===
|
|
267
|
+
|
|
268
|
+
_getView(tabId) {
|
|
269
|
+
return this.tabs.get(tabId);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// ==================================================================
|
|
273
|
+
// Snapshot / Diff — make "act → read back → compare" a one-liner
|
|
274
|
+
// ==================================================================
|
|
275
|
+
//
|
|
276
|
+
// Why this exists: the project's hard rule is that a write is only proven by
|
|
277
|
+
// reading state back and comparing. Doing that by hand meant rewriting bespoke
|
|
278
|
+
// comparison JS on every site (node counts, widget values, positions, edge
|
|
279
|
+
// lists…), which is exactly where mistakes creep in — this session I twice
|
|
280
|
+
// measured the wrong thing (serialized byte length, viewport deltas) and got
|
|
281
|
+
// a confident false reading.
|
|
282
|
+
//
|
|
283
|
+
// Contract, deliberately narrow:
|
|
284
|
+
// ui.snapshot {js?, label?} → evaluate js (default: a site-agnostic
|
|
285
|
+
// structural DOM summary), store, return id.
|
|
286
|
+
// ui.diff {snapshot, js?} → re-evaluate and return a STRUCTURED diff:
|
|
287
|
+
// added / removed / changed, each with a JSON
|
|
288
|
+
// path and both values, so "it changed" is
|
|
289
|
+
// provable rather than asserted.
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Collect a snapshot value. `js` must be a JSON-serializable expression.
|
|
293
|
+
* When omitted we use a structural DOM summary (role/label/value per element
|
|
294
|
+
* at a stable path) — the most broadly useful default, needing no per-site
|
|
295
|
+
* knowledge, which is the whole point of having a default.
|
|
296
|
+
*/
|
|
297
|
+
async _collectSnapshot(js, tabId) {
|
|
298
|
+
const tid = tabId !== undefined && tabId !== null ? tabId : this.activeTab;
|
|
299
|
+
const view = this._getView(tid);
|
|
300
|
+
if (!view) return { error: 'no-such-tab', tab: tid };
|
|
301
|
+
const expr = js && String(js).trim() ? String(js) : DEFAULT_SNAPSHOT_JS;
|
|
302
|
+
let value;
|
|
303
|
+
try {
|
|
304
|
+
value = await this.evaluate(expr, tid);
|
|
305
|
+
} catch (e) {
|
|
306
|
+
return { error: 'evaluate-failed: ' + (e && e.message ? e.message : String(e)), tab: tid };
|
|
307
|
+
}
|
|
308
|
+
if (value && typeof value === 'object' && value.__error) {
|
|
309
|
+
return { error: 'snapshot-expression-threw: ' + value.__error, tab: tid };
|
|
310
|
+
}
|
|
311
|
+
return { tab: tid, value };
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async snapshot(params = {}) {
|
|
315
|
+
const tid = params.tab !== undefined && params.tab !== null ? params.tab : this.activeTab;
|
|
316
|
+
const r = await this._collectSnapshot(params.js, tid);
|
|
317
|
+
if (r.error) return { ok: false, error: r.error, tab: r.tab };
|
|
318
|
+
const id = 'snap-' + (++this._snapshotSeq);
|
|
319
|
+
const entry = { tab: tid, js: params.js || null, value: r.value, at: new Date().toISOString() };
|
|
320
|
+
this._snapshots.set(id, entry);
|
|
321
|
+
this._evictSnapshots();
|
|
322
|
+
return {
|
|
323
|
+
ok: true,
|
|
324
|
+
snapshot: id,
|
|
325
|
+
tab: tid,
|
|
326
|
+
at: entry.at,
|
|
327
|
+
label: params.label || null,
|
|
328
|
+
bytes: safeByteLength(r.value),
|
|
329
|
+
// Short preview so the caller can confirm it snapshotted what they meant,
|
|
330
|
+
// without a second round-trip.
|
|
331
|
+
preview: previewOf(r.value),
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
async diff(params = {}) {
|
|
336
|
+
const id = params.snapshot;
|
|
337
|
+
if (!id) return { ok: false, error: 'missing-snapshot', hint: 'call ui.snapshot first, pass its id here' };
|
|
338
|
+
const base = this._snapshots.get(id);
|
|
339
|
+
if (!base) return { ok: false, error: 'unknown-snapshot', snapshot: id };
|
|
340
|
+
// A snapshot is bound to the tab it came from: diffing across tabs would
|
|
341
|
+
// compare unrelated pages and report a wall of spurious changes.
|
|
342
|
+
const tid = params.tab !== undefined && params.tab !== null ? params.tab : base.tab;
|
|
343
|
+
if (tid !== base.tab) {
|
|
344
|
+
return { ok: false, error: 'tab-mismatch', snapshot: id, snapshot_tab: base.tab, requested_tab: tid };
|
|
345
|
+
}
|
|
346
|
+
const now = await this._collectSnapshot(params.js || base.js, tid);
|
|
347
|
+
if (now.error) return { ok: false, error: now.error, tab: tid };
|
|
348
|
+
|
|
349
|
+
const d = diffValues(base.value, now.value);
|
|
350
|
+
const result = {
|
|
351
|
+
ok: true,
|
|
352
|
+
snapshot: id,
|
|
353
|
+
tab: tid,
|
|
354
|
+
changed: d.hasChanges,
|
|
355
|
+
// An explicit verdict so callers never have to infer "nothing happened"
|
|
356
|
+
// from an array they might have built wrong.
|
|
357
|
+
verdict: d.hasChanges ? 'changed' : 'unchanged',
|
|
358
|
+
counts: { added: d.added.length, removed: d.removed.length, changed: d.changed.length },
|
|
359
|
+
added: d.added,
|
|
360
|
+
removed: d.removed,
|
|
361
|
+
changed_paths: d.changed,
|
|
362
|
+
truncation: d.dropped ? { limit: DIFF_MAX_ENTRIES, dropped: d.dropped } : null,
|
|
363
|
+
};
|
|
364
|
+
// Chaining: re-arm the baseline so a caller can loop "assert no change"
|
|
365
|
+
// over many steps without re-snapshotting each time.
|
|
366
|
+
if (params.rearm) {
|
|
367
|
+
this._snapshots.set(id, { ...base, value: now.value, at: new Date().toISOString() });
|
|
368
|
+
result.rearmed = true;
|
|
369
|
+
}
|
|
370
|
+
if (params.forget) { this._snapshots.delete(id); result.forgotten = true; }
|
|
371
|
+
// Touch: Map iteration order is insertion order, so re-inserting a used
|
|
372
|
+
// baseline moves it to the back and makes eviction least-recently-USED
|
|
373
|
+
// rather than least-recently-created. A baseline an agent keeps diffing
|
|
374
|
+
// must not be evicted while an untouched one taken later survives.
|
|
375
|
+
if (!result.forgotten && this._snapshots.has(id)) {
|
|
376
|
+
const cur = this._snapshots.get(id);
|
|
377
|
+
this._snapshots.delete(id);
|
|
378
|
+
this._snapshots.set(id, cur);
|
|
379
|
+
}
|
|
380
|
+
return result;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Bound the snapshot store. Until this existed, nothing but an explicit
|
|
385
|
+
* `forget` ever removed a baseline: a long agent session accumulated every
|
|
386
|
+
* snapshot it had ever taken, each holding a full page snapshot (the default
|
|
387
|
+
* one carries the whole page's text — hundreds of KB on a real page).
|
|
388
|
+
* Eviction is least-recently-used, because diff() re-inserts a baseline it
|
|
389
|
+
* touches.
|
|
390
|
+
*/
|
|
391
|
+
_evictSnapshots() {
|
|
392
|
+
return evictOldest(this._snapshots);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Probe what this page affords, and which route an agent should take.
|
|
397
|
+
* Thin wrapper so the WS/MCP layers don't need to know the implementation.
|
|
398
|
+
*/
|
|
399
|
+
async capabilities(opts = {}) {
|
|
400
|
+
return probeCapabilities(this, opts);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* ui.downloads —— agent 问"我刚才点导出,文件在哪儿"。
|
|
405
|
+
*
|
|
406
|
+
* 为什么必须是独立查询而不是塞进 ui.act 的返回值:下载是异步的,click 返回时
|
|
407
|
+
* 它往往还没开始(站点常先发一个 XHR 再触发下载)。塞进去会让 agent 在错的
|
|
408
|
+
* 时间点看到一个空列表,然后以为没触发。
|
|
409
|
+
*/
|
|
410
|
+
downloads(params = {}) {
|
|
411
|
+
const tid = params.tab !== undefined && params.tab !== null ? params.tab : this.activeTab;
|
|
412
|
+
const r = this.downloadMonitor.list(tid);
|
|
413
|
+
return {
|
|
414
|
+
...r,
|
|
415
|
+
tab: tid,
|
|
416
|
+
// 只把 completed 的算作"可用",并明确告诉 agent 其余的状态 ——
|
|
417
|
+
// 一个 interrupted 的记录不代表文件存在。
|
|
418
|
+
ready: (r.downloads || []).filter((d) => d.state === 'completed'),
|
|
419
|
+
note: 'only state=completed entries have a file on disk; check .path',
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
_sendToView(tabId, channel, payload) {
|
|
424
|
+
const view = this._getView(tabId);
|
|
425
|
+
if (!view) return false;
|
|
426
|
+
view.webContents.send(channel, payload);
|
|
427
|
+
return true;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* 导航。返回 `{ok:true}` 或 `{ok:false,error}` —— 成功时形状与旧的布尔透出一致
|
|
432
|
+
* (外部调用方读 `.ok` 不受影响),失败时多带一个原因:否则 agent 只拿到
|
|
433
|
+
* `ok:false`,没有任何线索判断该重试还是该换 URL。
|
|
434
|
+
*
|
|
435
|
+
* loadURL 挂在 did-finish-load 上,而"永不触发 did-finish-load 的页面"是存在的
|
|
436
|
+
* (挂起的连接、不结束的加载)。这与 ui.act 是同一类故障:await 一个可能不返回的
|
|
437
|
+
* 东西。所以同样套 deadline。
|
|
438
|
+
*/
|
|
439
|
+
async navigate(url, tabId) {
|
|
440
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
441
|
+
const view = this._getView(tid);
|
|
442
|
+
if (!view) return { ok: false, error: 'Tab not found' };
|
|
443
|
+
this._axDiffCache.delete(tid); // fresh page → drop stale incremental snapshot
|
|
444
|
+
// loadURL resolves at did-finish-load, which is exactly when the old
|
|
445
|
+
// pixels may still be on screen — repaint here as well as from the event
|
|
446
|
+
// hooks, so a programmatic navigate never depends on hook timing.
|
|
447
|
+
try {
|
|
448
|
+
await withDeadline(view.webContents.loadURL(url), 30000, 'navigate');
|
|
449
|
+
} catch (e) {
|
|
450
|
+
if (e && e.__deadline) {
|
|
451
|
+
return { ok: false, error: `${e.message} — the page never finished loading; it may still be loading in the background` };
|
|
452
|
+
}
|
|
453
|
+
return { ok: false, error: String((e && e.message) || e).slice(0, 300) };
|
|
454
|
+
}
|
|
455
|
+
this.repaint(tid);
|
|
456
|
+
return { ok: true };
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
async getTree(focusedOnly = false, tabId) {
|
|
460
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
461
|
+
const view = this._getView(tid);
|
|
462
|
+
if (!view) return { tree: null, context: null };
|
|
463
|
+
|
|
464
|
+
const res = await new Promise((resolve, reject) => {
|
|
465
|
+
const id = ++this._requestId;
|
|
466
|
+
this._pendingRequests.set(id, { resolve, reject });
|
|
467
|
+
view.webContents.send('ai:extract', { focusedOnly, id });
|
|
468
|
+
setTimeout(() => {
|
|
469
|
+
if (this._pendingRequests.has(id)) {
|
|
470
|
+
this._pendingRequests.delete(id);
|
|
471
|
+
reject(new Error('getTree timeout'));
|
|
472
|
+
}
|
|
473
|
+
}, 5000);
|
|
474
|
+
});
|
|
475
|
+
// 截断的 node.value 带内部字段 value_fetch_ref —— 展开成含实际 tab 的
|
|
476
|
+
// value_fetch(mcp/ws 两种可直接执行的调用提示),再删掉内部字段。
|
|
477
|
+
if (res && res.tree) expandValueFetchTree(res.tree, tid);
|
|
478
|
+
return res;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// AX probe read layer (P0). Additive — the default getTree path is untouched.
|
|
482
|
+
// Reuses the per-tab shared debugger: enables the Accessibility + DOM domains
|
|
483
|
+
// into the same reference-counted _cdpTabs entry as Network/Runtime, so teardown
|
|
484
|
+
// and multi-client reference counting are unchanged.
|
|
485
|
+
async getTreeViaAx(tabId, opts = {}) {
|
|
486
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
487
|
+
const view = this._getView(tid);
|
|
488
|
+
if (!view) return { tree: null, context: null };
|
|
489
|
+
try {
|
|
490
|
+
this._ensureCdp(tid, view, 'Accessibility');
|
|
491
|
+
this._ensureCdp(tid, view, 'DOM');
|
|
492
|
+
} catch (e) {
|
|
493
|
+
return { tree: null, context: null, error: 'cdp-unavailable' };
|
|
494
|
+
}
|
|
495
|
+
// 与 ui.act 同一类风险:这些全是 await CDP 命令(Accessibility.getFullAXTree /
|
|
496
|
+
// DOM.getDocument / DOM.resolveNode),渲染进程主线程被占住时它们不返回。
|
|
497
|
+
// getTree 的 preload 路径有 5s 超时,这条 AX 路径此前没有 —— 补上。
|
|
498
|
+
try {
|
|
499
|
+
const main = await withDeadline(
|
|
500
|
+
axExtractor.extractFromDebugger(view.webContents.debugger),
|
|
501
|
+
15000, 'get_tree(ax)'
|
|
502
|
+
);
|
|
503
|
+
// #1: decorate link/button nodes with their absolute target URL (read-only).
|
|
504
|
+
// Generic — agent sees "where clicking takes me" instead of guessing labels.
|
|
505
|
+
if (main && main.tree && view.webContents.debugger) {
|
|
506
|
+
main.tree = await withDeadline(
|
|
507
|
+
axExtractor.attachHrefs(view.webContents.debugger, main.tree),
|
|
508
|
+
15000, 'get_tree(ax).attachHrefs'
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
return await this._finishAxTree(main, view, tid, opts);
|
|
512
|
+
} catch (e) {
|
|
513
|
+
if (e && e.__deadline) {
|
|
514
|
+
return { tree: null, context: null, error: e.message + ' — the page main thread is likely blocked' };
|
|
515
|
+
}
|
|
516
|
+
throw e;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
async _finishAxTree(main, view, tid, opts) {
|
|
521
|
+
// P0 down-drill: splice OOPIF sub-frame editors into the main AX tree so
|
|
522
|
+
// editors rendered in cross-process iframes (B站 write-article: york/read-draft)
|
|
523
|
+
// become visible to `ui.get_tree {ax:true}`. Each sub-frame is read by
|
|
524
|
+
// executeJavaScript in its own context (top-frame AX + querySelector can't
|
|
525
|
+
// reach OOPIF content), then merged purely by mergeFrameTrees.
|
|
526
|
+
const frames = await this._readOopifFrames(view, tid);
|
|
527
|
+
if (frames.length && main && main.tree) {
|
|
528
|
+
main.tree = axExtractor.mergeFrameTrees(main.tree, frames);
|
|
529
|
+
}
|
|
530
|
+
// #4: incremental diff — return only newly appeared / hidden interactive
|
|
531
|
+
// nodes since the last ax read on this tab (feeds: small payload per scroll).
|
|
532
|
+
if (opts.mode === 'diff') {
|
|
533
|
+
const lite = this._liteFromTree(main.tree);
|
|
534
|
+
const prev = this._axDiffCache.get(tid) || null;
|
|
535
|
+
if (prev && prev.size) {
|
|
536
|
+
this._axDiffCache.set(tid, lite);
|
|
537
|
+
return { tree: null, context: null, diff: axExtractor.diffLite(prev, lite) };
|
|
538
|
+
}
|
|
539
|
+
this._axDiffCache.set(tid, lite);
|
|
540
|
+
}
|
|
541
|
+
// #4: interactive subset — prune layout-only branches to cut token cost.
|
|
542
|
+
if (opts.subset === 'interactive' && main.tree) {
|
|
543
|
+
main.tree = axExtractor.filterInteractive(main.tree);
|
|
544
|
+
}
|
|
545
|
+
// 同 getTree:把 value_fetch_ref 展开成带实际 tab 的 value_fetch。
|
|
546
|
+
if (main && main.tree) expandValueFetchTree(main.tree, tid);
|
|
547
|
+
return main;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// Lite interactive snapshot derived from an already-built tree (no second CDP
|
|
551
|
+
// read). Keyed by backendDOMNodeId for cheap diff across reads.
|
|
552
|
+
_liteFromTree(tree) {
|
|
553
|
+
const out = new Map();
|
|
554
|
+
(function w(t) {
|
|
555
|
+
if (!t) return;
|
|
556
|
+
if (t.backendDOMNodeId != null && ((t.actions || []).length > 0 || t.editor_type)) {
|
|
557
|
+
out.set(t.backendDOMNodeId, { id: t.id, role: t.role, label: t.label || '' });
|
|
558
|
+
}
|
|
559
|
+
for (const c of t.children || []) w(c);
|
|
560
|
+
})(tree);
|
|
561
|
+
return out;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// Read editable/interactive fields out of each out-of-process sub-frame via
|
|
565
|
+
// Frame.executeJavaScript (runs in that frame's own context). Returns
|
|
566
|
+
// [{ key, label, tree }] — pure mergeFrameTrees input. Stamps idempotent
|
|
567
|
+
// data-ai-id = `axf-{frameSeq}-{idx}` inside each frame so the act layer can
|
|
568
|
+
// (a) find the owning frame by testing which frame has `[data-ai-id=...]`, and
|
|
569
|
+
// (b) run its JS in that frame's own context (top querySelector can't reach OOPIF).
|
|
570
|
+
async _readOopifFrames(view, tabId) {
|
|
571
|
+
try {
|
|
572
|
+
const root = view.webContents.mainFrame;
|
|
573
|
+
if (!root || !root.framesInSubtree) return [];
|
|
574
|
+
const out = [];
|
|
575
|
+
let frameSeq = 0;
|
|
576
|
+
for (const frame of root.framesInSubtree) {
|
|
577
|
+
if (frame === root) continue; // main frame handled by AX path
|
|
578
|
+
frameSeq += 1;
|
|
579
|
+
const key = 'frame-' + frameSeq;
|
|
580
|
+
const prefix = 'axf-' + frameSeq + '-';
|
|
581
|
+
try {
|
|
582
|
+
const dto = await frame.executeJavaScript(`(function(prefix){
|
|
583
|
+
if(!document||!document.body) return null;
|
|
584
|
+
var sel='[contenteditable],[role="textbox"],[role="combobox"],textarea,input[type="text"],input[type="title"],input[type="search"],button,[role="button"],a[href]';
|
|
585
|
+
var els=Array.prototype.slice.call(document.querySelectorAll(sel));
|
|
586
|
+
var out=[]; var idx=0;
|
|
587
|
+
for(var k=0;k<els.length;k++){
|
|
588
|
+
var e=els[k];
|
|
589
|
+
if(e.isContentEditable){ // skip editable boxes nested inside a contenteditable root
|
|
590
|
+
var inside=false;for(var p=e.parentElement;p;p=p.parentElement){if(p.isContentEditable&&p!==e){inside=true;break;}}
|
|
591
|
+
}
|
|
592
|
+
var r=e.getBoundingClientRect();
|
|
593
|
+
if(r.width<4||r.height<4)continue; // hidden
|
|
594
|
+
if(e.tagName==='BUTTON'||(e.getAttribute&&e.getAttribute('role')==='button')){} // buttons not relevant for read-editor; keep anyway
|
|
595
|
+
var ai=prefix+idx;
|
|
596
|
+
e.setAttribute('data-ai-id',ai);
|
|
597
|
+
var isEditable=e.isContentEditable||e.tagName==='TEXTAREA'||((e.tagName==='INPUT'));
|
|
598
|
+
var role;
|
|
599
|
+
if(e.isContentEditable)role='textbox';
|
|
600
|
+
else if(e.tagName==='TEXTAREA')role='textbox';
|
|
601
|
+
else if(e.tagName==='INPUT')role='textbox';
|
|
602
|
+
else if(e.tagName==='BUTTON'||e.getAttribute&&e.getAttribute('role')==='button')role='button';
|
|
603
|
+
else if(e.tagName==='SELECT'||e.getAttribute&&e.getAttribute('role')==='combobox')role='select';
|
|
604
|
+
else if(e.tagName==='A')role='link';
|
|
605
|
+
else role='textbox';
|
|
606
|
+
var label=e.getAttribute('aria-label')||e.getAttribute('placeholder')||e.getAttribute('title')||'';
|
|
607
|
+
if(!label&&e.getAttribute&&e.getAttribute('data-placeholder'))label=e.getAttribute('data-placeholder');
|
|
608
|
+
if(!label&&(e.tagName==='BUTTON'||(e.getAttribute&&e.getAttribute('role')==='button'))){label=(e.innerText||e.textContent||'').trim();}
|
|
609
|
+
if(!label&&e.tagName==='A'){label=(e.innerText||e.textContent||'').trim();}
|
|
610
|
+
var url=(e.tagName==='A'&&e.href)||(e.closest&&e.closest('a[href]')?e.closest('a[href]').href:'');
|
|
611
|
+
var editor_type=null;
|
|
612
|
+
if(e.isContentEditable)editor_type='richtext';
|
|
613
|
+
else if(e.tagName==='TEXTAREA')editor_type='textarea';
|
|
614
|
+
else if(e.tagName==='INPUT')editor_type='textbox';
|
|
615
|
+
out.push({id:ai,role:role,label:label,editor_type:editor_type,url:url,bounds:{x:Math.round(r.x),y:Math.round(r.y),width:Math.round(r.width),height:Math.round(r.height)}});
|
|
616
|
+
idx++;
|
|
617
|
+
}
|
|
618
|
+
return out.length?out:null;
|
|
619
|
+
})(${JSON.stringify(prefix)})`);
|
|
620
|
+
if (!dto || !dto.length) continue;
|
|
621
|
+
const tree = {
|
|
622
|
+
id: key,
|
|
623
|
+
role: 'generic',
|
|
624
|
+
label: frame.name || ('frame-' + frameSeq),
|
|
625
|
+
states: [],
|
|
626
|
+
actions: [],
|
|
627
|
+
bounds: { x: 0, y: 0, width: 0, height: 0 },
|
|
628
|
+
children: dto.map((n) => ({
|
|
629
|
+
id: n.id,
|
|
630
|
+
role: n.role,
|
|
631
|
+
label: n.label || '',
|
|
632
|
+
...(n.url ? { url: n.url } : {}),
|
|
633
|
+
states: n.editor_type ? ['editable=' + n.editor_type, 'focusable'] : ['focusable'],
|
|
634
|
+
...(n.editor_type ? { editor_type: n.editor_type } : {}), // first-class, mirrors main AX path
|
|
635
|
+
actions: n.role === 'button' ? ['click', 'focus'] : ['click', 'focus', 'type', 'setContent', 'clear'],
|
|
636
|
+
bounds: n.bounds,
|
|
637
|
+
})),
|
|
638
|
+
};
|
|
639
|
+
out.push({ key, label: frame.name || ('frame-' + frameSeq), tree });
|
|
640
|
+
} catch (e) { /* skip unreachable frame */ }
|
|
641
|
+
}
|
|
642
|
+
return out;
|
|
643
|
+
} catch (e) { return []; }
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// C (plan §9.5): materialize data-ai-id for interactive elements at the act/peek
|
|
647
|
+
// boundary only. Reading (getTreeViaAx/axRead) never touches the DOM. Idempotent:
|
|
648
|
+
// existing handles (default extractor's `e:...`) are preserved, never overwritten.
|
|
649
|
+
async _ensureAxHandles(view, tabId) {
|
|
650
|
+
if (!view || !view.webContents || !view.webContents.debugger) return;
|
|
651
|
+
try {
|
|
652
|
+
this._ensureCdp(tabId, view, 'Accessibility');
|
|
653
|
+
this._ensureCdp(tabId, view, 'DOM');
|
|
654
|
+
await axExtractor.ensureHandles(view.webContents.debugger);
|
|
655
|
+
} catch (e) { /* no CDP: fall through to the preload path */ }
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
// A (plan §9.5): `ui.peek` — safely explore the "next step". Hover-reveal is
|
|
659
|
+
// non-committing (mouseMoved without press), so it never double-clicks or
|
|
660
|
+
// mis-triggers; we return the AX diff (newly revealed / hidden interactive
|
|
661
|
+
// nodes) so the agent can discover folded entries (e.g. B站 写文章) without
|
|
662
|
+
// guessing URLs. Revert moves the pointer away to close the submenu.
|
|
663
|
+
async peek(tabId, target, opts = {}) {
|
|
664
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
665
|
+
const view = this._getView(tid);
|
|
666
|
+
if (!view) return { revealed: [], hidden: [], mode: 'hover', error: 'Tab not found' };
|
|
667
|
+
if (!target) return { revealed: [], hidden: [], mode: 'hover', error: 'No target' };
|
|
668
|
+
// 取消上一次 peek 挂起的自动回撤。回撤是"把鼠标挪走"的延迟动作,若前一次的
|
|
669
|
+
// 定时器还在,它会在本次 peek 已经把鼠标放到新目标上之后才触发,把刚刚探出来的
|
|
670
|
+
// 菜单又关掉 —— 表现为 revertMs=0(语义:别回撤)依然失效。谁后动手谁说了算。
|
|
671
|
+
const staleRevert = this._revertTimers.get(tid);
|
|
672
|
+
if (staleRevert) { clearTimeout(staleRevert); this._revertTimers.delete(tid); }
|
|
673
|
+
const out = { revealed: [], hidden: [], mode: 'hover' };
|
|
674
|
+
try {
|
|
675
|
+
this._ensureCdp(tid, view, 'Accessibility');
|
|
676
|
+
this._ensureCdp(tid, view, 'DOM');
|
|
677
|
+
} catch (e) { return { ...out, error: 'cdp-unavailable' }; }
|
|
678
|
+
const dbg = view.webContents.debugger;
|
|
679
|
+
try {
|
|
680
|
+
const beforeRaw = await axExtractor.axRaw(dbg);
|
|
681
|
+
if (!beforeRaw.length) return out;
|
|
682
|
+
await axExtractor.ensureHandles(dbg); // give the target a stable append id
|
|
683
|
+
const center = await this.evaluate(
|
|
684
|
+
`(function(){var el=document.querySelector('[data-ai-id="${target}"]');if(!el)return null;` +
|
|
685
|
+
`el.scrollIntoView({block:'center',inline:'center',behavior:'instant'});` +
|
|
686
|
+
`var r=el.getBoundingClientRect();return {x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)};})()`,
|
|
687
|
+
tid);
|
|
688
|
+
// 定位不到 target 必须如实报错。此前直接返回未带 error 的空 out,与"hover 了
|
|
689
|
+
// 但这里确实没有可展开项"完全无法区分 —— agent 会误判成"菜单没有更深的入口"
|
|
690
|
+
// 而放弃探索,实际只是 target 拼错/句柄失效。
|
|
691
|
+
if (!center) {
|
|
692
|
+
return { ...out, error: 'target_not_found: no element with data-ai-id=' + String(target) };
|
|
693
|
+
}
|
|
694
|
+
try {
|
|
695
|
+
// 抢焦点会打断人正在做的事(agent 干活时人几乎不能用电脑)。
|
|
696
|
+
// 策略见 _focusWindow:默认只保证窗口可见,不把它拽到最前、不抢焦点。
|
|
697
|
+
this._focusWindow(view);
|
|
698
|
+
} catch (e) {}
|
|
699
|
+
const send = (method, params) => dbg.sendCommand(method, params);
|
|
700
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: center.x, y: center.y });
|
|
701
|
+
// 区分"没传"与"显式传 0":0 是合法语义(hoverMs=0 不等、revertMs=0 不回撤)。
|
|
702
|
+
// 原来的 `Number(x) || 450` 把 0 当 falsy 顶成默认值,revertMs=0 永远失效。
|
|
703
|
+
const msOr = (v, dflt) => (v === undefined || v === null || v === '') ? dflt : Math.max(0, Number(v) || 0);
|
|
704
|
+
const hoverMs = msOr(opts.hoverMs, 450);
|
|
705
|
+
await new Promise((r) => setTimeout(r, hoverMs));
|
|
706
|
+
const afterRaw = await axExtractor.axRaw(dbg);
|
|
707
|
+
const diff = axExtractor.diffAx(beforeRaw, afterRaw);
|
|
708
|
+
out.revealed = diff.revealed;
|
|
709
|
+
out.hidden = diff.hidden;
|
|
710
|
+
// revertMs=0 表示"别回撤,我要接着点" —— 只有显式传 0 才不该回撤,
|
|
711
|
+
// 未传则沿用 1500ms 默认(同上,0 不再是 falsy)。
|
|
712
|
+
const revertMs = msOr(opts.revertMs, 1500);
|
|
713
|
+
if (revertMs > 0) {
|
|
714
|
+
const h = setTimeout(() => {
|
|
715
|
+
this._revertTimers.delete(tid);
|
|
716
|
+
try { send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: 4, y: 4 }); } catch (e) {}
|
|
717
|
+
}, revertMs);
|
|
718
|
+
this._revertTimers.set(tid, h);
|
|
719
|
+
}
|
|
720
|
+
} catch (e) {
|
|
721
|
+
out.error = String((e && e.message) || e).slice(0, 200);
|
|
722
|
+
}
|
|
723
|
+
return out;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* ui.act 的外壳:唯一职责是**保证它一定会在有限时间内返回**。
|
|
728
|
+
*
|
|
729
|
+
* 为什么必须有这一层:`ui.act` 内部会 await CDP 命令(Input.dispatchMouseEvent /
|
|
730
|
+
* Accessibility.getFullAXTree / DOM.getDocument),而这些命令在渲染进程主线程被
|
|
731
|
+
* 占住时**永不返回**—— 事件要等 ack,ack 等主线程空出来。实测一个跑 while 循环
|
|
732
|
+
* 的页面能让 ui.act 45 秒无回包(其余路径各自有 5s 超时并正确报错,见
|
|
733
|
+
* shared/deadline.js 顶部)。后果不是"慢",是 agent 的工具调用永久悬挂、
|
|
734
|
+
* 整个会话死掉,且没有任何错误可看。
|
|
735
|
+
*
|
|
736
|
+
* 放在这一层而不是各个子路径里,是因为挂住的位置不止一个(AX handles、指针目标
|
|
737
|
+
* 解析、拖拽按键分发都可能),逐个补等于逐个漏。
|
|
738
|
+
*/
|
|
739
|
+
async executeAction(action, target, params = {}, tabId) {
|
|
740
|
+
const ms = actionDeadlineMs(action, params);
|
|
741
|
+
try {
|
|
742
|
+
return await withDeadline(
|
|
743
|
+
this._executeActionInner(action, target, params, tabId),
|
|
744
|
+
ms,
|
|
745
|
+
`action:${action}`
|
|
746
|
+
);
|
|
747
|
+
} catch (e) {
|
|
748
|
+
if (e && e.__deadline) {
|
|
749
|
+
// 明确告诉 agent 两件事:① 它该重试/换策略;② 这个动作**可能已经部分生效**
|
|
750
|
+
// (超时时指针可能已经按下、字符可能已经输入了一半)—— 后者必须说,否则
|
|
751
|
+
// agent 会以为页面没被动过而直接重试,造成双倍输入。
|
|
752
|
+
return {
|
|
753
|
+
success: false,
|
|
754
|
+
error: `${e.message} — the page's main thread is likely blocked; the action may have partially applied`,
|
|
755
|
+
timed_out: true,
|
|
756
|
+
deadline_ms: ms,
|
|
757
|
+
action,
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
throw e;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
async _executeActionInner(action, target, params = {}, tabId) {
|
|
765
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
766
|
+
const view = this._getView(tid);
|
|
767
|
+
if (!view) return { success: false, error: 'Tab not found' };
|
|
768
|
+
// press 是唯一不要求目标的动作:快捷键通常挂在 document/window 上(删除节点、
|
|
769
|
+
// 撤销重做、取消选中),发给"当前焦点"就是正确语义。
|
|
770
|
+
// drag 给了显式起点坐标(params.from)时同样不要求目标 —— "从空白处起拖"正是
|
|
771
|
+
// 平移画布、框选这类操作的表达,而空白画布上根本没有可寻址的元素。
|
|
772
|
+
// wheel 也不要求目标:不指定就落在视口中心(缩放画布正是这样用的)。
|
|
773
|
+
const hasExplicitDragFrom = action === 'drag' && params.from && Number.isFinite(Number(params.from.x));
|
|
774
|
+
const targetOptional = action === 'press' || action === 'wheel' || hasExplicitDragFrom;
|
|
775
|
+
if (!target && !targetOptional) return { success: false, error: 'No target' };
|
|
776
|
+
target = target || '';
|
|
777
|
+
|
|
778
|
+
// 只读分支:取 node.value 的完整内容(树里只给前 200 code point)。
|
|
779
|
+
// 不聚焦、不触发 input/change、不改页面,也不走输入动作的 fallback。
|
|
780
|
+
if (action === 'get_value') return await this._getValue(view, target, params, tid);
|
|
781
|
+
|
|
782
|
+
// CDP 受信输入优先(像人操作:受信点击/键盘/文本,对任何框架通用),
|
|
783
|
+
// 无 debugger 时回退 preload 合成事件。
|
|
784
|
+
if (action === 'click' || action === 'hover') {
|
|
785
|
+
// #2: 按目标 URL 点击(复合卡片的通用解法)——agent 想看"点了会去哪个 URL"
|
|
786
|
+
// 就指定 params.url,不靠猜 label 命中卡片里的多个 <a>。先走 URL→链接定位;
|
|
787
|
+
// 未给 url 才退回元素指针点击(原路径,不回归)。
|
|
788
|
+
if (params.url && action === 'click') {
|
|
789
|
+
try {
|
|
790
|
+
await this._ensureAxHandles(view, tid);
|
|
791
|
+
const res = await this._resolveClickByUrl(view, target, params.url, tid);
|
|
792
|
+
if (!res || !res.success) return { ...(res || {}), success: false, opened_tab: null };
|
|
793
|
+
// #3: 若此点击触发了 window.open → 新 tab,记录并默认接管焦点。
|
|
794
|
+
res.opened_tab = this._consumeOpenedTab(tid, params.keep_tab === true);
|
|
795
|
+
return res;
|
|
796
|
+
} catch (e) {
|
|
797
|
+
return { success: false, error: String((e && e.message) || e).slice(0, 200), opened_tab: null };
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
try {
|
|
801
|
+
await this._ensureAxHandles(view, tid);
|
|
802
|
+
const via = await this._cdpPointerTarget(view, action, target, tid);
|
|
803
|
+
if (via) return { success: true, clicked_via: via, target, opened_tab: this._consumeOpenedTab(tid, params.keep_tab === true) };
|
|
804
|
+
} catch (e) { /* fall through to preload */ }
|
|
805
|
+
} else if (action === 'type' || action === 'setContent' || action === 'clear') {
|
|
806
|
+
try {
|
|
807
|
+
await this._ensureAxHandles(view, tid);
|
|
808
|
+
// clear 也走 CDP 受信输入链路(空文本即清空+校验)。顶层预加载 clear 用
|
|
809
|
+
// innerHTML="" 只对顶层元素生效;OOPIF 子帧元素 querySelector 命中不到,
|
|
810
|
+
// clear 会静默落空。路由到这里后 OOPIF 走 _clearInFrame(execCommand
|
|
811
|
+
// selectAll+delete),顶层走 _selectAllFallback+受信 Delete,均带校验。
|
|
812
|
+
// 语义分工(此前 type 与 setContent 完全等价,都是整篇替换 —— 插入话题
|
|
813
|
+
// 标签后再输入正文会把标签清掉,无法续写):
|
|
814
|
+
// type = 在光标处追加(默认末尾),不碰已有内容
|
|
815
|
+
// setContent = 整篇替换(全选 → 删除 → 输入)
|
|
816
|
+
// clear = 全选删除
|
|
817
|
+
// 想重填一个输入框请显式 setContent(或先 clear 再 type)。
|
|
818
|
+
const r = await this._inputViaCdp(
|
|
819
|
+
view,
|
|
820
|
+
target,
|
|
821
|
+
action === 'clear' ? '' : (params.text || ''),
|
|
822
|
+
tid,
|
|
823
|
+
{ replace: action !== 'type', at: params.at || 'end' }
|
|
824
|
+
);
|
|
825
|
+
if (r) return r;
|
|
826
|
+
} catch (e) { /* fall through to preload */ }
|
|
827
|
+
} else if (action === 'upload') {
|
|
828
|
+
try {
|
|
829
|
+
await this._ensureAxHandles(view, tid);
|
|
830
|
+
const r = await this._uploadViaCdp(view, target, params.file || '', tid);
|
|
831
|
+
if (r) return r;
|
|
832
|
+
} catch (e) { /* fall through to preload */ }
|
|
833
|
+
// 无 CDP 时 preload 无法真正赋值文件(受安全限制),返回失败让 agent 感知。
|
|
834
|
+
return { success: false, error: 'Upload requires CDP (input file set) — no file input found' };
|
|
835
|
+
} else if (action === 'drag') {
|
|
836
|
+
try {
|
|
837
|
+
const r = await this._cdpDrag(view, target, params, tid);
|
|
838
|
+
if (r) return { ...r, opened_tab: null };
|
|
839
|
+
} catch (e) {
|
|
840
|
+
return { success: false, error: String((e && e.message) || e).slice(0, 200), opened_tab: null };
|
|
841
|
+
}
|
|
842
|
+
// 拖拽没有 preload 等价物:合成 PointerEvent 不被采用 Pointer Capture 的
|
|
843
|
+
// 应用信任(实测 React Flow 上 moved 恒为 0),因此明确报错而不是假装成功。
|
|
844
|
+
return { success: false, error: 'Drag requires CDP — the debugger is not attached to this tab' };
|
|
845
|
+
} else if (action === 'wheel') {
|
|
846
|
+
try {
|
|
847
|
+
const r = await this._cdpWheel(view, target, params, tid);
|
|
848
|
+
if (r) return { ...r, opened_tab: null };
|
|
849
|
+
} catch (e) {
|
|
850
|
+
return { success: false, error: String((e && e.message) || e).slice(0, 200), opened_tab: null };
|
|
851
|
+
}
|
|
852
|
+
return { success: false, error: 'wheel requires CDP — the debugger is not attached to this tab' };
|
|
853
|
+
} else if (action === 'press') {
|
|
854
|
+
try {
|
|
855
|
+
const r = await this._cdpPress(view, target, params, tid);
|
|
856
|
+
if (r) return { ...r, opened_tab: null };
|
|
857
|
+
} catch (e) {
|
|
858
|
+
return { success: false, error: String((e && e.message) || e).slice(0, 200), opened_tab: null };
|
|
859
|
+
}
|
|
860
|
+
return { success: false, error: 'press requires CDP — the debugger is not attached to this tab' };
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
const preloadRes = await this._executeViaPreload(action, target, params, tid);
|
|
864
|
+
// preload 合成点击也可能触发 window.open → 新 tab,同样回报 opened_tab。
|
|
865
|
+
if (action === 'click' && preloadRes && preloadRes.success) {
|
|
866
|
+
preloadRes.opened_tab = this._consumeOpenedTab(tid, params.keep_tab === true);
|
|
867
|
+
// 路径可区分:CDP 受信点击报 clicked_via='cdp-click',回退路径报
|
|
868
|
+
// 'preload-click'。两种都只报 success 的话,agent(和人)无从判断这一下
|
|
869
|
+
// 到底是真实输入还是合成事件 —— 双发 bug 正是靠这个字段才暴露出来的。
|
|
870
|
+
preloadRes.clicked_via = 'preload-click';
|
|
871
|
+
}
|
|
872
|
+
return preloadRes;
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// === 只读取全量 node.value(ui.act {action:'get_value'})===
|
|
876
|
+
// 语义树里 node.value 只给前 200 Unicode code point;超长的节点带
|
|
877
|
+
// value_fetch 提示,调用方用它取全量(支持 offset/limit 分页)。
|
|
878
|
+
// 严格只读:不聚焦、不派发 input/change、不改 DOM,也不走输入动作 fallback。
|
|
879
|
+
async _getValue(view, target, params, tabId) {
|
|
880
|
+
const p = params || {};
|
|
881
|
+
const offset = Math.max(0, Number(p.offset) || 0);
|
|
882
|
+
const limit = (p.limit === undefined || p.limit === null) ? undefined : Number(p.limit);
|
|
883
|
+
// target 经 JSON.stringify 变成双引号 JS 字面量,拼进单引号选择器里 —— 无法
|
|
884
|
+
// 通过 target 内容跳出字符串。
|
|
885
|
+
const js =
|
|
886
|
+
"(function(){"
|
|
887
|
+
+ "var el=document.querySelector('[data-ai-id=' + " + JSON.stringify(String(target)) + " + ']');"
|
|
888
|
+
+ "if(!el)return {__miss:true};"
|
|
889
|
+
+ "var tag=(el.tagName||'').toUpperCase();"
|
|
890
|
+
+ "if(tag==='INPUT'){var ty=String(el.type||'').toLowerCase();"
|
|
891
|
+
+ "if(ty==='password')return {__sensitive:'password'};"
|
|
892
|
+
+ "if(ty==='file')return {__sensitive:'file'};"
|
|
893
|
+
+ "return {text:el.value==null?'':String(el.value)};}"
|
|
894
|
+
+ "if(tag==='TEXTAREA')return {text:el.value==null?'':String(el.value)};"
|
|
895
|
+
+ "if(tag==='SELECT'){var o=el.selectedOptions&&el.selectedOptions.length?el.selectedOptions[0]:null;"
|
|
896
|
+
+ "return {text:o?String(o.value||o.textContent||''):String(el.value||'')};}"
|
|
897
|
+
+ "if(el.isContentEditable===true||(el.getAttribute&&el.getAttribute('contenteditable')==='true'))"
|
|
898
|
+
+ "return {text:String(el.innerText||el.textContent||'')};"
|
|
899
|
+
+ "if(typeof el.value==='string')return {text:el.value};"
|
|
900
|
+
+ "return {text:String(el.innerText||el.textContent||'')};"
|
|
901
|
+
+ "})()";
|
|
902
|
+
|
|
903
|
+
const run = async () => {
|
|
904
|
+
// OOPIF:在所属子帧上下文里执行(顶层 querySelector 看不到跨进程 iframe)。
|
|
905
|
+
if (/^axf-/.test(target)) {
|
|
906
|
+
const frame = await this._owningFrame(view, target);
|
|
907
|
+
if (frame) {
|
|
908
|
+
try { return await frame.executeJavaScript(js); } catch (e) { /* fall through */ }
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
try { return await this.evaluate(js, tabId); } catch (e) { return null; }
|
|
912
|
+
};
|
|
913
|
+
|
|
914
|
+
let raw = await run();
|
|
915
|
+
if (!raw || raw.__miss) {
|
|
916
|
+
// AX 读层的 handle 是按需 stamp 的(ensureHandles)—— 从未 act 过的节点
|
|
917
|
+
// DOM 上可能还没有 data-ai-id。补一次再读,仍没有才报 not found。
|
|
918
|
+
await this._ensureAxHandles(view, tabId);
|
|
919
|
+
raw = await run();
|
|
920
|
+
}
|
|
921
|
+
if (!raw || raw.__miss) return { success: false, error: 'target_not_found: no element with data-ai-id=' + String(target) };
|
|
922
|
+
if (raw.__sensitive) return { success: false, error: 'sensitive_value_not_readable: input[type=' + raw.__sensitive + '] is never exposed' };
|
|
923
|
+
if (typeof raw.text !== 'string') return { success: false, error: 'target_has_no_value' };
|
|
924
|
+
const sliced = valueContract.getValueResult(raw.text, offset, limit);
|
|
925
|
+
return { success: true, target, ...sliced };
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
// #3 helper: read & clear the "new tab opened by this source tab" entry.
|
|
929
|
+
// Returns { id, url } or null. Unless keep===true, auto-activates the new tab.
|
|
930
|
+
_consumeOpenedTab(tid, keep) {
|
|
931
|
+
const entry = this._openedTabs.get(tid);
|
|
932
|
+
this._openedTabs.delete(tid);
|
|
933
|
+
if (!entry) return null;
|
|
934
|
+
if (keep !== true) this.setActive(entry.id);
|
|
935
|
+
return { id: entry.id, url: entry.url };
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// #2: 按目标 URL 定位并点击链接 —— 通用替代"猜卡片里的 <a>"。顶层在
|
|
939
|
+
// Runtime.evaluate 里找 a[href] 精确匹配 url(或 host/path 子串),命中即
|
|
940
|
+
// scrollIntoView+focus+click(link 原生 click 可靠);OOPIF(target 为 axf-…)
|
|
941
|
+
// 在所属帧内用同一逻辑。无 url 传参/找不到返回 clean error。
|
|
942
|
+
async _resolveClickByUrl(view, target, url, tabId) {
|
|
943
|
+
if (!view || !url) return { success: false, error: 'no url provided' };
|
|
944
|
+
const owning = await this._owningFrameFor(view, target, tabId);
|
|
945
|
+
const expr = `(function(url){
|
|
946
|
+
var best=null,bestScore=-1;
|
|
947
|
+
var als=document.querySelectorAll('a[href]');
|
|
948
|
+
for(var i=0;i<als.length;i++){
|
|
949
|
+
var a=als[i], h=a.href||'';
|
|
950
|
+
if(!h)continue;
|
|
951
|
+
// 精确匹配优先,其次 host+path 子串(容忍站点改写查询串/协议)。
|
|
952
|
+
var s = (h===url) ? 3 : (h.indexOf(url)>0 ? 2 : (url.indexOf(h)>0 ? 1 : 0));
|
|
953
|
+
// 子串还要继续看 path 是否一致,避免 /video 与 /video/xxx 互误伤
|
|
954
|
+
if(s===2&&(h.split('#')[0].split('?')[0]!==url.split('#')[0].split('?')[0]))s=1;
|
|
955
|
+
if(s>bestScore){bestScore=s;best=a;}
|
|
956
|
+
}
|
|
957
|
+
if(!best||bestScore<=0)return {ok:false};
|
|
958
|
+
best.scrollIntoView({block:'center',behavior:'instant'});
|
|
959
|
+
best.focus();best.click();
|
|
960
|
+
return {ok:true,matched:best.href||url};
|
|
961
|
+
})(${JSON.stringify(url)})`;
|
|
962
|
+
// OOPIF / 顶层统一:resolve 结果空串归一为 null;跨帧走 executeJavaScript。
|
|
963
|
+
const result = owning
|
|
964
|
+
? await owning.executeJavaScript(expr)
|
|
965
|
+
: await this._evaluateViaCdp(view, expr);
|
|
966
|
+
if (!result || !result.ok) return { success: false, error: 'no link matches url: ' + url };
|
|
967
|
+
return { success: true, clicked_via: 'urllink', matched: result.matched || url };
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
async _executeViaPreload(action, target, params, tabId) {
|
|
971
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
972
|
+
const view = this._getView(tid);
|
|
973
|
+
if (!view) return { success: false, error: 'Tab not found' };
|
|
974
|
+
|
|
975
|
+
return new Promise((resolve, reject) => {
|
|
976
|
+
const id = ++this._requestId;
|
|
977
|
+
this._pendingRequests.set(id, { resolve, reject });
|
|
978
|
+
view.webContents.send('ai:action', { action, target, params, id });
|
|
979
|
+
setTimeout(() => {
|
|
980
|
+
if (this._pendingRequests.has(id)) {
|
|
981
|
+
this._pendingRequests.delete(id);
|
|
982
|
+
resolve({ success: false, error: 'Action timeout' });
|
|
983
|
+
}
|
|
984
|
+
}, 5000);
|
|
985
|
+
});
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
async getFocused(tabId) {
|
|
989
|
+
const result = await this.getTree(true, tabId);
|
|
990
|
+
return result?.tree?.focused_element_id || null;
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
async evaluate(js, tabId) {
|
|
994
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
995
|
+
const view = this._getView(tid);
|
|
996
|
+
if (!view) throw new Error('Tab not found');
|
|
997
|
+
|
|
998
|
+
// OOPIF routing: if the JS targets a sub-frame `data-ai-id` (`axf-...`), run
|
|
999
|
+
// it inside that frame's own context via Frame.executeJavaScript — the
|
|
1000
|
+
// top-frame CDP Runtime.evaluate cannot see OOPIF (cross-process iframe)
|
|
1001
|
+
// DOM, so probes/`ui.evaluate` would otherwise read `null` back.
|
|
1002
|
+
const m = /data-ai-id="(axf-\d+-\d+)"/.exec(js);
|
|
1003
|
+
if (m) {
|
|
1004
|
+
const frame = await this._owningFrame(view, m[1]);
|
|
1005
|
+
if (frame) {
|
|
1006
|
+
try { return await frame.executeJavaScript(js.startsWith('(') ? js : `(function(){return ${js};})()`); }
|
|
1007
|
+
catch (e) { /* fall through to top-frame path */ }
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
// Preferred path: CDP Runtime.evaluate. It runs on the DevTools protocol
|
|
1011
|
+
// channel and is NOT subject to the page's CSP code-generation (eval)
|
|
1012
|
+
// restrictions, so it works on strict sites (Bing, Gmail, ...) where the
|
|
1013
|
+
// in-page `eval()` used by the preload bridge is blocked. It runs in the
|
|
1014
|
+
// page's main world, identical privilege to the page's own JS (no Node:
|
|
1015
|
+
// nodeIntegration:false + contextIsolation:true are unchanged).
|
|
1016
|
+
try {
|
|
1017
|
+
return await this._evaluateViaCdp(view, js);
|
|
1018
|
+
} catch (e) {
|
|
1019
|
+
// Fallback: the original preload (ai:evaluate) path, for views without a
|
|
1020
|
+
// usable CDP debugger (and for the vitest unit tests that mock IPC).
|
|
1021
|
+
return await this._evaluateViaPreload(view, js);
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
// Execute JS via chromedebugger + Runtime.evaluate on a per-tab webContents.
|
|
1026
|
+
async _evaluateViaCdp(view, js) {
|
|
1027
|
+
const wc = view.webContents;
|
|
1028
|
+
const dbg = wc && wc.debugger;
|
|
1029
|
+
if (!dbg || typeof dbg.isAttached !== 'function') throw new Error('no-cdp');
|
|
1030
|
+
try {
|
|
1031
|
+
if (!dbg.isAttached()) dbg.attach('1.3');
|
|
1032
|
+
} catch (e) {
|
|
1033
|
+
// Already attached (e.g. by the network monitor); reuse the same session.
|
|
1034
|
+
}
|
|
1035
|
+
const cmd = dbg.sendCommand('Runtime.evaluate', {
|
|
1036
|
+
expression: js,
|
|
1037
|
+
returnByValue: true,
|
|
1038
|
+
awaitPromise: false,
|
|
1039
|
+
});
|
|
1040
|
+
const timeout = new Promise((_, rej) =>
|
|
1041
|
+
setTimeout(() => rej(new Error('cdp evaluate timeout')), 5000));
|
|
1042
|
+
const res = await Promise.race([cmd, timeout]);
|
|
1043
|
+
if (res && res.exceptionDetails) {
|
|
1044
|
+
const ex = res.exceptionDetails.exception;
|
|
1045
|
+
const msg = (ex && (ex.description || ex.value)) || res.exceptionDetails.text || 'evaluate error';
|
|
1046
|
+
throw new Error(String(msg).slice(0, 300));
|
|
1047
|
+
}
|
|
1048
|
+
const r = res && res.result;
|
|
1049
|
+
if (r && r.subtype === 'error') throw new Error(String(r.description || 'evaluate error').slice(0, 300));
|
|
1050
|
+
return r ? r.value : undefined;
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
_evaluateViaPreload(view, js) {
|
|
1054
|
+
return new Promise((resolve, reject) => {
|
|
1055
|
+
const id = ++this._requestId;
|
|
1056
|
+
this._pendingRequests.set(id, { resolve, reject });
|
|
1057
|
+
view.webContents.send('ai:evaluate', { js, id });
|
|
1058
|
+
setTimeout(() => {
|
|
1059
|
+
if (this._pendingRequests.has(id)) {
|
|
1060
|
+
this._pendingRequests.delete(id);
|
|
1061
|
+
reject(new Error('evaluate timeout'));
|
|
1062
|
+
}
|
|
1063
|
+
}, 5000);
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// === CDP 受信输入(通用化,无框架特判)===
|
|
1068
|
+
// 任何网页/富文本/编辑器(Draft、ProseMirror、Quill、antd、shadcn…)都无需
|
|
1069
|
+
// 专项适配:点击/键盘/文本全部走 Chromium 真实输入管线(受信事件),等价于
|
|
1070
|
+
// 真实用户操作。识别层面只判断"是不是可编辑区"(contenteditable / input /
|
|
1071
|
+
// textarea),不嗅探任何框架类名;失败只返回错误让 agent 重新读取重试。
|
|
1072
|
+
|
|
1073
|
+
_canCdp(view) {
|
|
1074
|
+
try {
|
|
1075
|
+
const d = view.webContents.debugger;
|
|
1076
|
+
return !!(d && typeof d.isAttached === 'function');
|
|
1077
|
+
} catch(e) { return false; }
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
async _cdpKey(inputCmd, p) {
|
|
1081
|
+
const base = {
|
|
1082
|
+
windowsVirtualKeyCode: p.vk,
|
|
1083
|
+
nativeVirtualKeyCode: p.vk,
|
|
1084
|
+
code: p.code,
|
|
1085
|
+
key: p.key,
|
|
1086
|
+
text: p.text || '',
|
|
1087
|
+
modifiers: p.mod || 0,
|
|
1088
|
+
};
|
|
1089
|
+
await inputCmd('Input.dispatchKeyEvent', { ...base, type: 'rawKeyDown' });
|
|
1090
|
+
// CDP 的 rawKeyDown 会忽略 text —— 带字符的键(换行)必须再发一个带
|
|
1091
|
+
// text 的 char 事件才会真的把字符插进去,否则 Enter 在 textarea /
|
|
1092
|
+
// 原生 contenteditable 里不产生任何换行("a\nb\nc" 静默变成 "abc")。
|
|
1093
|
+
if (base.text) await inputCmd('Input.dispatchKeyEvent', { ...base, type: 'char' });
|
|
1094
|
+
await inputCmd('Input.dispatchKeyEvent', { ...base, type: 'keyUp' });
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
// 解析某个 data-ai-id 属于哪个帧。**不能只认 axf- 前缀**:
|
|
1098
|
+
// DOM 抽取器会给同源 iframe 里的元素也写上普通的 data-ai-id(写在 iframe 自己的
|
|
1099
|
+
// document 上),这些 id 在顶层 document 里查不到 —— 实测 ComfyUI 的工具栏按钮
|
|
1100
|
+
// (节点库 / Zoom In / Fit View)就是这样,ui.act 一直报 "Target not found"。
|
|
1101
|
+
// 所以这里先看顶层,找不到再逐个帧找。返回 null 表示就在主帧里。
|
|
1102
|
+
async _owningFrameFor(view, aiId, tabId) {
|
|
1103
|
+
if (!aiId) return null;
|
|
1104
|
+
if (/^axf-/.test(aiId)) return await this._owningFrame(view, aiId);
|
|
1105
|
+
try {
|
|
1106
|
+
const inTop = await this.evaluate(
|
|
1107
|
+
`!!document.querySelector('[data-ai-id="${aiId}"]')`,
|
|
1108
|
+
tabId
|
|
1109
|
+
);
|
|
1110
|
+
if (inTop) return null;
|
|
1111
|
+
} catch (e) { /* 顶层查询失败就继续找帧 */ }
|
|
1112
|
+
return await this._owningFrame(view, aiId);
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
// Locate the frame that owns a sub-frame handle (`axf-{frameSeq}-{idx}`).
|
|
1116
|
+
// Returns the WebFrameMain, or null when the handle lives in the main frame
|
|
1117
|
+
// (or is unlocatable) — callers fall back to the top-frame path.
|
|
1118
|
+
async _owningFrame(view, aiId) {
|
|
1119
|
+
const root = view.webContents.mainFrame;
|
|
1120
|
+
const frames = root && root.framesInSubtree ? root.framesInSubtree : [];
|
|
1121
|
+
for (const frame of frames) {
|
|
1122
|
+
if (frame === root) continue;
|
|
1123
|
+
try {
|
|
1124
|
+
const has = await frame.executeJavaScript(
|
|
1125
|
+
`!!document.querySelector('[data-ai-id="${aiId}"]')`
|
|
1126
|
+
);
|
|
1127
|
+
if (has) return frame;
|
|
1128
|
+
} catch (e) { /* skip */ }
|
|
1129
|
+
}
|
|
1130
|
+
return null;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// OOPIF coordinate translation: sub-frame getBoundingClientRect is relative to
|
|
1134
|
+
// the iframe's own layout viewport; add the hosting <iframe> element's top-left
|
|
1135
|
+
// (top document) to get browser-viewport coordinates for Input.dispatchMouseEvent.
|
|
1136
|
+
// 找到承载某个子帧的 <iframe> 元素,顺便判断它是否**同源**(同源通常意味着进程内,
|
|
1137
|
+
// 坐标 + CDP 真实鼠标事件可用;跨进程 OOPIF 则只能用帧内原生 .click())。
|
|
1138
|
+
async _hostIframe(view, frame, tabId) {
|
|
1139
|
+
try {
|
|
1140
|
+
const path = await frame.executeJavaScript('(location.pathname||"")');
|
|
1141
|
+
const info = await this.evaluate(
|
|
1142
|
+
`(function(path){
|
|
1143
|
+
var best=null;
|
|
1144
|
+
Array.prototype.forEach.call(document.querySelectorAll('iframe'),function(f){
|
|
1145
|
+
var r=f.getBoundingClientRect();
|
|
1146
|
+
if(r.width<4||r.height<4)return;
|
|
1147
|
+
var s=f.src||'';
|
|
1148
|
+
var match = (f.name&&f.name.length) ? (path.indexOf((f.name||''))===0) : (s.indexOf(path)>=0);
|
|
1149
|
+
if(!match) return;
|
|
1150
|
+
var sameOrigin=false;
|
|
1151
|
+
try { sameOrigin = !!(f.contentDocument && f.contentDocument.body); } catch(e) { sameOrigin=false; }
|
|
1152
|
+
best={x:Math.round(r.left),y:Math.round(r.top),sameOrigin:sameOrigin};
|
|
1153
|
+
});
|
|
1154
|
+
return best;
|
|
1155
|
+
})(${JSON.stringify(path)})`,
|
|
1156
|
+
tabId);
|
|
1157
|
+
return info;
|
|
1158
|
+
} catch (e) { return null; }
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
async _frameOffset(view, frame, tabId) {
|
|
1162
|
+
const info = await this._hostIframe(view, frame, tabId);
|
|
1163
|
+
return info ? { x: info.x, y: info.y } : null;
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
// 受信鼠标事件:先把元素滚入视口(屏外元素直接 dispatchMouseEvent 时坐标落在
|
|
1167
|
+
// 视口外,事件命中 body/空白处——探针证实点击不聚焦却误报成功),再按滚动后的
|
|
1168
|
+
// 中心点发 mousemove → mousePressed/mouseReleased(click)或仅 move(hover)。
|
|
1169
|
+
// OOPIF 目标:定位在所属 frame 内执行,坐标 + frame 偏移换算到顶层视口。
|
|
1170
|
+
async _cdpPointerTarget(view, action, aiId, tabId) {
|
|
1171
|
+
if (!this._canCdp(view)) return null;
|
|
1172
|
+
const owning = await this._owningFrameFor(view, aiId, tabId);
|
|
1173
|
+
const inFrame = !!owning;
|
|
1174
|
+
// 帧内 click 的两种走法,必须分开:
|
|
1175
|
+
// · **同源(进程内)子帧** → 走下面的坐标 + CDP 真实鼠标事件。
|
|
1176
|
+
// 实测 ComfyUI 工具栏的 Zoom In 只监听 mousedown/pointerdown,帧内原生
|
|
1177
|
+
// el.click() 完全不生效(而 Fit View 只监听 click,el.click() 才生效)——
|
|
1178
|
+
// 合成事件覆盖不了全部按钮,真实鼠标事件才能。
|
|
1179
|
+
// · **跨进程 OOPIF** → 沿用帧内原生 .click():CDP 坐标对跨进程子帧的命中/路由
|
|
1180
|
+
// 不稳(实测报告成功却未触发),原生 .click() 贴合编辑器 React click 语义。
|
|
1181
|
+
if (inFrame && action === 'click') {
|
|
1182
|
+
try {
|
|
1183
|
+
// 抢焦点会打断人正在做的事(agent 干活时人几乎不能用电脑)。
|
|
1184
|
+
// 策略见 _focusWindow:默认只保证窗口可见,不把它拽到最前、不抢焦点。
|
|
1185
|
+
this._focusWindow(view);
|
|
1186
|
+
} catch (e) {}
|
|
1187
|
+
let host = null;
|
|
1188
|
+
try { host = await this._hostIframe(view, owning, tabId); } catch (e) { host = null; }
|
|
1189
|
+
if (host && host.sameOrigin) {
|
|
1190
|
+
// 落到这里,走与 hover 相同的坐标路径(真实 mousemove/press/release)
|
|
1191
|
+
} else {
|
|
1192
|
+
try {
|
|
1193
|
+
const ok = await owning.executeJavaScript(
|
|
1194
|
+
`(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return false;` +
|
|
1195
|
+
`el.scrollIntoView({block:'center',inline:'center',behavior:'instant'});` +
|
|
1196
|
+
`el.focus();el.click();return true;})()`);
|
|
1197
|
+
return ok ? 'oopif-click' : null;
|
|
1198
|
+
} catch (e) { return null; }
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
// 定位 → 滚入视口居中 → 返回滚动后的中心点(frame 内局部坐标)。
|
|
1202
|
+
const center = inFrame
|
|
1203
|
+
? await owning.executeJavaScript(
|
|
1204
|
+
`(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return null;` +
|
|
1205
|
+
`el.scrollIntoView({block:'center',inline:'center',behavior:'instant'});` +
|
|
1206
|
+
`var r=el.getBoundingClientRect();return {x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)};})()`)
|
|
1207
|
+
: await this.evaluate(
|
|
1208
|
+
`(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return null;` +
|
|
1209
|
+
`el.scrollIntoView({block:'center',inline:'center',behavior:'instant'});` +
|
|
1210
|
+
`var r=el.getBoundingClientRect();return {x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)};})()`,
|
|
1211
|
+
tabId);
|
|
1212
|
+
if (!center) return null;
|
|
1213
|
+
// frame 局部 → 顶层视口坐标
|
|
1214
|
+
let vp = center;
|
|
1215
|
+
if (inFrame) {
|
|
1216
|
+
const off = await this._frameOffset(view, owning, tabId);
|
|
1217
|
+
if (off) vp = { x: center.x + Math.round(off.x), y: center.y + Math.round(off.y) };
|
|
1218
|
+
}
|
|
1219
|
+
try {
|
|
1220
|
+
// 抢焦点会打断人正在做的事(agent 干活时人几乎不能用电脑)。
|
|
1221
|
+
// 策略见 _focusWindow:默认只保证窗口可见,不把它拽到最前、不抢焦点。
|
|
1222
|
+
this._focusWindow(view);
|
|
1223
|
+
} catch(e) {}
|
|
1224
|
+
const dbg = view.webContents.debugger;
|
|
1225
|
+
const send = (method, params) => dbg.sendCommand(method, params);
|
|
1226
|
+
// 命中校验必须在派发【之前】做,且派发成功就不再回退 —— 否则一次 act 会点两次。
|
|
1227
|
+
// 事后校验的致命漏洞:mousePressed/mouseReleased 一发出,点击就已生效;而"点了
|
|
1228
|
+
// 之后自身消失/收起"的目标(话题候选收起列表、提交后置灰、菜单项、@提及、点赞)
|
|
1229
|
+
// 此刻已经被自己点没了,elementFromPoint 必然命中不到 → 返回 null → executeAction
|
|
1230
|
+
// 回退 preload 再点一次 → DOM 上出现两份插入/两次提交。派发前的坐标命中才是
|
|
1231
|
+
// "这一下会不会落在目标上"的唯一可信判断;校验失败(被遮挡/屏外/已隐藏)才回退。
|
|
1232
|
+
if (action === 'click') {
|
|
1233
|
+
const hit = inFrame
|
|
1234
|
+
? await owning.executeJavaScript(
|
|
1235
|
+
`(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return false;` +
|
|
1236
|
+
`var h=document.elementFromPoint(${center.x},${center.y});return !!(h&&(h===el||el.contains(h)));})()`)
|
|
1237
|
+
: await this.evaluate(
|
|
1238
|
+
`(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return false;` +
|
|
1239
|
+
`var h=document.elementFromPoint(${center.x},${center.y});return !!(h&&(h===el||el.contains(h)));})()`,
|
|
1240
|
+
tabId);
|
|
1241
|
+
if (!hit) return null; // 坐标落不到目标上 —— 交给 preload 的 focus()+click 兜底
|
|
1242
|
+
}
|
|
1243
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: vp.x, y: vp.y });
|
|
1244
|
+
if (action === 'click') {
|
|
1245
|
+
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: vp.x, y: vp.y, button: 'left', buttons: 1, clickCount: 1 });
|
|
1246
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: vp.x, y: vp.y, button: 'left', buttons: 0, clickCount: 1 });
|
|
1247
|
+
}
|
|
1248
|
+
// 走到这里说明受信点击已经真实派发成功 —— 绝不回退 preload 再点第二次。
|
|
1249
|
+
return action === 'hover' ? 'cdp-hover' : 'cdp-click';
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
// 受信滚轮(ui.act {action:'wheel'})。
|
|
1253
|
+
//
|
|
1254
|
+
// 为什么需要:`ui.scroll` 只是 `window.scrollBy`,对**画布类应用完全无效** ——
|
|
1255
|
+
// 无限画布/地图/图表靠 wheel 事件缩放与平移,而不是文档滚动。实测在 rhtv 画布上
|
|
1256
|
+
// ui.scroll 一点用都没有,agent 因此既看不见全图、也缩不进去。
|
|
1257
|
+
// params.dy < 0 → 向上滚 / 缩小;> 0 → 向下滚 / 放大(配合 hold:['Control'])
|
|
1258
|
+
// params.dx 横向滚动
|
|
1259
|
+
// params.at 显式落点坐标;否则落在 target 元素中心(先滚入视口),再否则视口中心
|
|
1260
|
+
// params.keys 修饰键(画布缩放通常要 Control)
|
|
1261
|
+
async _cdpWheel(view, aiId, params, tabId) {
|
|
1262
|
+
if (!this._canCdp(view)) return null;
|
|
1263
|
+
const owning = await this._owningFrameFor(view, aiId, tabId);
|
|
1264
|
+
const run = async (body) => {
|
|
1265
|
+
const fn = `(function(){${body}})()`;
|
|
1266
|
+
return owning ? owning.executeJavaScript(fn) : this.evaluate(fn, tabId);
|
|
1267
|
+
};
|
|
1268
|
+
try {
|
|
1269
|
+
this._focusWindow(view);
|
|
1270
|
+
} catch (e) {}
|
|
1271
|
+
|
|
1272
|
+
const dims = await run('return {w:window.innerWidth,h:window.innerHeight};');
|
|
1273
|
+
let at = null;
|
|
1274
|
+
if (params.at && Number.isFinite(Number(params.at.x))) {
|
|
1275
|
+
at = { x: Math.round(Number(params.at.x)), y: Math.round(Number(params.at.y)) };
|
|
1276
|
+
} else if (aiId) {
|
|
1277
|
+
const c = await run(
|
|
1278
|
+
`var el=document.querySelector(${JSON.stringify(`[data-ai-id="${aiId}"]`)});if(!el)return null;` +
|
|
1279
|
+
`el.scrollIntoView({block:'center',inline:'center',behavior:'instant'});` +
|
|
1280
|
+
`var r=el.getBoundingClientRect();return {x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)};`
|
|
1281
|
+
);
|
|
1282
|
+
if (c) at = c;
|
|
1283
|
+
}
|
|
1284
|
+
if (!at) at = { x: Math.round(dims.w / 2), y: Math.round(dims.h / 2) };
|
|
1285
|
+
if (at.x < 0 || at.y < 0 || at.x > dims.w || at.y > dims.h) {
|
|
1286
|
+
return { success: false, error: `wheel: point (${at.x},${at.y}) is outside the viewport (${dims.w}x${dims.h})`, at };
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
const dx = Number(params.dx) || 0;
|
|
1290
|
+
const dy = Number(params.dy) || 0;
|
|
1291
|
+
if (!dx && !dy) {
|
|
1292
|
+
return { success: false, error: 'wheel: need dx or dy (e.g. dy:-600 zooms/pans up; add hold:["Control"] for canvas zoom)' };
|
|
1293
|
+
}
|
|
1294
|
+
const mod = modifierMask(params.hold || params.keys);
|
|
1295
|
+
const steps = Math.min(20, Math.max(1, Number(params.steps) || 1));
|
|
1296
|
+
const dbg = view.webContents.debugger;
|
|
1297
|
+
for (let i = 0; i < steps; i++) {
|
|
1298
|
+
await dbg.sendCommand('Input.dispatchMouseEvent', {
|
|
1299
|
+
type: 'mouseWheel',
|
|
1300
|
+
x: at.x,
|
|
1301
|
+
y: at.y,
|
|
1302
|
+
deltaX: Math.round(dx / steps),
|
|
1303
|
+
deltaY: Math.round(dy / steps),
|
|
1304
|
+
modifiers: mod,
|
|
1305
|
+
});
|
|
1306
|
+
if (steps > 1) await new Promise((r) => setTimeout(r, 30));
|
|
1307
|
+
}
|
|
1308
|
+
return { success: true, wheeled_via: 'cdp-wheel', at, delta: { x: dx, y: dy }, modifiers: mod, steps };
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
// 受信拖拽(ui.act {action:'drag'})—— 画布/看板/滑块/排序/框选这类交互的
|
|
1312
|
+
// 唯一通路。实测:合成 PointerEvent 拖不动采用 Pointer Capture 的应用
|
|
1313
|
+
// (React Flow 的画布节点,moved 恒为 0,0),因为 setPointerCapture 要求
|
|
1314
|
+
// 真实指针;合成 KeyboardEvent/MouseEvent 同理不被信任。所以拖拽**必须**
|
|
1315
|
+
// 走 CDP 的 Input.dispatchMouseEvent 序列,不是"有更好"。
|
|
1316
|
+
//
|
|
1317
|
+
// 端点解析:
|
|
1318
|
+
// target 起点元素(data-ai-id)
|
|
1319
|
+
// params.from / params.to {x,y} 视口绝对坐标(给了就优先,用于空白处起拖/落点)
|
|
1320
|
+
// params.to_target 终点元素(data-ai-id)
|
|
1321
|
+
// params.dx / params.dy 相对起点的位移(移动节点最常用)
|
|
1322
|
+
// params.from_anchor/to_anchor 'center'(默认)|'left'|'right'|'top'|'bottom'
|
|
1323
|
+
// 方位锚点会优先落在元素内部的**连接点**(类名含 handle/port/anchor/
|
|
1324
|
+
// connector 的小元素)上 —— 这正是"从 A 的右侧连到 B 的左侧"的自然表达,
|
|
1325
|
+
// 不需要 agent 知道连接点的 DOM 类名或 data-ai-id(它们通常没有)。
|
|
1326
|
+
// params.steps / params.duration_ms 中间 mouseMoved 步数与总时长
|
|
1327
|
+
// params.keys 拖拽期间按住的修饰键(如 ['Shift'] 轴向锁定)
|
|
1328
|
+
async _cdpDrag(view, aiId, params, tabId) {
|
|
1329
|
+
if (!this._canCdp(view)) return null;
|
|
1330
|
+
const owning = await this._owningFrameFor(view, aiId, tabId);
|
|
1331
|
+
const run = async (body) => {
|
|
1332
|
+
const fn = `(function(){${body}})()`;
|
|
1333
|
+
return owning ? owning.executeJavaScript(fn) : this.evaluate(fn, tabId);
|
|
1334
|
+
};
|
|
1335
|
+
try {
|
|
1336
|
+
this._focusWindow(view);
|
|
1337
|
+
} catch (e) {}
|
|
1338
|
+
|
|
1339
|
+
const anchorOf = (a) => {
|
|
1340
|
+
const s = String(a || 'center').toLowerCase();
|
|
1341
|
+
return ['center', 'left', 'right', 'top', 'bottom'].indexOf(s) >= 0 ? s : 'center';
|
|
1342
|
+
};
|
|
1343
|
+
// 元素内的锚点解析(页内执行)。
|
|
1344
|
+
// center → 元素的几何中心,**不吸附**任何内部元素。这是"移动/平移"的语义,
|
|
1345
|
+
// 若吸附到连接点,在 React Flow 这类库里会变成"从连接点拉线",
|
|
1346
|
+
// 与调用方的意图相反。
|
|
1347
|
+
// 四周 → 优先该方位的**连接点**(handle/port/connector 类的小元素),
|
|
1348
|
+
// 没有则退化为该边的中点。这是"连线"的语义。
|
|
1349
|
+
// 连接点识别用 classList 逐 token 词边界匹配 —— 上一版用
|
|
1350
|
+
// `[class*="anchor"]` 会把 `is-node-anchored` 这类无关类名当成连接点
|
|
1351
|
+
// (实测抢到了"标签"按钮),拖拽起点完全错位。
|
|
1352
|
+
const pointFor = async (sel, anchor, offset) => {
|
|
1353
|
+
const body =
|
|
1354
|
+
`var el=document.querySelector(${JSON.stringify(sel)});if(!el)return null;` +
|
|
1355
|
+
`var r=el.getBoundingClientRect();var side=${JSON.stringify(anchor)};` +
|
|
1356
|
+
`var cx=r.x+r.width/2,cy=r.y+r.height/2;` +
|
|
1357
|
+
`function isHandle(h){var cl=h.classList;for(var i=0;i<cl.length;i++){var t=cl[i];` +
|
|
1358
|
+
`if(/(^|[-_])handle([-_]|$)/.test(t)||/(^|[-_])port([-_]|$)/.test(t)||` +
|
|
1359
|
+
`/(^|[-_])connector([-_]|$)/.test(t)||/(^|[-_])anchor([-_]|$)/.test(t))return true;}return false;}` +
|
|
1360
|
+
`if(side==='center')return {x:cx,y:cy,via:'center'};` +
|
|
1361
|
+
`var M=12;` + // 连接点通常骑在边框上(一半在内一半在外),给一点容差
|
|
1362
|
+
`var hs=el.querySelectorAll('[data-handlepos],[class]');` +
|
|
1363
|
+
`var best=null,bestScore=-1e9;` +
|
|
1364
|
+
`for(var i=0;i<hs.length;i++){var h=hs[i];` +
|
|
1365
|
+
`if(!h.hasAttribute('data-handlepos')&&!isHandle(h))continue;` +
|
|
1366
|
+
`var hr=h.getBoundingClientRect();if(hr.width<=0&&hr.height<=0)continue;` +
|
|
1367
|
+
`var hx=hr.x+hr.width/2,hy=hr.y+hr.height/2;` +
|
|
1368
|
+
// 连接点必须附着在元素**自身**范围内。不加这条,元素内部某个用绝对定位
|
|
1369
|
+
// 甩到很远的子孙会被当成"最右侧的连接点"(实测解析出的点离节点 300px,
|
|
1370
|
+
// 于是连线整条落空、还报 success)。
|
|
1371
|
+
`if(hx<r.left-M||hx>r.right+M||hy<r.top-M||hy>r.bottom+M)continue;` +
|
|
1372
|
+
`var sc;` +
|
|
1373
|
+
`if(side==='right')sc=(hx-cx);else if(side==='left')sc=(cx-hx);` +
|
|
1374
|
+
`else if(side==='bottom')sc=(hy-cy);else if(side==='top')sc=(cy-hy);` +
|
|
1375
|
+
`if(sc>bestScore){bestScore=sc;best={x:hx,y:hy,via:'handle',el:h};}}` +
|
|
1376
|
+
// 自证可命中:选中的"连接点"必须真的是指针落在那个坐标时会命中的元素。
|
|
1377
|
+
// 很多站点把连接点画成 `pointer-events:none` + `opacity:0` 的**装饰**
|
|
1378
|
+
// (实测 rhtv 的画布:handle 全是 pe:none / op:0),此时按下去命中的其实是
|
|
1379
|
+
// 节点本体 —— 却仍报 via:'handle',把"按错了地方"伪装成"按在连接点上"。
|
|
1380
|
+
//
|
|
1381
|
+
// 判定必须是"命中即是连接点,或连接点的子孙"。**不能**接受
|
|
1382
|
+
// `hit.contains(handle)`:节点本体当然包含自己的连接点,那样会把这个
|
|
1383
|
+
// 失败情形原样放行(第一版就是这么写的,实测仍然报 handle)。
|
|
1384
|
+
`if(best){var hit=document.elementFromPoint(best.x,best.y);` +
|
|
1385
|
+
`if(!(hit&&(hit===best.el||best.el.contains(hit))))best=null;}` +
|
|
1386
|
+
`if(best){delete best.el;return best;}` +
|
|
1387
|
+
`if(side==='right')return {x:r.right,y:r.y+r.height/2,via:'edge'};` +
|
|
1388
|
+
`if(side==='left')return {x:r.left,y:r.y+r.height/2,via:'edge'};` +
|
|
1389
|
+
`if(side==='bottom')return {x:r.x+r.width/2,y:r.bottom,via:'edge'};` +
|
|
1390
|
+
`return {x:r.x+r.width/2,y:r.y,via:'edge'};`;
|
|
1391
|
+
const p = await run(body);
|
|
1392
|
+
if (!p) return null;
|
|
1393
|
+
const ox = offset && Number.isFinite(Number(offset.x)) ? Number(offset.x) : 0;
|
|
1394
|
+
const oy = offset && Number.isFinite(Number(offset.y)) ? Number(offset.y) : 0;
|
|
1395
|
+
return { x: Math.round(p.x + ox), y: Math.round(p.y + oy), via: p.via };
|
|
1396
|
+
};
|
|
1397
|
+
|
|
1398
|
+
// 起点/落点上"实际是哪个元素"——拖拽最容易悄悄打偏(坐标落到了浮层、
|
|
1399
|
+
// 遮挡层或相邻的另一个节点上),不报出来就又是一次"报成功但没做对"。
|
|
1400
|
+
// 同时向上找最近的 data-ai-id:如果它**不是**本次目标,说明抓错了节点
|
|
1401
|
+
// (实测踩过:两个节点视觉重叠时,拖 A 实际按在叠在上面的 B 的浮层上,
|
|
1402
|
+
// 事件照常派发、success 照常返回,但什么都没动)。
|
|
1403
|
+
const atPointJs = (p, selfId) =>
|
|
1404
|
+
`var e=document.elementFromPoint(${p.x},${p.y});if(!e)return null;` +
|
|
1405
|
+
`var anc=e.closest?e.closest('[data-ai-id]'):null;` +
|
|
1406
|
+
`var id=anc?anc.getAttribute('data-ai-id'):null;` +
|
|
1407
|
+
`return {el:(e.tagName||'')+' '+(typeof e.className==='string'?e.className:'').slice(0,60),` +
|
|
1408
|
+
`id:id,is_target:(id===null||id===${JSON.stringify(String(selfId))})};`;
|
|
1409
|
+
|
|
1410
|
+
// 起点
|
|
1411
|
+
let from = null;
|
|
1412
|
+
if (params.from && Number.isFinite(Number(params.from.x))) {
|
|
1413
|
+
from = { x: Math.round(Number(params.from.x)), y: Math.round(Number(params.from.y)), via: 'explicit' };
|
|
1414
|
+
} else if (aiId) {
|
|
1415
|
+
from = await pointFor(`[data-ai-id="${aiId}"]`, anchorOf(params.from_anchor), params.from_offset);
|
|
1416
|
+
}
|
|
1417
|
+
if (!from) return { success: false, error: 'drag: start element not found: ' + aiId };
|
|
1418
|
+
|
|
1419
|
+
// 终点:显式坐标 > to_target 元素 > dx/dy 相对位移
|
|
1420
|
+
// 绝对坐标同时接受 {to:{x,y}} 与扁平的 to_x/to_y —— MCP 的 schema 用扁平写法
|
|
1421
|
+
// (更好填),raw WS 习惯嵌套写法。两种都在**这里**归一,避免"某一层认识、
|
|
1422
|
+
// 另一层不认识"的隐性不一致(实测踩过:raw WS 传 to_x 被当成没给落点)。
|
|
1423
|
+
let to = null;
|
|
1424
|
+
const explicitTo =
|
|
1425
|
+
params.to && Number.isFinite(Number(params.to.x))
|
|
1426
|
+
? params.to
|
|
1427
|
+
: (Number.isFinite(Number(params.to_x)) || Number.isFinite(Number(params.to_y)))
|
|
1428
|
+
? { x: params.to_x, y: params.to_y }
|
|
1429
|
+
: null;
|
|
1430
|
+
if (explicitTo && Number.isFinite(Number(explicitTo.x)) && Number.isFinite(Number(explicitTo.y))) {
|
|
1431
|
+
to = { x: Math.round(Number(explicitTo.x)), y: Math.round(Number(explicitTo.y)), via: 'explicit' };
|
|
1432
|
+
} else if (params.to_target) {
|
|
1433
|
+
to = await pointFor(`[data-ai-id="${params.to_target}"]`, anchorOf(params.to_anchor), params.to_offset);
|
|
1434
|
+
if (!to) return { success: false, error: 'drag: to_target not found: ' + params.to_target };
|
|
1435
|
+
} else if (Number.isFinite(Number(params.dx)) || Number.isFinite(Number(params.dy))) {
|
|
1436
|
+
to = { x: Math.round(from.x + Number(params.dx || 0)), y: Math.round(from.y + Number(params.dy || 0)), via: 'delta' };
|
|
1437
|
+
}
|
|
1438
|
+
if (!to) return { success: false, error: 'drag: need one of params.to / to_target / dx,dy' };
|
|
1439
|
+
|
|
1440
|
+
// 视口校验:视口外的点直接派发会落到别的元素上,必须明确报错而不是静默误报成功。
|
|
1441
|
+
const dims = await run('return {w:window.innerWidth,h:window.innerHeight};');
|
|
1442
|
+
const inVp = (p) => p && p.x >= 0 && p.y >= 0 && p.x <= dims.w && p.y <= dims.h;
|
|
1443
|
+
if (!inVp(from)) {
|
|
1444
|
+
return { success: false, error: `drag: start point (${from.x},${from.y}) is outside the viewport (${dims.w}x${dims.h})`, drag_from: from };
|
|
1445
|
+
}
|
|
1446
|
+
if (!inVp(to)) {
|
|
1447
|
+
return { success: false, error: `drag: end point (${to.x},${to.y}) is outside the viewport (${dims.w}x${dims.h})`, drag_from: from, drag_to: to };
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
// 修饰键 → CDP modifiers 位掩码
|
|
1451
|
+
const mod = modifierMask(params.hold || params.keys);
|
|
1452
|
+
|
|
1453
|
+
const dbg = view.webContents.debugger;
|
|
1454
|
+
const send = (m, p) => dbg.sendCommand(m, p);
|
|
1455
|
+
// 按键:画布类应用常把平移绑在中键/右键拖上(左键留给框选),所以要能选按键。
|
|
1456
|
+
const btnParam = String(params.button || 'left').toLowerCase();
|
|
1457
|
+
const btn = btnParam === 'middle' ? 'middle' : btnParam === 'right' ? 'right' : 'left';
|
|
1458
|
+
const downButtons = btn === 'middle' ? 4 : btn === 'right' ? 2 : 1;
|
|
1459
|
+
const steps = Math.min(60, Math.max(2, Number(params.steps) || 14));
|
|
1460
|
+
const total = Math.min(5000, Math.max(60, Number(params.duration_ms) || 420));
|
|
1461
|
+
const per = Math.max(4, Math.round(total / steps));
|
|
1462
|
+
|
|
1463
|
+
// 记录起点/落点上到底是哪个元素:命中遮挡层或**另一个节点**时,调用方能立刻
|
|
1464
|
+
// 看出这一拖是不是打偏了,而不是拿到一个笼统的 success 去猜。
|
|
1465
|
+
const fromAt = await run(atPointJs(from, aiId)).catch(() => null);
|
|
1466
|
+
const toAt = await run(atPointJs(to, aiId)).catch(() => null);
|
|
1467
|
+
// 起点抓到了别的节点 = 视觉重叠下的抓错目标。事件会照常派发、页面却纹丝不动,
|
|
1468
|
+
// 必须显式告诉调用方,否则调用方只会以为"拖拽不生效"。
|
|
1469
|
+
const startOffTarget = !!(fromAt && fromAt.id && fromAt.id !== aiId);
|
|
1470
|
+
|
|
1471
|
+
// 同坐标先 move 一次:让页面进入 hover 态(部分实现只在 hover 后才可拖)。
|
|
1472
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: from.x, y: from.y, modifiers: mod });
|
|
1473
|
+
await new Promise((r) => setTimeout(r, 30));
|
|
1474
|
+
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: from.x, y: from.y, button: btn, buttons: downButtons, clickCount: 1, modifiers: mod });
|
|
1475
|
+
// 按下后必须在**同一点**再补一次 mouseMoved:拖拽实现普遍以"按下后的第一个
|
|
1476
|
+
// mousemove"为位移基准点,若直接进位移循环,这一步的偏移会被吃掉 —— 实测
|
|
1477
|
+
// 请求 (-180,150) 只走到 (-167,139),恰好是 13/14,稳定复现。
|
|
1478
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: from.x, y: from.y, button: btn, buttons: downButtons, modifiers: mod });
|
|
1479
|
+
await new Promise((r) => setTimeout(r, Math.max(20, per)));
|
|
1480
|
+
// 中间位移是拖拽的本质:没有它,mousePressed/mouseReleased 同坐标就只是"点击"。
|
|
1481
|
+
for (let i = 1; i <= steps; i++) {
|
|
1482
|
+
const t = i / steps;
|
|
1483
|
+
const x = Math.round(from.x + (to.x - from.x) * t);
|
|
1484
|
+
const y = Math.round(from.y + (to.y - from.y) * t);
|
|
1485
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y, button: btn, buttons: downButtons, modifiers: mod });
|
|
1486
|
+
await new Promise((r) => setTimeout(r, per));
|
|
1487
|
+
}
|
|
1488
|
+
// 收尾再补一次终点位置并等一拍再释放:实测「最后一步 mouseMoved 会被紧跟着的
|
|
1489
|
+
// mouseReleased 吃掉」,位移会稳定少 1/steps(14 步时实测 -180,150 只走了
|
|
1490
|
+
// -167,139 = 13/14)。多补一次终点让落点精确。
|
|
1491
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: to.x, y: to.y, button: btn, buttons: downButtons, modifiers: mod });
|
|
1492
|
+
await new Promise((r) => setTimeout(r, Math.max(30, per)));
|
|
1493
|
+
// 终点"停稳":再补几个亚像素级的微动。高缩放的画布上连接点只有 1–3 像素,
|
|
1494
|
+
// 一次到点未必被框架的命中判定捕捉到(实测 8.3% 缩放的 React Flow 画布上
|
|
1495
|
+
// 连线时好时坏);人手在目标上也会有这种细微停顿/抖动。
|
|
1496
|
+
const settle = Number(params.settle_moves) >= 0 ? Number(params.settle_moves) : 3;
|
|
1497
|
+
for (let k = 0; k < settle; k++) {
|
|
1498
|
+
const jx = to.x + (k % 2 === 0 ? 1 : -1);
|
|
1499
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: jx, y: to.y, button: btn, buttons: downButtons, modifiers: mod });
|
|
1500
|
+
await new Promise((r) => setTimeout(r, Math.max(12, Math.round(per / 3))));
|
|
1501
|
+
}
|
|
1502
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: to.x, y: to.y, button: btn, buttons: downButtons, modifiers: mod });
|
|
1503
|
+
await new Promise((r) => setTimeout(r, Math.max(20, Math.round(per / 2))));
|
|
1504
|
+
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: to.x, y: to.y, button: btn, buttons: 0, clickCount: 1, modifiers: mod });
|
|
1505
|
+
return {
|
|
1506
|
+
success: true,
|
|
1507
|
+
dragged_via: 'cdp-drag',
|
|
1508
|
+
button: btn,
|
|
1509
|
+
target: aiId,
|
|
1510
|
+
drag_from: from,
|
|
1511
|
+
drag_to: to,
|
|
1512
|
+
from_at: fromAt && fromAt.el,
|
|
1513
|
+
from_at_id: fromAt && fromAt.id,
|
|
1514
|
+
to_at: toAt && toAt.el,
|
|
1515
|
+
to_at_id: toAt && toAt.id,
|
|
1516
|
+
start_hit_target: !startOffTarget,
|
|
1517
|
+
steps,
|
|
1518
|
+
// 抓在了别的元素上:这一拖很可能什么也没动。明确警告而不是让调用方
|
|
1519
|
+
// 从"什么都没发生"里自己猜原因。
|
|
1520
|
+
warning: startOffTarget
|
|
1521
|
+
? `drag started on a different element (${fromAt.id}) than the target (${aiId}) — often caused by visual overlap; pass from_offset to aim at another point of the element`
|
|
1522
|
+
: undefined,
|
|
1523
|
+
};
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
// 受信按键(ui.act {action:'press'})——删除节点/撤销重做/取消选中/方向键微调
|
|
1527
|
+
// 这类应用快捷键的唯一通路。target 可选:给了就先在页内聚焦它,否则按键发给
|
|
1528
|
+
// 当前焦点(多数画布应用的快捷键挂在 document/window 上,先点一下画布即可)。
|
|
1529
|
+
async _cdpPress(view, aiId, params, tabId) {
|
|
1530
|
+
if (!this._canCdp(view)) return null;
|
|
1531
|
+
const owning = await this._owningFrameFor(view, aiId, tabId);
|
|
1532
|
+
const run = async (body) => {
|
|
1533
|
+
const fn = `(function(){${body}})()`;
|
|
1534
|
+
return owning ? owning.executeJavaScript(fn) : this.evaluate(fn, tabId);
|
|
1535
|
+
};
|
|
1536
|
+
try {
|
|
1537
|
+
this._focusWindow(view);
|
|
1538
|
+
} catch (e) {}
|
|
1539
|
+
if (aiId) {
|
|
1540
|
+
await run(`var el=document.querySelector(${JSON.stringify('[data-ai-id="' + String(aiId) + '"]')});if(el&&el.focus)el.focus();return true;`);
|
|
1541
|
+
}
|
|
1542
|
+
const list = Array.isArray(params.keys) && params.keys.length
|
|
1543
|
+
? params.keys
|
|
1544
|
+
: [params.key !== undefined ? params.key : 'Enter'];
|
|
1545
|
+
const dbg = view.webContents.debugger;
|
|
1546
|
+
const send = (m, p) => dbg.sendCommand(m, p);
|
|
1547
|
+
const done = [];
|
|
1548
|
+
for (const name of list) {
|
|
1549
|
+
const spec = keySpec(name, params.modifiers);
|
|
1550
|
+
if (!spec) return { success: false, error: 'press: unsupported key: ' + String(name), pressed: done };
|
|
1551
|
+
await this._cdpKey(send, spec);
|
|
1552
|
+
done.push(spec.key);
|
|
1553
|
+
await new Promise((r) => setTimeout(r, Math.max(20, Number(params.interval_ms) || 60)));
|
|
1554
|
+
}
|
|
1555
|
+
return { success: true, pressed_via: 'cdp-key', pressed: done };
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
// 文件上传(封面/附件等,跨框架通用):HTMLInputElement[type=file] 只能由
|
|
1559
|
+
// DevTools 的 DOM.setFileInputFiles 写入真实文件路径(网页脚本无法伪造
|
|
1560
|
+
// FileList,preload 同样受限),对任何站点/上传组件一视同仁。找到目标 input
|
|
1561
|
+
// 的 objectId 后直接注入文件路径,等价于用户在系统文件选择器里选中。
|
|
1562
|
+
async _uploadViaCdp(view, aiId, filePath, tabId) {
|
|
1563
|
+
if (!this._canCdp(view)) return null;
|
|
1564
|
+
if (!filePath) throw new Error('upload requires a file path');
|
|
1565
|
+
const dbg = view.webContents.debugger;
|
|
1566
|
+
// 用 Runtime.evaluate 拿 objectId,不依赖 DOM nodeId 映射。不额外 click:
|
|
1567
|
+
// DOM.setFileInputFiles 会直接在目标 input 上写入文件并触发 change(等价于
|
|
1568
|
+
// 用户在系统选择器选中);若再对它 click 会重新唤起系统文件框并清空 files,
|
|
1569
|
+
// 导致刚写入的状态被覆盖。由调用方负责先让上传组件就绪。
|
|
1570
|
+
const idRes = await dbg.sendCommand('Runtime.evaluate', {
|
|
1571
|
+
expression: `(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');return el&&el.__proto__&&el.matches?el.matches('input[type=file],input')?el:null:null;})()`,
|
|
1572
|
+
returnByValue: false,
|
|
1573
|
+
});
|
|
1574
|
+
const objId = idRes && idRes.result && idRes.result.objectId;
|
|
1575
|
+
if (!objId) return null;
|
|
1576
|
+
await dbg.sendCommand('DOM.setFileInputFiles', { files: [filePath], objectId: objId });
|
|
1577
|
+
// 兜底校验:确认 files 已写入
|
|
1578
|
+
const ver = await dbg.sendCommand('Runtime.evaluate', {
|
|
1579
|
+
expression: `(function(){var f=document.querySelector('[data-ai-id="${aiId}"]').files;return f&&f.length>0?{name:f[0].name,size:f[0].size}:null;})()`,
|
|
1580
|
+
returnByValue: true,
|
|
1581
|
+
});
|
|
1582
|
+
const v = ver && ver.result && ver.result.value;
|
|
1583
|
+
if (!v) return { success: false, error: 'Upload not written to input' };
|
|
1584
|
+
return { success: true, uploaded: v.name, size: v.size };
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
// 通用文本输入(type/setContent 共用):先让窗口/视图拿到焦点,再在页面里
|
|
1588
|
+
// DOM 原生聚焦(一次 evaluate 同步完成)→ CDP 受信 Cmd+A 全选 + Delete 清空
|
|
1589
|
+
// → 逐字符受信键入 + Enter。清空必须走编辑器自身的受信按键链路:Draft 等
|
|
1590
|
+
// 编辑器是受控组件,DOM 直接改(execCommand/clearContent)不会同步内部
|
|
1591
|
+
// EditorState,后续输入会被忽略(知乎正文 execCommand 清空后输入丢失证实)。
|
|
1592
|
+
// Cmd+A 触发编辑器自身 selectAll,Delete 触发 removeRange——但 Draft 的选区
|
|
1593
|
+
// 更新是异步的(React 渲染提交),Delete 必须在全选提交后发出,否则落在旧
|
|
1594
|
+
// 选区上。逐字符走 Input.dispatchKeyEvent(rawKeyDown/char/keyUp,仅 char
|
|
1595
|
+
// 携带 text),等价真实键盘输入,编辑器会应用当前行内样式(加粗/斜体);
|
|
1596
|
+
// Input.insertText 是 IME 插入路径,会丢失行内样式。
|
|
1597
|
+
// 不识别编辑器框架;失败返回错误,由 agent 重读重试。
|
|
1598
|
+
async _inputViaCdp(view, aiId, text, tabId, opts = {}) {
|
|
1599
|
+
if (!this._canCdp(view)) return null;
|
|
1600
|
+
const owning = await this._owningFrameFor(view, aiId, tabId);
|
|
1601
|
+
const inFrame = !!owning;
|
|
1602
|
+
// Run a JS snippet either in the owning frame (OOPIF) or the top frame.
|
|
1603
|
+
const run = async (jsBody) => {
|
|
1604
|
+
const fn = `(function(){${jsBody}})()`;
|
|
1605
|
+
return inFrame ? owning.executeJavaScript(fn) : this.evaluate(fn, tabId);
|
|
1606
|
+
};
|
|
1607
|
+
// 先激活窗口/视图,再在页面内聚焦(顺序保证渲染进程处于激活态)。
|
|
1608
|
+
// 被遮挡时窗口 visibilityState=hidden,键盘事件同样会被渲染器丢弃。
|
|
1609
|
+
try {
|
|
1610
|
+
// 抢焦点会打断人正在做的事(agent 干活时人几乎不能用电脑)。
|
|
1611
|
+
// 策略见 _focusWindow:默认只保证窗口可见,不把它拽到最前、不抢焦点。
|
|
1612
|
+
this._focusWindow(view);
|
|
1613
|
+
} catch(e) {}
|
|
1614
|
+
// 定位可编辑区并原生聚焦(不做 DOM 全选——交给 Cmd+A 受信按键)
|
|
1615
|
+
const prep = await run(
|
|
1616
|
+
`var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return {ok:false,err:'no-ref'};` +
|
|
1617
|
+
`var ed=el;if(ed.querySelector){var inner=ed.querySelector('[contenteditable=true],input,textarea');if(inner)ed=inner;}` +
|
|
1618
|
+
`var editable=(ed.contentEditable==='true')||(ed.tagName==='TEXTAREA')||(ed.tagName==='INPUT');` +
|
|
1619
|
+
`if(!editable)return {ok:false,err:'not-editable'};` +
|
|
1620
|
+
`ed.focus();` +
|
|
1621
|
+
`return {ok:true};`
|
|
1622
|
+
);
|
|
1623
|
+
if (!prep || !prep.ok) return null;
|
|
1624
|
+
// 清空收尾:拆掉残留的空块外壳。selectAll + 受信 Delete 只删文本不删外层块
|
|
1625
|
+
// 标签,浏览器会保留至少一个空块承载光标 —— 清一个 H2 段落得到的是
|
|
1626
|
+
// `<h2><br></h2>` 而不是空。残留外壳会让随后的排版一层层嵌进旧外壳(实测出现
|
|
1627
|
+
// h2>blockquote>pre>ul>li 的套娃),editor_blocks 的 role 也随之失真。
|
|
1628
|
+
// 只在"确实已经没有文本内容"时拆,有内容的块一个都不动。
|
|
1629
|
+
const stripEmptyShells = () => run(
|
|
1630
|
+
`var el=document.querySelector(${JSON.stringify('[data-ai-id="' + String(aiId) + '"]')});if(!el)return false;` +
|
|
1631
|
+
`var ed=el;if(ed.querySelector){var inner=ed.querySelector('[contenteditable=true],input,textarea');if(inner)ed=inner;}` +
|
|
1632
|
+
`if(ed.tagName==='INPUT'||ed.tagName==='TEXTAREA')return false;` +
|
|
1633
|
+
`if(ed.contentEditable!=='true')return false;` +
|
|
1634
|
+
`if((ed.innerText||'').trim()!=='')return false;` +
|
|
1635
|
+
`var BLOCK=/^(H1|H2|H3|H4|H5|H6|P|BLOCKQUOTE|PRE|UL|OL|LI|DIV|SECTION|ARTICLE)$/;` +
|
|
1636
|
+
`for(var g=0;g<12;g++){` +
|
|
1637
|
+
`if(ed.children.length!==1)break;` +
|
|
1638
|
+
`var k=ed.children[0];` +
|
|
1639
|
+
`if(!BLOCK.test(k.tagName))break;` +
|
|
1640
|
+
`if((k.innerText||'').trim()!=='')break;` +
|
|
1641
|
+
`while(k.firstChild)ed.insertBefore(k.firstChild,k);` +
|
|
1642
|
+
`ed.removeChild(k);}` +
|
|
1643
|
+
`return true;`
|
|
1644
|
+
);
|
|
1645
|
+
// clear 时先拆一遍:编辑器可能已经是"空但带外壳"的状态,不拆的话下面的
|
|
1646
|
+
// 幂等短路会把它当成"已经是空的"直接返回,外壳就永远留在那儿。
|
|
1647
|
+
if (!text) await stripEmptyShells();
|
|
1648
|
+
// 校验用的归一化。\r\n 统一成 \n,再压掉"排版空白"(空格/tab),但换行
|
|
1649
|
+
// 本身必须参与比对:把 \s 全剥掉会让 "a\nb\nc" 与 "abc" 等价,段落结构
|
|
1650
|
+
// 丢失就被判成成功 —— 那是对调用方的谎报,出了问题无从察觉。
|
|
1651
|
+
const norm = (s) => String(s || '').replace(/\r\n?/g, '\n').replace(/[^\S\n]+/g, '');
|
|
1652
|
+
// 幂等短路:若可编辑区当前内容已恰好等于目标文本,直接返回而不做清空+重打。
|
|
1653
|
+
// 每次全量重写都会把草稿标脏,重新触发编辑器自身的"草稿备份"/版本气泡——对
|
|
1654
|
+
// 相同文本的重复调用(重试/校正)不应让它反复闪烁。
|
|
1655
|
+
const curText = await run(
|
|
1656
|
+
`var e=document.activeElement;return (e&&(e.innerText||e.value||''))||'';`
|
|
1657
|
+
);
|
|
1658
|
+
if (norm(curText) === norm(text)) {
|
|
1659
|
+
// "目标就是当前内容" 与 "目标本来就是空的" 必须区分开。
|
|
1660
|
+
// 后者几乎总是调用方的错误:ui.act 的 WS 契约把动作参数放在 params.params
|
|
1661
|
+
// 里(params.params.text),照着 MCP 的扁平形状写成 params.text 就会静默
|
|
1662
|
+
// 变成空文本 —— 而空字段恰好命中下面的短路,于是**什么都没打却报
|
|
1663
|
+
// success / method=cdp-input-unchanged**。这正是本项目最忌讳的谎报。
|
|
1664
|
+
// 实测就是这样被发现的:调用回执"成功",输入框却仍是空的。
|
|
1665
|
+
if (!text) {
|
|
1666
|
+
return {
|
|
1667
|
+
success: false,
|
|
1668
|
+
error: 'type called with empty text — nothing was typed. '
|
|
1669
|
+
+ 'Over WS the action params go in params.params (e.g. {action:"type", target:"<id>", params:{text:"..."}}), '
|
|
1670
|
+
+ 'not at the top level. Use action:"clear" to empty a field.',
|
|
1671
|
+
method: 'cdp-input-empty-text',
|
|
1672
|
+
chars: 0,
|
|
1673
|
+
};
|
|
1674
|
+
}
|
|
1675
|
+
return { success: true, method: 'cdp-input-unchanged', changes: 0, chars: 0 };
|
|
1676
|
+
}
|
|
1677
|
+
const dbg = view.webContents.debugger;
|
|
1678
|
+
const send = (method, params) => dbg.sendCommand(method, params);
|
|
1679
|
+
// 全选 + 整段删除。刻意不发送受信 Cmd+A:在本应用里注入受信 Cmd+A 会触发
|
|
1680
|
+
// macOS 的应用 About 面板(原生 NSAlert,"版本"弹窗),每次输入都弹。故全选
|
|
1681
|
+
// 一律走页内程序化选择。顶层文本流(contenteditable/input/textarea)用 Range/
|
|
1682
|
+
// setSelectionRange(_selectAllFallback) + 受信 Delete;OOPIF 子帧编辑器不做受信
|
|
1683
|
+
// 删除(受信 Delete 前是 DOM-Range 选区,部分富文本编辑器不把这些选区同步进自身
|
|
1684
|
+
// EditorState,Delete 落空→clear 无效),改为在所属帧内 execCommand selectAll+delete,
|
|
1685
|
+
// 走编辑器原生选区/删除链路,B站 read-editor 验证可被清到空。顶层路径行为不变。
|
|
1686
|
+
// 追加模式(type):不清空,只把光标放到指定位置 —— 这样"插入话题标签后
|
|
1687
|
+
// 继续写正文"才成立。at: 'end' | 'start' | 'cursor'(cursor 表示不干预)。
|
|
1688
|
+
const replace = opts.replace !== false;
|
|
1689
|
+
if (!replace) {
|
|
1690
|
+
const at = opts.at === 'start' || opts.at === 'cursor' ? opts.at : 'end';
|
|
1691
|
+
if (at !== 'cursor') {
|
|
1692
|
+
await run(
|
|
1693
|
+
`var el=document.querySelector(${JSON.stringify('[data-ai-id="' + String(aiId) + '"]')});if(!el)return false;` +
|
|
1694
|
+
`var ed=el;if(ed.querySelector){var inner=ed.querySelector('[contenteditable=true],input,textarea');if(inner)ed=inner;}` +
|
|
1695
|
+
`if(ed.tagName==='INPUT'||ed.tagName==='TEXTAREA'){var L=(ed.value||'').length;` +
|
|
1696
|
+
`var p=(${JSON.stringify(at)}==='start')?0:L;try{ed.setSelectionRange(p,p);}catch(e){}return true;}` +
|
|
1697
|
+
`if(ed.contentEditable!=='true')return false;` +
|
|
1698
|
+
`try{var r=document.createRange();r.selectNodeContents(ed);` +
|
|
1699
|
+
`r.collapse(${JSON.stringify(at)}==='start');` +
|
|
1700
|
+
`var s=window.getSelection();s.removeAllRanges();s.addRange(r);return true;}catch(e){return false;}`
|
|
1701
|
+
);
|
|
1702
|
+
await new Promise((r) => setTimeout(r, 80)); // 等光标位置同步进编辑器状态
|
|
1703
|
+
}
|
|
1704
|
+
} else {
|
|
1705
|
+
const cleared = inFrame
|
|
1706
|
+
? await this._clearInFrame(owning, aiId)
|
|
1707
|
+
: await (async () => {
|
|
1708
|
+
await this._selectAllFallback(view, aiId, tabId, null);
|
|
1709
|
+
await new Promise((r) => setTimeout(r, 120)); // 等选区同步进框架状态
|
|
1710
|
+
await this._cdpKey(send, { key: 'Delete', code: 'Delete', vk: 46, mod: 0 });
|
|
1711
|
+
await new Promise((r) => setTimeout(r, 80)); // 等删除后的空状态提交
|
|
1712
|
+
return true;
|
|
1713
|
+
})();
|
|
1714
|
+
if (!cleared && text) {
|
|
1715
|
+
// 清空失败但目标非空:先不空转,仍尝试继续键入(若编辑器未清空,键入会拼接而非替换)。
|
|
1716
|
+
await this._cdpKey(send, { key: 'Delete', code: 'Delete', vk: 46, mod: 0 });
|
|
1717
|
+
}
|
|
1718
|
+
if (!text) {
|
|
1719
|
+
// selectAll + 受信 Delete 对**多块内容**可能清不干净。此前每段都被
|
|
1720
|
+
// type 整篇替换掉所以没暴露;type 改为追加语义后编辑器会累积多段,
|
|
1721
|
+
// 实测残留 "第一段<div>第二段</div><h2>第三段</h2>"。因此清完校验一次,
|
|
1722
|
+
// 仍有文本就换编辑器原生 selectAll+delete 再清一遍。
|
|
1723
|
+
const left = await run(
|
|
1724
|
+
`var el=document.querySelector(${JSON.stringify('[data-ai-id="' + String(aiId) + '"]')});if(!el)return 0;` +
|
|
1725
|
+
`var ed=el;if(ed.querySelector){var inner=ed.querySelector('[contenteditable=true],input,textarea');if(inner)ed=inner;}` +
|
|
1726
|
+
`return String((ed.innerText!=null?ed.innerText:ed.value)||'').trim().length;`
|
|
1727
|
+
);
|
|
1728
|
+
if (left > 0) {
|
|
1729
|
+
await run(
|
|
1730
|
+
`var el=document.querySelector(${JSON.stringify('[data-ai-id="' + String(aiId) + '"]')});if(!el)return false;` +
|
|
1731
|
+
`var ed=el;if(ed.querySelector){var inner=ed.querySelector('[contenteditable=true],input,textarea');if(inner)ed=inner;}` +
|
|
1732
|
+
`try{ed.focus();document.execCommand('selectAll');document.execCommand('delete');}catch(e){}return true;`
|
|
1733
|
+
);
|
|
1734
|
+
await new Promise((r) => setTimeout(r, 150)); // 等编辑器状态提交
|
|
1735
|
+
}
|
|
1736
|
+
await stripEmptyShells();
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
const paras = String(text == null ? '' : text).split('\n');
|
|
1740
|
+
// 段落内容快照:用来判断"这一下 Enter 到底有没有生效"。
|
|
1741
|
+
const snap = () => run(
|
|
1742
|
+
`var e=document.activeElement;var t=String((e&&(e.innerText||e.value))||'');` +
|
|
1743
|
+
`var b=(e&&e.querySelectorAll)?e.querySelectorAll('p,div,br').length:0;` +
|
|
1744
|
+
`return {len:t.length,nl:(t.match(/\\n/g)||[]).length,blocks:b};`
|
|
1745
|
+
);
|
|
1746
|
+
for (let i = 0; i < paras.length; i++) {
|
|
1747
|
+
if (paras[i]) await this._typeChars(send, paras[i]);
|
|
1748
|
+
if (i < paras.length - 1) {
|
|
1749
|
+
// 段间换行。富文本编辑器(ProseMirror/Draft…)自己接 keydown 处理
|
|
1750
|
+
// Enter 并已分好段;原生 textarea / 原生 contenteditable 没人接管,
|
|
1751
|
+
// 而 rawKeyDown 不带 text 时默认编辑动作不触发 —— 换行会静默丢失。
|
|
1752
|
+
// 故:先发受信 Enter,若内容毫无变化(说明没人处理它),再补一个带
|
|
1753
|
+
// text 的 char 事件把换行真正插进去。给已分段的编辑器补发会多插空行,
|
|
1754
|
+
// 所以这一步必须"看效果再决定",不能无脑补。
|
|
1755
|
+
const before = await snap();
|
|
1756
|
+
await this._cdpKey(send, { key: 'Enter', code: 'Enter', vk: 13, mod: 0 });
|
|
1757
|
+
const after = await snap();
|
|
1758
|
+
const moved =
|
|
1759
|
+
Number(after && after.len) > Number(before && before.len) ||
|
|
1760
|
+
Number(after && after.nl) > Number(before && before.nl) ||
|
|
1761
|
+
Number(after && after.blocks) > Number(before && before.blocks);
|
|
1762
|
+
if (!moved) await this._cdpKey(send, { key: '\r', code: 'Enter', vk: 13, mod: 0, text: '\r' });
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
// 通用校验:非空文本要求内容包含目标文本;空文本要求已清空(只允许空块残留)。
|
|
1766
|
+
// 任一框架均可,无占位符/块数特判。
|
|
1767
|
+
const after = await run(
|
|
1768
|
+
`var e=document.activeElement;return {text:(e.innerText||e.value||'')};`
|
|
1769
|
+
);
|
|
1770
|
+
const want = norm(text);
|
|
1771
|
+
const got = norm(after && after.text);
|
|
1772
|
+
const ok = want
|
|
1773
|
+
? got.includes(want)
|
|
1774
|
+
: got === '';
|
|
1775
|
+
return ok
|
|
1776
|
+
? { success: true, method: 'cdp-input', chars: (after && after.text) ? after.text.length : 0 }
|
|
1777
|
+
: { success: false, error: 'Input not verified — please re-read the tree and retry', method: 'cdp-input-failed' };
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1780
|
+
// OOPIF 内整段清空:在所属子帧上下文里聚焦 contenteditable,用 execCommand
|
|
1781
|
+
// selectAll + delete 走编辑器原生选区/删除链路(受信 Delete 前置 DOM-Range 选区对
|
|
1782
|
+
// 部分富文本编辑器无效)。返回是否已清到空。
|
|
1783
|
+
async _clearInFrame(frame, aiId) {
|
|
1784
|
+
try {
|
|
1785
|
+
const r = await frame.executeJavaScript(`(function(){
|
|
1786
|
+
var ref=document.querySelector('[data-ai-id="${aiId}"]');if(!ref)return {ok:false,reason:'no-ref'};
|
|
1787
|
+
var e=ref;if(ref.querySelector){var inner=ref.querySelector('[contenteditable="true"],textarea,input[type="text"],input[type="search"],input[type="title"],input:not([type])');if(inner)e=inner;}
|
|
1788
|
+
var isInput=(e.tagName==='INPUT'||e.tagName==='TEXTAREA');
|
|
1789
|
+
var isCE=e.contentEditable==='true';
|
|
1790
|
+
if(!(isInput||isCE))return {ok:false,reason:'not-editable'};
|
|
1791
|
+
e.focus();
|
|
1792
|
+
var ae=document.activeElement&&(document.activeElement===e?'same':'other:'+(document.activeElement.tagName||''));
|
|
1793
|
+
var sa=false;
|
|
1794
|
+
if(isCE){ sa=document.execCommand('selectAll'); }
|
|
1795
|
+
else if(isInput){ var len=(e.value||'').length; if(len){ try{e.setSelectionRange(0,len);}catch(_){ if(e.select)e.select(); } } }
|
|
1796
|
+
var del=document.execCommand('delete');
|
|
1797
|
+
var txt=isInput?(e.value||''):(e.innerText||e.textContent||'');
|
|
1798
|
+
return {ok:txt.trim()==='',sa:!!sa,del:!!del,left:(txt||'').trim().length,ae:ae};
|
|
1799
|
+
})()`);
|
|
1800
|
+
if (!(r && r.ok)) console.error('[clearInFrame] FAIL', JSON.stringify(r));
|
|
1801
|
+
return !!(r && r.ok);
|
|
1802
|
+
} catch (e) { console.error('[clearInFrame] THROW', e.message); return false; }
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
// 全选回退:Cmd+A 后选区仍为空时,用浏览器原生选区 API 选中可编辑区全部文本。
|
|
1806
|
+
// 返回 true 表示已有非空选区(Cmd+A 已生效或回退成功);框架通过
|
|
1807
|
+
// selectionchange 同步内部状态,随后的受信 Delete 即可整段删除。
|
|
1808
|
+
async _selectAllFallback(view, aiId, tabId, frame) {
|
|
1809
|
+
const r = frame
|
|
1810
|
+
? await frame.executeJavaScript(
|
|
1811
|
+
`(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return false;` +
|
|
1812
|
+
`var ed=el;if(ed.querySelector){var inner=ed.querySelector('[contenteditable=true],input,textarea');if(inner)ed=inner;}` +
|
|
1813
|
+
`var s=window.getSelection();` +
|
|
1814
|
+
`if(s&&!s.isCollapsed&&s.toString().length>0)return true;` +
|
|
1815
|
+
`if(ed.tagName==='TEXTAREA'||ed.tagName==='INPUT'){var v=ed.value||'';if(v){ed.focus();ed.setSelectionRange(0,v.length);}return v.length>0;}` +
|
|
1816
|
+
`var walker=document.createTreeWalker(ed,NodeFilter.SHOW_TEXT);` +
|
|
1817
|
+
`var first=null,last=null,t;` +
|
|
1818
|
+
`while(t=walker.nextNode()){if(t.nodeValue&&t.nodeValue.length){if(!first)first=t;last=t;}}` +
|
|
1819
|
+
`if(!first)return false;` +
|
|
1820
|
+
`var rng=document.createRange();rng.setStart(first,0);rng.setEnd(last,last.nodeValue.length);` +
|
|
1821
|
+
`s.removeAllRanges();s.addRange(rng);` +
|
|
1822
|
+
`return rng.toString().length>0;})()`)
|
|
1823
|
+
: await this.evaluate(
|
|
1824
|
+
`(function(){var el=document.querySelector('[data-ai-id="${aiId}"]');if(!el)return false;` +
|
|
1825
|
+
`var ed=el;if(ed.querySelector){var inner=ed.querySelector('[contenteditable=true],input,textarea');if(inner)ed=inner;}` +
|
|
1826
|
+
`var s=window.getSelection();` +
|
|
1827
|
+
`if(s&&!s.isCollapsed&&s.toString().length>0)return true;` +
|
|
1828
|
+
`if(ed.tagName==='TEXTAREA'||ed.tagName==='INPUT'){var v=ed.value||'';if(v){ed.focus();ed.setSelectionRange(0,v.length);}return v.length>0;}` +
|
|
1829
|
+
`var walker=document.createTreeWalker(ed,NodeFilter.SHOW_TEXT);` +
|
|
1830
|
+
`var first=null,last=null,t;` +
|
|
1831
|
+
`while(t=walker.nextNode()){if(t.nodeValue&&t.nodeValue.length){if(!first)first=t;last=t;}}` +
|
|
1832
|
+
`if(!first)return false;` +
|
|
1833
|
+
`var rng=document.createRange();rng.setStart(first,0);rng.setEnd(last,last.nodeValue.length);` +
|
|
1834
|
+
`s.removeAllRanges();s.addRange(rng);` +
|
|
1835
|
+
`return rng.toString().length>0;})()`,
|
|
1836
|
+
tabId);
|
|
1837
|
+
return !!r;
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
// 逐字符受信键入:rawKeyDown→char→keyUp,等价真实键盘输入。
|
|
1841
|
+
// 注意:text 字段对 keyDown 和 char 事件都会插入文本,若 keyDown 也带 text 会
|
|
1842
|
+
// 造成每个字符插入两遍(知乎标题曾出现"测测试试标标题题");rawKeyDown 会忽略
|
|
1843
|
+
// text,按下事件不携带文本,仅 char 事件携带,与 Puppeteer keyboard.type 一致。
|
|
1844
|
+
// 编辑器(Draft/ProseMirror/Quill…)会按当前光标处的行内样式应用格式。
|
|
1845
|
+
async _typeChars(send, text) {
|
|
1846
|
+
for (const ch of Array.from(text)) {
|
|
1847
|
+
const base = { key: ch, code: '', windowsVirtualKeyCode: 0, modifiers: 0 };
|
|
1848
|
+
await send('Input.dispatchKeyEvent', { ...base, type: 'rawKeyDown' });
|
|
1849
|
+
await send('Input.dispatchKeyEvent', { ...base, type: 'char', text: ch, unmodifiedText: ch });
|
|
1850
|
+
await send('Input.dispatchKeyEvent', { ...base, type: 'keyUp' });
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
// === Tab lifecycle ===
|
|
1855
|
+
|
|
1856
|
+
newTab(url = null) {
|
|
1857
|
+
const tabId = ++this.tabCounter;
|
|
1858
|
+
const self = this;
|
|
1859
|
+
const view = new BrowserView({
|
|
1860
|
+
webPreferences: {
|
|
1861
|
+
preload: this._preloadPath,
|
|
1862
|
+
contextIsolation: true,
|
|
1863
|
+
nodeIntegration: false,
|
|
1864
|
+
// Required for the CommonJS preload (bridge.cjs) to `require()` local
|
|
1865
|
+
// modules (./extractor.cjs, ./actions.cjs, ./watcher.cjs). A sandboxed
|
|
1866
|
+
// preload can only require Electron's whitelist. Security is unchanged:
|
|
1867
|
+
// nodeIntegration stays false + contextIsolation true, so page code
|
|
1868
|
+
// has no Node access regardless of the OS-level renderer sandbox.
|
|
1869
|
+
sandbox: false,
|
|
1870
|
+
// 与 installSessionHooks 用的 session.fromPartition(config.viewPartition)
|
|
1871
|
+
// 必须一致 —— 否则 will-download / 权限钩子装到了另一个 session 上,
|
|
1872
|
+
// 表现是"页面确实请求了下载,但什么都没发生"(实测踩过,见 config.js)。
|
|
1873
|
+
partition: config.viewPartition,
|
|
1874
|
+
persistStorage: true,
|
|
1875
|
+
webSecurity: true,
|
|
1876
|
+
allowRunningInsecureContent: false,
|
|
1877
|
+
},
|
|
1878
|
+
});
|
|
1879
|
+
this.tabs.set(tabId, view);
|
|
1880
|
+
|
|
1881
|
+
// Tabs created after a subscribe should inherit the CDP domains already
|
|
1882
|
+
// active, so network_response / js_error keep flowing without re-subscribing.
|
|
1883
|
+
// _ensureCdp attach + enable is idempotent (attach guarded by _cdpTabs).
|
|
1884
|
+
if (this._runtimeSubscribers.size > 0) this._ensureCdp(tabId, view, 'Runtime');
|
|
1885
|
+
// Network:默认在首个真实导航前就启用(不再等订阅),否则初始 HTML/API 与
|
|
1886
|
+
// 导航阶段发出的 POST 全部丢失 —— agent 永远拿不到它们的请求体。
|
|
1887
|
+
// 关闭开关:AI_BROWSER_NETWORK_CAPTURE=0 或 ui.network_configure{enabled:false}。
|
|
1888
|
+
this._ensureNetworkCapture(tabId, view);
|
|
1889
|
+
|
|
1890
|
+
// Mask automation fingerprint: remove Electron from UA
|
|
1891
|
+
view.webContents.setUserAgent(config.userAgent);
|
|
1892
|
+
|
|
1893
|
+
// Intercept window.open / new-window → create new tab instead
|
|
1894
|
+
view.webContents.setWindowOpenHandler(({ url: urlToOpen }) => {
|
|
1895
|
+
const childId = self.newTab(urlToOpen);
|
|
1896
|
+
// Track which source tab opened a new tab so `executeAction` can report the
|
|
1897
|
+
// resulting tab and auto-follow it. Generic (any target=_blank / window.open).
|
|
1898
|
+
self._openedTabs.set(tabId, { id: childId, url: urlToOpen });
|
|
1899
|
+
return { action: 'deny' };
|
|
1900
|
+
});
|
|
1901
|
+
|
|
1902
|
+
// setActive handles addBrowserView + layout — don't add twice
|
|
1903
|
+
this.setActive(tabId);
|
|
1904
|
+
|
|
1905
|
+
// canvas 绘制记录必须在**首个真实导航之前**装好,否则页面早画完了什么都录不到。
|
|
1906
|
+
// 但它又必须在 setActive 之后:BrowserView 还没被加进窗口时,debugger 通道
|
|
1907
|
+
// 不响应(实测 Runtime.enable 直接挂死)。
|
|
1908
|
+
// newTab 仍同步返回 tabId(调用方依赖),导航推迟到 hook 装完;安装失败或超时
|
|
1909
|
+
// 也照常导航 —— 绝不能因为要录 canvas 就把 tab 卡死。
|
|
1910
|
+
// hook 关闭时(当前默认)走原来的同步导航:把导航推迟成异步会引入时序抖动
|
|
1911
|
+
// (实测 richtext 偶发 110/111),这个代价不值得为一个未验证通过的功能付。
|
|
1912
|
+
if (isCanvasHookEnabled(process.env.AI_BROWSER_CANVAS_HOOK)) {
|
|
1913
|
+
// CDP 的 await 命令在页面加载**之后**才可用(刚创建的 tab 上会挂死),
|
|
1914
|
+
// 所以装 hook 的时机放在首次加载完成。代价是首屏已经画完的内容拿不回来
|
|
1915
|
+
// (configure({reload:true}) 可以补齐)。
|
|
1916
|
+
const install = () => { this._ensureCanvasHook(tabId, view).catch(() => false); };
|
|
1917
|
+
view.webContents.once('did-finish-load', install);
|
|
1918
|
+
if (url) {
|
|
1919
|
+
view.webContents.loadURL(url);
|
|
1920
|
+
}
|
|
1921
|
+
} else if (url) {
|
|
1922
|
+
view.webContents.loadURL(url);
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1925
|
+
// Force repaint after load — BrowserView content can be loaded but not
|
|
1926
|
+
// painted by the compositor. Remove + re-add forces a full repaint.
|
|
1927
|
+
//
|
|
1928
|
+
// Hook every navigation signal, not just the two load events. Reported
|
|
1929
|
+
// symptom: page HAS navigated to the new view (agent reads the new DOM /
|
|
1930
|
+
// canvas through the tree and evaluate) but the window keeps showing the
|
|
1931
|
+
// previous one until you click tabs back and forth — clicking a tab is
|
|
1932
|
+
// exactly what calls removeBrowserView + addBrowserView + setBounds, i.e.
|
|
1933
|
+
// it is the only thing that forces the compositor to repaint.
|
|
1934
|
+
//
|
|
1935
|
+
// did-navigate-in-page — SPA route changes (history.pushState / hash).
|
|
1936
|
+
// Fires on every router push, which is how canvas-style apps swap views.
|
|
1937
|
+
// did-frame-finish-load — a sub-frame finished loading. ComfyUI-style
|
|
1938
|
+
// editors boot inside an iframe long after the top document finished,
|
|
1939
|
+
// so did-finish-load has already fired and repainted an empty shell.
|
|
1940
|
+
// did-navigate — any cross-document navigation, including programmatic
|
|
1941
|
+
// location.href assignment the app does on its own.
|
|
1942
|
+
if (url) {
|
|
1943
|
+
const repaint = () => { self._scheduleRepaint(tabId); };
|
|
1944
|
+
view.webContents.on('dom-ready', repaint);
|
|
1945
|
+
view.webContents.on('did-finish-load', repaint);
|
|
1946
|
+
view.webContents.on('did-navigate', repaint);
|
|
1947
|
+
view.webContents.on('did-navigate-in-page', repaint);
|
|
1948
|
+
view.webContents.on('did-frame-finish-load', repaint);
|
|
1949
|
+
} else {
|
|
1950
|
+
// No URL at construction time (new_tab with no url, then navigate):
|
|
1951
|
+
// register the same hooks now, or the later navigate() would never
|
|
1952
|
+
// repaint — which was one of the paths that showed a stale view.
|
|
1953
|
+
this._attachRepaintHooks(tabId, view);
|
|
1954
|
+
}
|
|
1955
|
+
return tabId;
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
/**
|
|
1959
|
+
* Attach compositor-repaint hooks to a tab's webContents.
|
|
1960
|
+
* Idempotent per tab: re-registering would schedule redundant remove/add
|
|
1961
|
+
* cycles, so a WeakSet of hooked views guards it.
|
|
1962
|
+
*/
|
|
1963
|
+
_attachRepaintHooks(tabId, view) {
|
|
1964
|
+
if (!view || !view.webContents || view.webContents.isDestroyed()) return;
|
|
1965
|
+
if (!this._repaintHooked) this._repaintHooked = new WeakSet();
|
|
1966
|
+
if (this._repaintHooked.has(view)) return;
|
|
1967
|
+
this._repaintHooked.add(view);
|
|
1968
|
+
const repaint = () => { this._scheduleRepaint(tabId); };
|
|
1969
|
+
view.webContents.on('dom-ready', repaint);
|
|
1970
|
+
view.webContents.on('did-finish-load', repaint);
|
|
1971
|
+
view.webContents.on('did-navigate', repaint);
|
|
1972
|
+
view.webContents.on('did-navigate-in-page', repaint);
|
|
1973
|
+
view.webContents.on('did-frame-finish-load', repaint);
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
/**
|
|
1977
|
+
* Coalesce repaint requests. A single page load fires dom-ready +
|
|
1978
|
+
* did-navigate + did-navigate-in-page + did-frame-finish-load within a few
|
|
1979
|
+
* hundred ms; a load like ComfyUI's fires dozens of did-frame-finish-load.
|
|
1980
|
+
* Without coalescing that is dozens of remove/add cycles — visible flicker
|
|
1981
|
+
* and wasted compositing work. Debounce ~120ms: late enough to collapse the
|
|
1982
|
+
* burst, early enough that the user sees the new view essentially at once.
|
|
1983
|
+
*/
|
|
1984
|
+
_scheduleRepaint(tabId, delay = 120) {
|
|
1985
|
+
const prev = this._repaintTimers.get(tabId);
|
|
1986
|
+
if (prev) clearTimeout(prev);
|
|
1987
|
+
const t = setTimeout(() => {
|
|
1988
|
+
this._repaintTimers.delete(tabId);
|
|
1989
|
+
const view = this._getView(tabId);
|
|
1990
|
+
if (view) this._forceRepaint(view);
|
|
1991
|
+
}, delay);
|
|
1992
|
+
if (t && typeof t.unref === 'function') t.unref();
|
|
1993
|
+
this._repaintTimers.set(tabId, t);
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
/**
|
|
1997
|
+
* Remove + re-add the BrowserView to force the compositor to paint it.
|
|
1998
|
+
* Safe for a non-active tab too: the view is re-added in the same position
|
|
1999
|
+
* (only the active view is normally attached, so re-adding an inactive one
|
|
2000
|
+
* would show it — guard by re-asserting the active view afterwards).
|
|
2001
|
+
*/
|
|
2002
|
+
_forceRepaint(view) {
|
|
2003
|
+
const tabId = this._tabIdOfView(view);
|
|
2004
|
+
const wasActive = tabId === this.activeTab;
|
|
2005
|
+
try {
|
|
2006
|
+
this.window.removeBrowserView(view);
|
|
2007
|
+
this.window.addBrowserView(view);
|
|
2008
|
+
this._layoutView(view);
|
|
2009
|
+
} catch(e) { return false; }
|
|
2010
|
+
// Cumulative counter, reported by ui.repaint. Not just a debug aid: it is
|
|
2011
|
+
// the only externally observable proof that a repaint actually happened.
|
|
2012
|
+
// "The window looks stale" has no other symptom to check, and a caller
|
|
2013
|
+
// cannot poll for it — polling calls repaint itself, which *consumes* the
|
|
2014
|
+
// queued debounce timer and destroys the very evidence being looked for.
|
|
2015
|
+
this._repaintCount = (this._repaintCount || 0) + 1;
|
|
2016
|
+
// If we just re-added an inactive view it is now on top of the stack —
|
|
2017
|
+
// put the real active view back so the user sees the tab they selected.
|
|
2018
|
+
if (!wasActive) {
|
|
2019
|
+
const active = this._getView(this.activeTab);
|
|
2020
|
+
if (active) {
|
|
2021
|
+
try {
|
|
2022
|
+
this.window.removeBrowserView(view);
|
|
2023
|
+
this.window.addBrowserView(active);
|
|
2024
|
+
this._layoutView(active);
|
|
2025
|
+
} catch(e) {}
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
return true;
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
_tabIdOfView(view) {
|
|
2032
|
+
for (const [id, v] of this.tabs) if (v === view) return id;
|
|
2033
|
+
return null;
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
/**
|
|
2037
|
+
* Repaint on demand (ui.repaint). The "page shows stale content" case has
|
|
2038
|
+
* no observable state to detect it from — the DOM and evaluate already report
|
|
2039
|
+
* the NEW page while the compositor shows the OLD one — so the agent needs an
|
|
2040
|
+
* explicit "I know this window looks wrong, redraw it" call rather than a
|
|
2041
|
+
* heuristic.
|
|
2042
|
+
*/
|
|
2043
|
+
repaint(tabId) {
|
|
2044
|
+
const tid = tabId !== undefined && tabId !== null ? tabId : this.activeTab;
|
|
2045
|
+
const view = this._getView(tid);
|
|
2046
|
+
if (!view) return { ok: false, error: 'no-such-tab', tab: tid };
|
|
2047
|
+
// Cancel any queued repaint so this one is the last word.
|
|
2048
|
+
const pending = this._repaintTimers.get(tid);
|
|
2049
|
+
// `was_pending` is how a caller tells whether the navigation hooks are
|
|
2050
|
+
// alive: right after a navigation there is normally a queued repaint; once
|
|
2051
|
+
// it has drained, there is none. It also tells the agent "you did not need
|
|
2052
|
+
// to ask, the browser already scheduled one" — i.e. the window was stale
|
|
2053
|
+
// for ~120ms and has since caught up on its own.
|
|
2054
|
+
if (pending) { clearTimeout(pending); this._repaintTimers.delete(tid); }
|
|
2055
|
+
const ok = this._forceRepaint(view);
|
|
2056
|
+
return { ok, tab: tid, was_active: tid === this.activeTab, was_pending: !!pending, repaint_count: this._repaintCount || 0 };
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
closeTab(tabId) {
|
|
2060
|
+
const view = this._getView(tabId);
|
|
2061
|
+
if (!view) return false;
|
|
2062
|
+
this.window.removeBrowserView(view);
|
|
2063
|
+
if (view.webContents && !view.webContents.isDestroyed()) {
|
|
2064
|
+
try { view.webContents.debugger.detach(); } catch(e) {}
|
|
2065
|
+
view.webContents.close();
|
|
2066
|
+
}
|
|
2067
|
+
this.tabs.delete(tabId);
|
|
2068
|
+
// Release per-tab network resources so a closed tab can never leak its
|
|
2069
|
+
// debugger attachment or cached request entries / stale monitors.
|
|
2070
|
+
this._cleanupTabResources(tabId);
|
|
2071
|
+
if (this.activeTab === tabId || this.activeTab === null) {
|
|
2072
|
+
// switch to first remaining tab — must re-addBrowserView
|
|
2073
|
+
const first = this.tabs.keys().next();
|
|
2074
|
+
this.activeTab = first.done ? null : first.value;
|
|
2075
|
+
if (!first.done) {
|
|
2076
|
+
const nextView = this._getView(this.activeTab);
|
|
2077
|
+
this.window.addBrowserView(nextView);
|
|
2078
|
+
this._layoutView(nextView);
|
|
2079
|
+
// Force repaint
|
|
2080
|
+
nextView.webContents.setBackgroundThrottling(false);
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
return true;
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
// Tear down a closed tab's network-monitoring bookkeeping:
|
|
2087
|
+
// - detach that tab's shared per-tab debugger (other tabs/subscribers are
|
|
2088
|
+
// unaffected — each tab owns its own webContents debugger)
|
|
2089
|
+
// - drop cached request entries made on that tab
|
|
2090
|
+
_cleanupTabResources(tabId) {
|
|
2091
|
+
const entry = this._cdpTabs.get(tabId);
|
|
2092
|
+
if (entry) {
|
|
2093
|
+
try { entry.wc.debugger.detach(); } catch(e) {}
|
|
2094
|
+
this._cdpTabs.delete(tabId);
|
|
2095
|
+
}
|
|
2096
|
+
for (const [requestId, r] of this._networkRequestMap) {
|
|
2097
|
+
if (r.tabId === tabId) this._networkRequestMap.delete(requestId);
|
|
2098
|
+
}
|
|
2099
|
+
// 释放该 tab 的抓包日志与 body 内存,避免关掉的 tab 继续占着配额。
|
|
2100
|
+
this.networkMonitor.clearBodyFetcher(tabId);
|
|
2101
|
+
this.networkMonitor.clear(tabId);
|
|
2102
|
+
// 该 tab 上取的快照基线一并没有意义了:页面已经没了,之后拿它 diff 只会
|
|
2103
|
+
// 得到一堵"全变了";而留着它则会让整份快照值一直占着内存。
|
|
2104
|
+
for (const [sid, s] of this._snapshots) {
|
|
2105
|
+
if (s.tab === tabId) this._snapshots.delete(sid);
|
|
2106
|
+
}
|
|
2107
|
+
// 同理,下载记录只对"这个 tab 还在"有意义。注意**只清记录,不删文件** ——
|
|
2108
|
+
// 文件是用户要的东西,删文件是另一个决定,不能藏在关 tab 里。
|
|
2109
|
+
this.downloadMonitor.clearTab(tabId);
|
|
2110
|
+
}
|
|
2111
|
+
|
|
2112
|
+
listTabs() {
|
|
2113
|
+
const list = [];
|
|
2114
|
+
for (const [id, view] of this.tabs) {
|
|
2115
|
+
list.push({
|
|
2116
|
+
id,
|
|
2117
|
+
url: view.webContents.getURL(),
|
|
2118
|
+
title: view.webContents.getTitle(),
|
|
2119
|
+
active: id === this.activeTab,
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
return list;
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
setActive(tabId) {
|
|
2126
|
+
if (!this.tabs.has(tabId)) return false;
|
|
2127
|
+
// Remove all browser views from window, then re-add only the active one
|
|
2128
|
+
for (const [id, view] of this.tabs) {
|
|
2129
|
+
try { this.window.removeBrowserView(view); } catch(e) {}
|
|
2130
|
+
}
|
|
2131
|
+
this.activeTab = tabId;
|
|
2132
|
+
const view = this._getView(tabId);
|
|
2133
|
+
if (view) {
|
|
2134
|
+
this.window.addBrowserView(view);
|
|
2135
|
+
this._layoutView(view);
|
|
2136
|
+
this._focusWebContents(view);
|
|
2137
|
+
view.webContents.setBackgroundThrottling(false);
|
|
2138
|
+
}
|
|
2139
|
+
return true;
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
/**
|
|
2143
|
+
* 让**页面**处于活动状态,但**不**把窗口拽到前台。
|
|
2144
|
+
*
|
|
2145
|
+
* 这是实测出来的(e2e/focus_ws.cjs):ui.new_tab / ui.set_active_tab 每次都调
|
|
2146
|
+
* webContents.focus(),而它会把整个窗口激活 —— agent 每开一个标签页就把人正在
|
|
2147
|
+
* 做的事打断一次。其它所有操作(act / get_tree / evaluate / peek / scroll /
|
|
2148
|
+
* canvas / network)经逐项测量都不抢焦点,只有这两个不是。
|
|
2149
|
+
*
|
|
2150
|
+
* AI_BROWSER_FOCUS=auto(默认):不碰真实窗口焦点,改用 CDP
|
|
2151
|
+
* Emulation.setFocusEmulationEnabled 让页面内的 document.hasFocus() 等状态
|
|
2152
|
+
* 保持正常(依赖它的懒加载 / 动画 / 富文本聚焦态不会僵住),但键盘焦点留在
|
|
2153
|
+
* 用户那边 —— 这正是我们要的。
|
|
2154
|
+
* always:旧行为,直接把焦点交给 web contents(个别站点确需时用)。
|
|
2155
|
+
* never:两者都不做。
|
|
2156
|
+
*/
|
|
2157
|
+
_focusWebContents(view) {
|
|
2158
|
+
const policy = String(process.env.AI_BROWSER_FOCUS || 'auto').toLowerCase();
|
|
2159
|
+
try {
|
|
2160
|
+
if (!view || !view.webContents || view.webContents.isDestroyed()) return;
|
|
2161
|
+
if (policy === 'never') return;
|
|
2162
|
+
if (policy === 'always') {
|
|
2163
|
+
// 必须与 _focusWindow 的 always 分支做同样的事:只调 webContents.focus()
|
|
2164
|
+
// **不会**把窗口带到前台(实测 focus 套件的 G2 对照组里 new_tab /
|
|
2165
|
+
// set_active_tab 因此不抢,而 act type/click 抢 —— 同一个 always 策略
|
|
2166
|
+
// 两套行为)。always 的契约就是"每次操作都把窗口带到前台",这里补齐。
|
|
2167
|
+
try {
|
|
2168
|
+
if (this.window.show) this.window.show();
|
|
2169
|
+
if (this.window.moveTop) this.window.moveTop();
|
|
2170
|
+
if (this.window.focus) this.window.focus();
|
|
2171
|
+
} catch (e) { /* 窗口已销毁 */ }
|
|
2172
|
+
view.webContents.focus();
|
|
2173
|
+
return;
|
|
2174
|
+
}
|
|
2175
|
+
// auto:只"伪造"页面内的焦点状态。CDP 尚未 attach(首次导航前的短暂窗口期)
|
|
2176
|
+
// 会 reject,静默忽略 —— 绝不因此影响标签页创建或导航。
|
|
2177
|
+
const p = view.webContents.debugger.sendCommand(
|
|
2178
|
+
'Emulation.setFocusEmulationEnabled', { enabled: true }
|
|
2179
|
+
);
|
|
2180
|
+
if (p && typeof p.catch === 'function') p.catch(() => {});
|
|
2181
|
+
} catch (e) { /* debugger 未 attach / 页面已销毁 —— 忽略 */ }
|
|
2182
|
+
}
|
|
2183
|
+
|
|
2184
|
+
_layoutView(view) {
|
|
2185
|
+
const bounds = this.window.getContentBounds();
|
|
2186
|
+
const TAB_BAR_HEIGHT = 36;
|
|
2187
|
+
view.setBounds({ x: 0, y: TAB_BAR_HEIGHT, width: bounds.width, height: bounds.height - TAB_BAR_HEIGHT });
|
|
2188
|
+
}
|
|
2189
|
+
|
|
2190
|
+
addSubscription(sessionId, events) {
|
|
2191
|
+
const existing = this.subscriptions.get(sessionId) || new Set();
|
|
2192
|
+
events.forEach(e => existing.add(e));
|
|
2193
|
+
this.subscriptions.set(sessionId, existing);
|
|
2194
|
+
if (events.includes('network') || events.includes('network_response')) this._startNetworkMonitor(sessionId);
|
|
2195
|
+
if (events.includes('js_error') || events.includes('unhandledrejection')) this._startRuntimeMonitor(sessionId);
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
_networkRequestMap = new Map(); // requestId -> {url, tabId, finished}
|
|
2199
|
+
|
|
2200
|
+
// DEPRECATED:被 ui.network_list / ui.network_get 取代(下一主版本删除)。
|
|
2201
|
+
// 兼容行为:按 url_pattern 选【最近完成】的匹配项(旧实现按 Map 正向遍历,
|
|
2202
|
+
// 同 URL 多次请求时拿到的是最旧那条),响应仍保持旧形状 {body:string|null}。
|
|
2203
|
+
async getNetworkBody(urlPattern, tabId) {
|
|
2204
|
+
const tid = tabId !== undefined ? tabId : this.activeTab;
|
|
2205
|
+
const view = this._getView(tid);
|
|
2206
|
+
const dbg = view && view.webContents && view.webContents.debugger;
|
|
2207
|
+
if (!dbg || typeof dbg.isAttached !== 'function' || !dbg.isAttached()) return null;
|
|
2208
|
+
|
|
2209
|
+
const rec = this.networkMonitor.findLatestFinished(tid, urlPattern || '');
|
|
2210
|
+
if (rec) {
|
|
2211
|
+
try {
|
|
2212
|
+
const detail = await this.networkMonitor.get(rec.network_id, {
|
|
2213
|
+
include_response_body: true,
|
|
2214
|
+
body_limit: 5000,
|
|
2215
|
+
});
|
|
2216
|
+
const body = detail && detail.response && detail.response.body && detail.response.body.raw
|
|
2217
|
+
? detail.response.body.raw.data : null;
|
|
2218
|
+
return body == null ? null : body;
|
|
2219
|
+
} catch (e) {
|
|
2220
|
+
// body 被淘汰/不可用:旧契约只能表达 null,但至少不谎报成空响应。
|
|
2221
|
+
return null;
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
// 抓包关闭时退回旧索引路径(Network 域可能由订阅启用但未记日志)。
|
|
2226
|
+
for (const [requestId, entry] of this._networkRequestMap) {
|
|
2227
|
+
if (entry.tabId === tid && entry.finished && entry.url.indexOf(urlPattern) >= 0) {
|
|
2228
|
+
try {
|
|
2229
|
+
const result = await dbg.sendCommand('Network.getResponseBody', { requestId });
|
|
2230
|
+
return result.body ? result.body.slice(0, 5000) : null;
|
|
2231
|
+
} catch(e) { return null; }
|
|
2232
|
+
}
|
|
2233
|
+
}
|
|
2234
|
+
return null;
|
|
2235
|
+
}
|
|
2236
|
+
|
|
2237
|
+
removeSubscription(sessionId, events) {
|
|
2238
|
+
const existing = this.subscriptions.get(sessionId);
|
|
2239
|
+
if (!existing) return;
|
|
2240
|
+
events.forEach(e => existing.delete(e));
|
|
2241
|
+
if (existing.size === 0) {
|
|
2242
|
+
this.subscriptions.delete(sessionId);
|
|
2243
|
+
this._stopNetworkMonitor(sessionId);
|
|
2244
|
+
this._stopRuntimeMonitor(sessionId);
|
|
2245
|
+
}
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
// Per-tab shared CDP debugger. A webContents can only be debugger-attached
|
|
2249
|
+
// once, so attach + a single message listener happen exactly once per tab; the
|
|
2250
|
+
// Network and Runtime domains are enabled lazily per demand, and subscribers
|
|
2251
|
+
// are reference-counted so the shared debugger is detached only when the last
|
|
2252
|
+
// network AND runtime subscriber both leave.
|
|
2253
|
+
_cdpTabs = new Map(); // tabId -> { wc, domains:Set<'Network'|'Runtime'> }
|
|
2254
|
+
_networkSubscribers = new Set(); // sessionIds currently using network events
|
|
2255
|
+
_runtimeSubscribers = new Set(); // sessionIds currently using js_error events
|
|
2256
|
+
|
|
2257
|
+
// Attach the shared per-tab debugger if needed and enable a CDP domain.
|
|
2258
|
+
// ---- canvas 绘制调用记录 ----
|
|
2259
|
+
//
|
|
2260
|
+
// 必须在首个真实导航**之前**注入:canvas 是有状态的立即模式 API,页面加载时
|
|
2261
|
+
// 就把内容画完了,之后再注入什么都录不到。
|
|
2262
|
+
//
|
|
2263
|
+
// 两个关键点:
|
|
2264
|
+
// 1. Page.addScriptToEvaluateOnNewDocument **不传 worldName** 才进页面 main world。
|
|
2265
|
+
// contextIsolation:true 下 preload 活在 isolated world,改自己 world 的原型
|
|
2266
|
+
// 截不到页面代码的调用。
|
|
2267
|
+
// 2. 用 Runtime.addBinding 安装上报通道(binding 名每 tab 随机,避免跨 tab 串台),
|
|
2268
|
+
// 回传在 CDP 的 Runtime.bindingCalled 里收取。
|
|
2269
|
+
async _ensureCanvasHook(tabId, view) {
|
|
2270
|
+
if (!isCanvasHookEnabled(process.env.AI_BROWSER_CANVAS_HOOK)) {
|
|
2271
|
+
this.canvasMonitor.markHook(tabId, false, ['canvas hook disabled via AI_BROWSER_CANVAS_HOOK']);
|
|
2272
|
+
return false;
|
|
2273
|
+
}
|
|
2274
|
+
const DEBUG = process.env.AI_BROWSER_CANVAS_DEBUG === '1';
|
|
2275
|
+
const log = (...a) => { if (DEBUG) console.error('[canvas-hook]', ...a); };
|
|
2276
|
+
try {
|
|
2277
|
+
this._ensureCdp(tabId, view, 'Page');
|
|
2278
|
+
const dbg = view && view.webContents && view.webContents.debugger;
|
|
2279
|
+
if (!dbg || typeof dbg.isAttached !== 'function' || !dbg.isAttached()) {
|
|
2280
|
+
this.canvasMonitor.markHook(tabId, false, ['cdp debugger not attached']);
|
|
2281
|
+
return false;
|
|
2282
|
+
}
|
|
2283
|
+
const bindingName = `__aiCanvasReport_${tabId}_${Math.random().toString(36).slice(2, 8)}`;
|
|
2284
|
+
const source = buildCanvasHookSource({
|
|
2285
|
+
bindingName,
|
|
2286
|
+
bridgeKey: CANVAS_BRIDGE_KEY,
|
|
2287
|
+
mode: 'semantic',
|
|
2288
|
+
});
|
|
2289
|
+
|
|
2290
|
+
// ⚠️ 实测结论:在**刚创建的 BrowserView** 上,await dbg.sendCommand('Runtime.enable')
|
|
2291
|
+
// 永远不返回(试过 20 秒、8 次重试,全部挂死),而同样"发了不等"的
|
|
2292
|
+
// Network.enable 却是生效的。也就是说:命令能送达,但响应在这个阶段回不来。
|
|
2293
|
+
// 因此这里一律 **fire-and-forget**,改用 Runtime.evaluate 去**验证**是否真的装上。
|
|
2294
|
+
// await 的 CDP 命令在**页面加载之后**是可用的(network 的 getResponseBody
|
|
2295
|
+
// 就是证据),但在刚创建的 tab 上会挂死。因此本方法只在加载完成后被调用,
|
|
2296
|
+
// 这里一律 await,并把每一步的失败原因打出来便于定位。
|
|
2297
|
+
const step = async (label, method, params, ms = 4000) => {
|
|
2298
|
+
let timer = null;
|
|
2299
|
+
const t = new Promise((_, rej) => {
|
|
2300
|
+
timer = setTimeout(() => rej(new Error(label + ' timeout ' + ms + 'ms')), ms);
|
|
2301
|
+
});
|
|
2302
|
+
try {
|
|
2303
|
+
return await Promise.race([dbg.sendCommand(method, params), t]);
|
|
2304
|
+
} finally {
|
|
2305
|
+
if (timer) clearTimeout(timer);
|
|
2306
|
+
}
|
|
2307
|
+
};
|
|
2308
|
+
const tryStep = async (label, method, params, ms) => {
|
|
2309
|
+
try {
|
|
2310
|
+
await step(label, method, params, ms);
|
|
2311
|
+
log(label, 'ok');
|
|
2312
|
+
return true;
|
|
2313
|
+
} catch (e) {
|
|
2314
|
+
log(label, 'FAIL', (e && e.message) || e);
|
|
2315
|
+
return false;
|
|
2316
|
+
}
|
|
2317
|
+
};
|
|
2318
|
+
|
|
2319
|
+
await tryStep('Runtime.enable', 'Runtime.enable', {}, 4000);
|
|
2320
|
+
await tryStep('Page.enable', 'Page.enable', {}, 4000);
|
|
2321
|
+
await tryStep('Runtime.addBinding', 'Runtime.addBinding', { name: bindingName }, 4000);
|
|
2322
|
+
// 对**后续**导航生效(这才是拿到完整绘制历史的正路)
|
|
2323
|
+
await tryStep('Page.addScriptToEvaluateOnNewDocument', 'Page.addScriptToEvaluateOnNewDocument', { source }, 4000);
|
|
2324
|
+
|
|
2325
|
+
// 对**已经加载过**的当前页面直接注入,并抓运行时异常(脚本是否真跑起来)
|
|
2326
|
+
try {
|
|
2327
|
+
const r = await step('Runtime.evaluate', 'Runtime.evaluate',
|
|
2328
|
+
{ expression: source, returnByValue: true, awaitPromise: false }, 6000);
|
|
2329
|
+
const ed = r && r.exceptionDetails;
|
|
2330
|
+
if (ed) log('hook 脚本运行时异常:', JSON.stringify(ed).slice(0, 800));
|
|
2331
|
+
else log('Runtime.evaluate(注入当前页) ok');
|
|
2332
|
+
} catch (e) {
|
|
2333
|
+
log('Runtime.evaluate FAIL', (e && e.message) || e);
|
|
2334
|
+
}
|
|
2335
|
+
|
|
2336
|
+
this._canvasBindings.set(bindingName, tabId);
|
|
2337
|
+
|
|
2338
|
+
let ok = false;
|
|
2339
|
+
try {
|
|
2340
|
+
const r = await step('probe', 'Runtime.evaluate', {
|
|
2341
|
+
expression: `typeof window[${JSON.stringify(CANVAS_BRIDGE_KEY)}]`,
|
|
2342
|
+
returnByValue: true,
|
|
2343
|
+
}, 4000);
|
|
2344
|
+
const v = r && r.result && r.result.value;
|
|
2345
|
+
log('probe typeof bridge =', v);
|
|
2346
|
+
ok = v === 'object';
|
|
2347
|
+
} catch (e) {
|
|
2348
|
+
log('probe FAIL', (e && e.message) || e);
|
|
2349
|
+
}
|
|
2350
|
+
this.canvasMonitor.markHook(
|
|
2351
|
+
tabId,
|
|
2352
|
+
ok,
|
|
2353
|
+
ok ? [] : ['hook 命令已执行但页面内未生效(见 AI_BROWSER_CANVAS_DEBUG=1 日志)']
|
|
2354
|
+
);
|
|
2355
|
+
return ok;
|
|
2356
|
+
} catch (e) {
|
|
2357
|
+
this.canvasMonitor.markHook(tabId, false, [`canvas hook install failed: ${(e && e.message) || e}`]);
|
|
2358
|
+
return false;
|
|
2359
|
+
}
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2362
|
+
/**
|
|
2363
|
+
* 窗口焦点策略 —— 这是个真实的可用性问题:agent 每次点击/输入都把窗口
|
|
2364
|
+
* 拽到最前,等于人完全没法用电脑。
|
|
2365
|
+
*
|
|
2366
|
+
* AI_BROWSER_FOCUS:
|
|
2367
|
+
* auto (默认) 只在窗口被最小化/隐藏时恢复可见,**不** moveTop、**不** focus。
|
|
2368
|
+
* CDP 受信输入(Input.dispatchKeyEvent / dispatchMouseEvent)在窗口
|
|
2369
|
+
* 非前台时依然生效,加上已有的 disable-backgrounding-occluded-windows /
|
|
2370
|
+
* disable-renderer-backgrounding / disable-background-timer-throttling,
|
|
2371
|
+
* 后台窗口不会被降频或丢事件。
|
|
2372
|
+
* always 旧行为:每次都 show + moveTop + focus(需要抢焦点时用)。
|
|
2373
|
+
* never 完全不碰窗口状态。
|
|
2374
|
+
*/
|
|
2375
|
+
_focusWindow(view) {
|
|
2376
|
+
const policy = String(process.env.AI_BROWSER_FOCUS || 'auto').toLowerCase();
|
|
2377
|
+
if (policy === 'never') return;
|
|
2378
|
+
try {
|
|
2379
|
+
if (policy === 'always') {
|
|
2380
|
+
if (this.window.show) this.window.show();
|
|
2381
|
+
if (this.window.moveTop) this.window.moveTop();
|
|
2382
|
+
if (this.window.focus) this.window.focus();
|
|
2383
|
+
if (view && view.webContents && view.webContents.focus) view.webContents.focus();
|
|
2384
|
+
return;
|
|
2385
|
+
}
|
|
2386
|
+
// auto:产品承诺"窗口真实可见、人可随时介入",所以被最小化/隐藏时要恢复可见;
|
|
2387
|
+
// 但不主动抢到最前 —— 人可以继续用别的窗口,需要时自己点过来。
|
|
2388
|
+
// 用 showInactive(),语义才前后一致:auto 分支**从不**夺走键盘焦点。
|
|
2389
|
+
const hidden = (this.window.isVisible && !this.window.isVisible()) ||
|
|
2390
|
+
(this.window.isMinimized && this.window.isMinimized());
|
|
2391
|
+
if (hidden) {
|
|
2392
|
+
if (this.window.isMinimized && this.window.isMinimized() && this.window.restore) {
|
|
2393
|
+
try { this.window.restore(); } catch (e) { /* 已销毁 */ }
|
|
2394
|
+
}
|
|
2395
|
+
if (typeof this.window.showInactive === 'function') this.window.showInactive();
|
|
2396
|
+
else if (this.window.show) this.window.show();
|
|
2397
|
+
}
|
|
2398
|
+
} catch (e) { /* 窗口已销毁等情况,忽略 */ }
|
|
2399
|
+
}
|
|
2400
|
+
|
|
2401
|
+
_canvasTab(tabId) {
|
|
2402
|
+
return tabId !== undefined ? tabId : this.activeTab;
|
|
2403
|
+
}
|
|
2404
|
+
|
|
2405
|
+
canvasList(tabId) {
|
|
2406
|
+
return this.canvasMonitor.list(this._canvasTab(tabId));
|
|
2407
|
+
}
|
|
2408
|
+
|
|
2409
|
+
canvasRead(tabId, opts = {}) {
|
|
2410
|
+
return this.canvasMonitor.read(this._canvasTab(tabId), opts || {});
|
|
2411
|
+
}
|
|
2412
|
+
|
|
2413
|
+
async canvasConfigure(tabId, opts = {}) {
|
|
2414
|
+
const tid = this._canvasTab(tabId);
|
|
2415
|
+
const mode = this.canvasMonitor.setMode(tid, opts.mode);
|
|
2416
|
+
if (opts.clear) this.canvasMonitor.clear(tid);
|
|
2417
|
+
const view = this._getView(tid);
|
|
2418
|
+
if (view) {
|
|
2419
|
+
await this._ensureCanvasHook(tid, view);
|
|
2420
|
+
// 只有调用方显式要求才重载:hook 对"安装之前已经画完"的内容无能为力,
|
|
2421
|
+
// 重载是唯一能拿到完整绘制历史的办法 —— 但它会丢掉用户当前状态,
|
|
2422
|
+
// 所以绝不默认做。
|
|
2423
|
+
if (opts.reload === true) {
|
|
2424
|
+
try { view.webContents.reload(); } catch (e) {}
|
|
2425
|
+
}
|
|
2426
|
+
}
|
|
2427
|
+
const st = this.canvasMonitor.list(tid);
|
|
2428
|
+
return {
|
|
2429
|
+
mode,
|
|
2430
|
+
hook_installed: st.hook_installed,
|
|
2431
|
+
// hook 只对安装之后的绘制生效;安装前已经画完的内容拿不回来。
|
|
2432
|
+
effective: 'immediate_future_calls',
|
|
2433
|
+
past_content_recovered: false,
|
|
2434
|
+
reload_required_for_complete_history: !st.hook_installed,
|
|
2435
|
+
retention: st.retention,
|
|
2436
|
+
warnings: st.warnings,
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
// 截图兜底:只在语义层读不懂(WebGL/纯像素)时由调用方显式触发。
|
|
2441
|
+
// 默认关闭语义由调用方控制;这里只提供能力并明确警告它会消耗视觉 token。
|
|
2442
|
+
async canvasCapture(tabId, canvasId) {
|
|
2443
|
+
const tid = this._canvasTab(tabId);
|
|
2444
|
+
const view = this._getView(tid);
|
|
2445
|
+
if (!view) return null;
|
|
2446
|
+
const dbg = view.webContents && view.webContents.debugger;
|
|
2447
|
+
if (!dbg || typeof dbg.isAttached !== 'function' || !dbg.isAttached()) return null;
|
|
2448
|
+
const rect = await this.evaluate(
|
|
2449
|
+
`(function(id){var els=document.querySelectorAll('canvas');` +
|
|
2450
|
+
`for(var i=0;i<els.length;i++){var e=els[i];` +
|
|
2451
|
+
`if(e.__aiCanvasId===id||e.id===id){var r=e.getBoundingClientRect();` +
|
|
2452
|
+
`return {x:Math.round(r.x),y:Math.round(r.y),width:Math.round(r.width),height:Math.round(r.height)};}}` +
|
|
2453
|
+
`return null;})(${JSON.stringify(String(canvasId || ''))})`,
|
|
2454
|
+
tid
|
|
2455
|
+
);
|
|
2456
|
+
if (!rect || !rect.width || !rect.height) return null;
|
|
2457
|
+
try {
|
|
2458
|
+
const res = await dbg.sendCommand('Page.captureScreenshot', {
|
|
2459
|
+
format: 'png',
|
|
2460
|
+
clip: { x: rect.x, y: rect.y, width: rect.width, height: rect.height, scale: 1 },
|
|
2461
|
+
});
|
|
2462
|
+
return {
|
|
2463
|
+
mime_type: 'image/png',
|
|
2464
|
+
data_base64: (res && res.data) || null,
|
|
2465
|
+
width: rect.width,
|
|
2466
|
+
height: rect.height,
|
|
2467
|
+
warning: 'visual fallback; consumes caller visual tokens',
|
|
2468
|
+
};
|
|
2469
|
+
} catch (e) {
|
|
2470
|
+
return null;
|
|
2471
|
+
}
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
_ensureCdp(tabId, view, domain) {
|
|
2475
|
+
try {
|
|
2476
|
+
const wc = view.webContents;
|
|
2477
|
+
if (!this._cdpTabs.has(tabId)) {
|
|
2478
|
+
wc.debugger.attach('1.3');
|
|
2479
|
+
const entry = { wc, domains: new Set() };
|
|
2480
|
+
this._cdpTabs.set(tabId, entry);
|
|
2481
|
+
// Single shared listener per tab -> no duplicate broadcasts even when
|
|
2482
|
+
// many sessions subscribe to network/runtime events on the same tab.
|
|
2483
|
+
wc.debugger.on('message', (_event, method, params) => this._onCdpMessage(tabId, method, params));
|
|
2484
|
+
}
|
|
2485
|
+
const entry = this._cdpTabs.get(tabId);
|
|
2486
|
+
if (!entry.domains.has(domain)) {
|
|
2487
|
+
wc.debugger.sendCommand(domain + '.enable');
|
|
2488
|
+
entry.domains.add(domain);
|
|
2489
|
+
}
|
|
2490
|
+
} catch (e) { /* mid-navigation / already-attached elsewhere; skip this tab */ }
|
|
2491
|
+
}
|
|
2492
|
+
|
|
2493
|
+
// 抓包用的 Network 域:默认在 tab 创建后、首个导航前启用。
|
|
2494
|
+
// Network.enable 必须显式带 maxPostDataSize,否则 Chromium 可能不内联
|
|
2495
|
+
// request.postData(POST 请求体就拿不到,只能靠 getRequestPostData 补)。
|
|
2496
|
+
// 实验参数(durable messages / buffer size)在部分 Chromium 上不支持,
|
|
2497
|
+
// 逐个降级,最后一档是裸 Network.enable。
|
|
2498
|
+
_ensureNetworkCapture(tabId, view) {
|
|
2499
|
+
const cfg = this.networkMonitor.tabConfig(tabId);
|
|
2500
|
+
if (!cfg.enabled) return;
|
|
2501
|
+
try {
|
|
2502
|
+
const wc = view.webContents;
|
|
2503
|
+
if (!this._cdpTabs.has(tabId)) {
|
|
2504
|
+
wc.debugger.attach('1.3');
|
|
2505
|
+
const entry = { wc, domains: new Set() };
|
|
2506
|
+
this._cdpTabs.set(tabId, entry);
|
|
2507
|
+
wc.debugger.on('message', (_event, method, params) => this._onCdpMessage(tabId, method, params));
|
|
2508
|
+
}
|
|
2509
|
+
const entry = this._cdpTabs.get(tabId);
|
|
2510
|
+
if (entry.domains.has('Network')) return;
|
|
2511
|
+
const attempts = [
|
|
2512
|
+
{
|
|
2513
|
+
maxPostDataSize: MAX_POST_DATA_SIZE,
|
|
2514
|
+
maxTotalBufferSize: 32 * 1024 * 1024,
|
|
2515
|
+
maxResourceBufferSize: 8 * 1024 * 1024,
|
|
2516
|
+
enableDurableMessages: true,
|
|
2517
|
+
},
|
|
2518
|
+
{ maxPostDataSize: MAX_POST_DATA_SIZE },
|
|
2519
|
+
{},
|
|
2520
|
+
];
|
|
2521
|
+
this.networkMonitor.setBodyFetcher(tabId, async (requestId, kind) => {
|
|
2522
|
+
if (!wc.debugger || typeof wc.debugger.isAttached !== 'function' || !wc.debugger.isAttached()) return null;
|
|
2523
|
+
if (kind === 'request') {
|
|
2524
|
+
const r = await wc.debugger.sendCommand('Network.getRequestPostData', { requestId });
|
|
2525
|
+
if (!r || r.postData == null) return null;
|
|
2526
|
+
// multipart 的文件字节 CDP 不保证给出 —— 如实标 incomplete。
|
|
2527
|
+
return { data: r.postData, complete: true };
|
|
2528
|
+
}
|
|
2529
|
+
const r = await wc.debugger.sendCommand('Network.getResponseBody', { requestId });
|
|
2530
|
+
if (!r) return null;
|
|
2531
|
+
return { data: r.body || '', base64Encoded: !!r.base64Encoded, complete: true };
|
|
2532
|
+
});
|
|
2533
|
+
const tryEnable = (i) => {
|
|
2534
|
+
if (i >= attempts.length) return;
|
|
2535
|
+
Promise.resolve(wc.debugger.sendCommand('Network.enable', attempts[i]))
|
|
2536
|
+
.then(() => { entry.domains.add('Network'); })
|
|
2537
|
+
.catch(() => tryEnable(i + 1));
|
|
2538
|
+
};
|
|
2539
|
+
tryEnable(0);
|
|
2540
|
+
} catch (e) { /* mid-navigation / already attached; skip */ }
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
_teardownCdpIfIdle() {
|
|
2544
|
+
// 抓包默认开启时 debugger 是常驻资源,不能因为最后一个订阅者离开就 detach,
|
|
2545
|
+
// 否则后续 ui.network_list 拿不到任何东西。
|
|
2546
|
+
if (NETWORK_CAPTURE_DEFAULT) return;
|
|
2547
|
+
if (this._networkSubscribers.size > 0 || this._runtimeSubscribers.size > 0) return;
|
|
2548
|
+
for (const entry of this._cdpTabs.values()) {
|
|
2549
|
+
try { entry.wc.debugger.detach(); } catch(e) {}
|
|
2550
|
+
}
|
|
2551
|
+
this._cdpTabs.clear();
|
|
2552
|
+
}
|
|
2553
|
+
|
|
2554
|
+
async _startNetworkMonitor(sessionId) {
|
|
2555
|
+
if (this._networkSubscribers.has(sessionId)) return;
|
|
2556
|
+
this._networkSubscribers.add(sessionId);
|
|
2557
|
+
for (const [tabId, view] of this.tabs) this._ensureNetworkCapture(tabId, view);
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
async _startRuntimeMonitor(sessionId) {
|
|
2561
|
+
if (this._runtimeSubscribers.has(sessionId)) return;
|
|
2562
|
+
this._runtimeSubscribers.add(sessionId);
|
|
2563
|
+
for (const [tabId, view] of this.tabs) this._ensureCdp(tabId, view, 'Runtime');
|
|
2564
|
+
}
|
|
2565
|
+
|
|
2566
|
+
_onCdpMessage(tabId, method, params) {
|
|
2567
|
+
// 抓包:请求侧(method / headers / POST body / initiator)与响应侧都落进
|
|
2568
|
+
// network_monitor;旧的网络事件推送行为保持不变。
|
|
2569
|
+
if (typeof method === 'string' && method.indexOf('Network.') === 0) {
|
|
2570
|
+
this._onNetworkCdpMessage(tabId, method, params);
|
|
2571
|
+
return;
|
|
2572
|
+
}
|
|
2573
|
+
if (method === 'Runtime.bindingCalled') {
|
|
2574
|
+
// canvas hook 的上报通道。binding 名每 tab 随机,据此把载荷归到正确的 tab。
|
|
2575
|
+
const owner = this._canvasBindings.get(params && params.name);
|
|
2576
|
+
if (owner !== undefined && params && typeof params.payload === 'string') {
|
|
2577
|
+
try {
|
|
2578
|
+
this.canvasMonitor.ingest(owner, JSON.parse(params.payload));
|
|
2579
|
+
} catch (e) { /* 坏载荷不该影响页面,也不该冒到上层 */ }
|
|
2580
|
+
}
|
|
2581
|
+
return;
|
|
2582
|
+
}
|
|
2583
|
+
if (method === 'Runtime.exceptionThrown') {
|
|
2584
|
+
// CDP captures page main-world exceptions that preload window.onerror
|
|
2585
|
+
// (an isolated world) can never see under contextIsolation.
|
|
2586
|
+
const d = params.exceptionDetails || {};
|
|
2587
|
+
const ex = d.exception || {};
|
|
2588
|
+
this._broadcast('js_error', {
|
|
2589
|
+
text: d.text || '',
|
|
2590
|
+
message: (ex.description || ex.value) || d.text || '',
|
|
2591
|
+
url: d.url || '',
|
|
2592
|
+
lineNumber: d.lineNumber,
|
|
2593
|
+
columnNumber: d.columnNumber,
|
|
2594
|
+
tabId,
|
|
2595
|
+
});
|
|
2596
|
+
}
|
|
2597
|
+
}
|
|
2598
|
+
|
|
2599
|
+
_onNetworkCdpMessage(tabId, method, params) {
|
|
2600
|
+
const m = this.networkMonitor;
|
|
2601
|
+
switch (method) {
|
|
2602
|
+
case 'Network.requestWillBeSent': {
|
|
2603
|
+
const rec = m.onRequestWillBeSent(tabId, params);
|
|
2604
|
+
// 旧 getNetworkBody 的索引(结构保持 {url, tabId, finished})。
|
|
2605
|
+
this._networkRequestMap.set(params.requestId, {
|
|
2606
|
+
url: rec ? rec.url : (params.request && params.request.url) || '',
|
|
2607
|
+
tabId,
|
|
2608
|
+
finished: false,
|
|
2609
|
+
});
|
|
2610
|
+
return;
|
|
2611
|
+
}
|
|
2612
|
+
case 'Network.requestWillBeSentExtraInfo':
|
|
2613
|
+
m.onRequestWillBeSentExtraInfo(tabId, params);
|
|
2614
|
+
return;
|
|
2615
|
+
case 'Network.responseReceivedExtraInfo':
|
|
2616
|
+
m.onResponseReceivedExtraInfo(tabId, params);
|
|
2617
|
+
return;
|
|
2618
|
+
case 'Network.dataReceived':
|
|
2619
|
+
m.onDataReceived(tabId, params);
|
|
2620
|
+
return;
|
|
2621
|
+
case 'Network.responseReceived': {
|
|
2622
|
+
m.onResponseReceived(tabId, params);
|
|
2623
|
+
const r = params.response || {};
|
|
2624
|
+
const entry = this._networkRequestMap.get(params.requestId);
|
|
2625
|
+
if (entry) entry.url = r.url;
|
|
2626
|
+
else this._networkRequestMap.set(params.requestId, { url: r.url, tabId, finished: false });
|
|
2627
|
+
this._broadcast('network_response', { url: r.url, status: r.status, statusText: r.statusText, mimeType: r.mimeType, tabId });
|
|
2628
|
+
return;
|
|
2629
|
+
}
|
|
2630
|
+
case 'Network.loadingFinished': {
|
|
2631
|
+
m.onLoadingFinished(tabId, params);
|
|
2632
|
+
const entry = this._networkRequestMap.get(params.requestId);
|
|
2633
|
+
if (entry) entry.finished = true;
|
|
2634
|
+
return;
|
|
2635
|
+
}
|
|
2636
|
+
case 'Network.loadingFailed': {
|
|
2637
|
+
m.onLoadingFailed(tabId, params);
|
|
2638
|
+
const requestId = params.requestId || '';
|
|
2639
|
+
const entry = this._networkRequestMap.get(requestId);
|
|
2640
|
+
const url = entry ? entry.url : requestId;
|
|
2641
|
+
this._broadcast('network_response', { url, status: 0, statusText: 'Failed', errorText: params.errorText || '', tabId });
|
|
2642
|
+
return;
|
|
2643
|
+
}
|
|
2644
|
+
default:
|
|
2645
|
+
return;
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
|
|
2649
|
+
// ==== 网络抓包查询(ui.network_list / get / clear / configure)====
|
|
2650
|
+
|
|
2651
|
+
networkList(filter = {}) {
|
|
2652
|
+
return this.networkMonitor.list(filter);
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2655
|
+
async networkGet(networkId, opts = {}) {
|
|
2656
|
+
return await this.networkMonitor.get(networkId, opts);
|
|
2657
|
+
}
|
|
2658
|
+
|
|
2659
|
+
networkClear(tabId) {
|
|
2660
|
+
const cleared = this.networkMonitor.clear(tabId);
|
|
2661
|
+
// 只清内存日志。浏览器 cache / cookie / storage 一概不动 —— 命名与响应都
|
|
2662
|
+
// 要避免被调用方误以为清了会话。
|
|
2663
|
+
return {
|
|
2664
|
+
ok: true,
|
|
2665
|
+
...cleared,
|
|
2666
|
+
cleared_browser_cache: false,
|
|
2667
|
+
cleared_cookies: false,
|
|
2668
|
+
cleared_storage: false,
|
|
2669
|
+
scope: 'in_memory_request_log_only',
|
|
2670
|
+
warning: 'only the in-memory request log was cleared; browser cache, cookies and storage are untouched',
|
|
2671
|
+
};
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
networkConfigure(tabId, opts = {}) {
|
|
2675
|
+
// 上限只能由受控 env/config 改,调用方不能通过 WS 设成无限。
|
|
2676
|
+
const cfg = this.networkMonitor.configure(tabId, opts);
|
|
2677
|
+
if (cfg.enabled) {
|
|
2678
|
+
const view = this._getView(tabId);
|
|
2679
|
+
if (view) this._ensureNetworkCapture(tabId, view);
|
|
2680
|
+
}
|
|
2681
|
+
return {
|
|
2682
|
+
enabled: cfg.enabled,
|
|
2683
|
+
capture_bodies: cfg.capture_bodies,
|
|
2684
|
+
effective: cfg.enabled ? 'future_requests' : 'disabled',
|
|
2685
|
+
reload_required_for_initial_navigation: true,
|
|
2686
|
+
capabilities: {
|
|
2687
|
+
request_headers: true,
|
|
2688
|
+
request_body: true,
|
|
2689
|
+
durable_response_bodies: false,
|
|
2690
|
+
sensitive_headers_optin: this.networkMonitor.sensitiveAllowed,
|
|
2691
|
+
},
|
|
2692
|
+
limits: this.networkMonitor.captureInfo(tabId).limits,
|
|
2693
|
+
note: 'retention limits are controlled by AI_BROWSER_NETWORK_* env only, never by this call',
|
|
2694
|
+
};
|
|
2695
|
+
}
|
|
2696
|
+
|
|
2697
|
+
async _stopNetworkMonitor(sessionId) {
|
|
2698
|
+
if (!this._networkSubscribers.has(sessionId)) return;
|
|
2699
|
+
this._networkSubscribers.delete(sessionId);
|
|
2700
|
+
this._teardownCdpIfIdle();
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
async _stopRuntimeMonitor(sessionId) {
|
|
2704
|
+
if (!this._runtimeSubscribers.has(sessionId)) return;
|
|
2705
|
+
this._runtimeSubscribers.delete(sessionId);
|
|
2706
|
+
this._teardownCdpIfIdle();
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
close() {
|
|
2710
|
+
// Detach every shared per-tab CDP debugger once, then drop all state.
|
|
2711
|
+
for (const entry of this._cdpTabs.values()) {
|
|
2712
|
+
try { entry.wc.debugger.detach(); } catch(e) {}
|
|
2713
|
+
}
|
|
2714
|
+
this._cdpTabs.clear();
|
|
2715
|
+
this._networkSubscribers.clear();
|
|
2716
|
+
this._runtimeSubscribers.clear();
|
|
2717
|
+
this.networkMonitor.clear();
|
|
2718
|
+
// Reject any in-flight requests so callers don't hang on close.
|
|
2719
|
+
for (const [id, pending] of this._pendingRequests) {
|
|
2720
|
+
try { pending.reject(new Error('PageManager closing')); } catch (e) {}
|
|
2721
|
+
}
|
|
2722
|
+
this._pendingRequests.clear();
|
|
2723
|
+
// Same reasoning as closeTab: baselines and queued repaints belong to tabs
|
|
2724
|
+
// that are about to stop existing.
|
|
2725
|
+
this._snapshots.clear();
|
|
2726
|
+
this.downloadMonitor.clearAll();
|
|
2727
|
+
for (const t of this._repaintTimers.values()) clearTimeout(t);
|
|
2728
|
+
this._repaintTimers.clear();
|
|
2729
|
+
for (const [id, view] of this.tabs) {
|
|
2730
|
+
if (view.webContents && !view.webContents.isDestroyed()) {
|
|
2731
|
+
view.webContents.close();
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2734
|
+
this.tabs.clear();
|
|
2735
|
+
ipcMain.removeAllListeners('ai:tree');
|
|
2736
|
+
ipcMain.removeAllListeners('ai:action_result');
|
|
2737
|
+
ipcMain.removeAllListeners('ai:evaluate_result');
|
|
2738
|
+
ipcMain.removeAllListeners('ai:diff');
|
|
2739
|
+
ipcMain.removeAllListeners('ai:event');
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
|
|
2743
|
+
export { PageManager };
|