@pyrokine/mcp-chrome 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -43
- package/dist/core/browser-driver.d.ts +118 -1
- package/dist/core/browser-driver.d.ts.map +1 -1
- package/dist/core/browser-driver.js +11 -0
- package/dist/core/browser-driver.js.map +1 -1
- package/dist/core/error-sanitizer.js +2 -2
- package/dist/core/error-sanitizer.js.map +1 -1
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +17 -4
- package/dist/core/errors.js.map +1 -1
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/locator.d.ts.map +1 -1
- package/dist/core/locator.js +4 -3
- package/dist/core/locator.js.map +1 -1
- package/dist/core/session.d.ts +5 -2
- package/dist/core/session.d.ts.map +1 -1
- package/dist/core/session.js +107 -4
- package/dist/core/session.js.map +1 -1
- package/dist/core/types.d.ts +17 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +57 -4
- package/dist/core/types.js.map +1 -1
- package/dist/core/unified-session.d.ts +29 -9
- package/dist/core/unified-session.d.ts.map +1 -1
- package/dist/core/unified-session.js +226 -18
- package/dist/core/unified-session.js.map +1 -1
- package/dist/core/utils.d.ts +13 -0
- package/dist/core/utils.d.ts.map +1 -1
- package/dist/core/utils.js +121 -0
- package/dist/core/utils.js.map +1 -1
- package/dist/extension/bridge.d.ts +37 -22
- package/dist/extension/bridge.d.ts.map +1 -1
- package/dist/extension/bridge.js +256 -96
- package/dist/extension/bridge.js.map +1 -1
- package/dist/extension/http-server.d.ts.map +1 -1
- package/dist/extension/http-server.js +39 -10
- package/dist/extension/http-server.js.map +1 -1
- package/dist/tools/browse.d.ts.map +1 -1
- package/dist/tools/browse.js +160 -55
- package/dist/tools/browse.js.map +1 -1
- package/dist/tools/cookies.d.ts.map +1 -1
- package/dist/tools/cookies.js +8 -11
- package/dist/tools/cookies.js.map +1 -1
- package/dist/tools/evaluate.d.ts.map +1 -1
- package/dist/tools/evaluate.js +53 -30
- package/dist/tools/evaluate.js.map +1 -1
- package/dist/tools/extract.d.ts +1 -0
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +476 -244
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/input.d.ts.map +1 -1
- package/dist/tools/input.js +798 -441
- package/dist/tools/input.js.map +1 -1
- package/dist/tools/logs.d.ts.map +1 -1
- package/dist/tools/logs.js +79 -31
- package/dist/tools/logs.js.map +1 -1
- package/dist/tools/manage.d.ts +0 -3
- package/dist/tools/manage.d.ts.map +1 -1
- package/dist/tools/manage.js +339 -285
- package/dist/tools/manage.js.map +1 -1
- package/dist/tools/png.d.ts +16 -0
- package/dist/tools/png.d.ts.map +1 -0
- package/dist/tools/png.js +197 -0
- package/dist/tools/png.js.map +1 -0
- package/dist/tools/schema.d.ts +14 -112
- package/dist/tools/schema.d.ts.map +1 -1
- package/dist/tools/schema.js +21 -12
- package/dist/tools/schema.js.map +1 -1
- package/dist/tools/wait.d.ts.map +1 -1
- package/dist/tools/wait.js +73 -13
- package/dist/tools/wait.js.map +1 -1
- package/package.json +11 -11
package/dist/tools/input.js
CHANGED
|
@@ -33,6 +33,9 @@ const inputEventSchema = z.object({
|
|
|
33
33
|
'select',
|
|
34
34
|
'replace',
|
|
35
35
|
'drag',
|
|
36
|
+
'editorContext',
|
|
37
|
+
'editorInsert',
|
|
38
|
+
'editorCommand',
|
|
36
39
|
])
|
|
37
40
|
.describe('事件类型'),
|
|
38
41
|
key: z.string().optional().describe('按键(keydown/keyup)'),
|
|
@@ -60,6 +63,11 @@ const inputEventSchema = z.object({
|
|
|
60
63
|
ms: z.number().optional().describe('等待毫秒'),
|
|
61
64
|
find: z.string().optional().describe('要查找并选中的文本(select/replace)'),
|
|
62
65
|
nth: z.number().optional().describe('第 N 个匹配(select/replace,从 0 开始,默认 0 即第一个)'),
|
|
66
|
+
command: z.string().optional().describe('浏览器编辑命令(editorCommand),如 bold、italic、insertOrderedList'),
|
|
67
|
+
mode: z
|
|
68
|
+
.enum(['keyboard', 'controlled'])
|
|
69
|
+
.optional()
|
|
70
|
+
.describe('输入模式(type),keyboard=键盘事件,controlled=直接设置 value 并触发 input/change 事件'),
|
|
63
71
|
dispatch: z
|
|
64
72
|
.boolean()
|
|
65
73
|
.optional()
|
|
@@ -68,6 +76,7 @@ const inputEventSchema = z.object({
|
|
|
68
76
|
.boolean()
|
|
69
77
|
.optional()
|
|
70
78
|
.describe('强制执行(click),跳过可操作性检查(可见性、遮挡检测等),直接在目标元素上触发事件,用于已知需要绕过检查的场景'),
|
|
79
|
+
forceReason: z.string().optional().describe('force=true 时必填,说明为什么需要跳过可操作性检查'),
|
|
71
80
|
});
|
|
72
81
|
/**
|
|
73
82
|
* input 参数 schema
|
|
@@ -75,6 +84,7 @@ const inputEventSchema = z.object({
|
|
|
75
84
|
const inputSchema = z.object({
|
|
76
85
|
events: z.array(inputEventSchema).describe('事件序列'),
|
|
77
86
|
humanize: z.boolean().optional().describe('启用人类行为模拟(贝塞尔曲线移动、随机延迟)'),
|
|
87
|
+
diagnostics: z.boolean().optional().describe('执行后返回新增 console error/warning 和失败网络请求摘要'),
|
|
78
88
|
tabId: z
|
|
79
89
|
.string()
|
|
80
90
|
.optional()
|
|
@@ -85,6 +95,28 @@ const inputSchema = z.object({
|
|
|
85
95
|
.optional()
|
|
86
96
|
.describe('iframe 定位(可选,仅 Extension 模式),CSS 选择器(如 "iframe#main")或索引(如 0),不指定则在主框架操作'),
|
|
87
97
|
});
|
|
98
|
+
class StructuredToolError extends Error {
|
|
99
|
+
code;
|
|
100
|
+
suggestion;
|
|
101
|
+
context;
|
|
102
|
+
constructor(code, message, suggestion, context) {
|
|
103
|
+
super(message);
|
|
104
|
+
this.code = code;
|
|
105
|
+
this.suggestion = suggestion;
|
|
106
|
+
this.context = context;
|
|
107
|
+
this.name = 'StructuredToolError';
|
|
108
|
+
}
|
|
109
|
+
toJSON() {
|
|
110
|
+
return {
|
|
111
|
+
error: {
|
|
112
|
+
code: this.code,
|
|
113
|
+
message: this.message,
|
|
114
|
+
suggestion: this.suggestion,
|
|
115
|
+
context: this.context,
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
}
|
|
88
120
|
/**
|
|
89
121
|
* input 工具处理器
|
|
90
122
|
*/
|
|
@@ -95,31 +127,17 @@ async function handleInput(args) {
|
|
|
95
127
|
const humanize = args.humanize ?? false;
|
|
96
128
|
return await unifiedSession.withTabId(args.tabId, async () => {
|
|
97
129
|
return await unifiedSession.withFrame(args.frame, async () => {
|
|
98
|
-
|
|
130
|
+
const diagnosticsStart = args.diagnostics ? await captureDiagnosticsStart(unifiedSession) : undefined;
|
|
99
131
|
const warnings = [];
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
}
|
|
132
|
+
const eventResults = [];
|
|
133
|
+
const session = mode === 'extension' ? undefined : getSession();
|
|
134
|
+
for (const event of args.events) {
|
|
135
|
+
const result = await executeInputEvent({ unifiedSession, session, mode, humanize, timeout: args.timeout }, event);
|
|
136
|
+
if (typeof result === 'string') {
|
|
137
|
+
warnings.push(result);
|
|
107
138
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
// CDP 模式:逐事件分发(无 Extension bridge)
|
|
111
|
-
const session = getSession();
|
|
112
|
-
for (const event of args.events) {
|
|
113
|
-
if (event.type === 'select' || event.type === 'replace') {
|
|
114
|
-
// select/replace 通过 unifiedSession(内部自适应双模式)
|
|
115
|
-
const w = await executeEventExtension(unifiedSession, event, humanize, args.timeout);
|
|
116
|
-
if (w) {
|
|
117
|
-
warnings.push(w);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
else {
|
|
121
|
-
await executeEvent(session, event, humanize, args.timeout);
|
|
122
|
-
}
|
|
139
|
+
else if (result) {
|
|
140
|
+
eventResults.push(result);
|
|
123
141
|
}
|
|
124
142
|
}
|
|
125
143
|
const result = {
|
|
@@ -130,6 +148,12 @@ async function handleInput(args) {
|
|
|
130
148
|
if (warnings.length > 0) {
|
|
131
149
|
result.warnings = warnings;
|
|
132
150
|
}
|
|
151
|
+
if (eventResults.length > 0) {
|
|
152
|
+
result.eventResults = eventResults;
|
|
153
|
+
}
|
|
154
|
+
if (diagnosticsStart) {
|
|
155
|
+
result.diagnostics = await captureDiagnosticsDelta(unifiedSession, diagnosticsStart);
|
|
156
|
+
}
|
|
133
157
|
return formatResponse(result);
|
|
134
158
|
}); // withFrame
|
|
135
159
|
}); // withTabId
|
|
@@ -138,6 +162,44 @@ async function handleInput(args) {
|
|
|
138
162
|
return formatErrorResponse(error);
|
|
139
163
|
}
|
|
140
164
|
}
|
|
165
|
+
const unifiedOnlyEventTypes = new Set([
|
|
166
|
+
'select',
|
|
167
|
+
'replace',
|
|
168
|
+
'editorContext',
|
|
169
|
+
'editorInsert',
|
|
170
|
+
'editorCommand',
|
|
171
|
+
]);
|
|
172
|
+
async function executeInputEvent(context, event) {
|
|
173
|
+
if (context.mode === 'extension' || unifiedOnlyEventTypes.has(event.type)) {
|
|
174
|
+
return executeUnifiedEvent(context.unifiedSession, event, context.humanize, context.timeout);
|
|
175
|
+
}
|
|
176
|
+
if (!context.session) {
|
|
177
|
+
throw new Error('CDP 输入事件缺少 session');
|
|
178
|
+
}
|
|
179
|
+
await executeCdpEvent(context.session, event, context.humanize, context.timeout);
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
async function captureDiagnosticsStart(unifiedSession) {
|
|
183
|
+
await unifiedSession.enableConsole();
|
|
184
|
+
await unifiedSession.enableNetwork();
|
|
185
|
+
const consoleLogs = await unifiedSession.getConsoleLogs();
|
|
186
|
+
const network = await unifiedSession.getNetworkRequests();
|
|
187
|
+
return { consoleCount: consoleLogs.length, networkCount: network.length };
|
|
188
|
+
}
|
|
189
|
+
async function captureDiagnosticsDelta(unifiedSession, start) {
|
|
190
|
+
const consoleLogs = await unifiedSession.getConsoleLogs();
|
|
191
|
+
const network = await unifiedSession.getNetworkRequests();
|
|
192
|
+
return {
|
|
193
|
+
console: consoleLogs
|
|
194
|
+
.slice(start.consoleCount)
|
|
195
|
+
.filter((item) => ['error', 'warning', 'warn'].includes(item.level))
|
|
196
|
+
.slice(-20),
|
|
197
|
+
failedRequests: network
|
|
198
|
+
.slice(start.networkCount)
|
|
199
|
+
.filter((item) => item.errorText || (item.status !== undefined && item.status >= 400))
|
|
200
|
+
.slice(-20),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
141
203
|
/**
|
|
142
204
|
* 通过真实鼠标事件选中页面文本
|
|
143
205
|
*
|
|
@@ -151,6 +213,27 @@ async function handleInput(args) {
|
|
|
151
213
|
* 如果 target 是选择器类型,先通过 actionableClick 聚焦
|
|
152
214
|
* select/replace 事件用,保证选区建立前 activeElement 就是目标
|
|
153
215
|
*/
|
|
216
|
+
async function focusTargetForCommands(unifiedSession, target, timeout) {
|
|
217
|
+
if ('x' in target || 'y' in target) {
|
|
218
|
+
return true;
|
|
219
|
+
}
|
|
220
|
+
const params = targetToFindParams(target);
|
|
221
|
+
const els = await unifiedSession.find(params.selector, params.text, params.xpath, timeout);
|
|
222
|
+
const nth0 = params.nth ?? 0;
|
|
223
|
+
if (els.length <= nth0) {
|
|
224
|
+
return false;
|
|
225
|
+
}
|
|
226
|
+
const focused = await unifiedSession.evaluate(`(() => {
|
|
227
|
+
const ref = window.__mcpElementMap?.[${JSON.stringify(els[nth0].refId)}]
|
|
228
|
+
const el = ref?.deref()
|
|
229
|
+
if (!(el instanceof HTMLElement)) {
|
|
230
|
+
throw new Error('命令聚焦目标不存在')
|
|
231
|
+
}
|
|
232
|
+
el.focus()
|
|
233
|
+
return document.activeElement === el
|
|
234
|
+
})()`, undefined, timeout);
|
|
235
|
+
return focused === true;
|
|
236
|
+
}
|
|
154
237
|
async function focusTargetIfNeeded(unifiedSession, target, nth, timeout) {
|
|
155
238
|
if (!target || 'x' in target || 'y' in target) {
|
|
156
239
|
return;
|
|
@@ -168,6 +251,98 @@ async function focusTargetIfNeeded(unifiedSession, target, nth, timeout) {
|
|
|
168
251
|
}
|
|
169
252
|
}
|
|
170
253
|
}
|
|
254
|
+
async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout) {
|
|
255
|
+
return await unifiedSession.evaluate(`function(findText, nth, scopeTarget, scopeSelector, scopeText, scopeXpath) {
|
|
256
|
+
function selectorFor(el) {
|
|
257
|
+
if (!(el instanceof Element)) return null;
|
|
258
|
+
if (el.id) return '#' + CSS.escape(el.id);
|
|
259
|
+
var parts = [];
|
|
260
|
+
var current = el;
|
|
261
|
+
while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 4) {
|
|
262
|
+
var part = current.tagName.toLowerCase();
|
|
263
|
+
if (current.classList && current.classList.length) {
|
|
264
|
+
part += '.' + Array.from(current.classList).slice(0, 2).map(function(cls) { return CSS.escape(cls); }).join('.');
|
|
265
|
+
}
|
|
266
|
+
var parent = current.parentElement;
|
|
267
|
+
if (parent) {
|
|
268
|
+
var siblings = Array.from(parent.children).filter(function(child) { return child.tagName === current.tagName; });
|
|
269
|
+
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
|
270
|
+
}
|
|
271
|
+
parts.unshift(part);
|
|
272
|
+
current = parent;
|
|
273
|
+
}
|
|
274
|
+
return parts.join(' > ');
|
|
275
|
+
}
|
|
276
|
+
function summarizeElement(el) {
|
|
277
|
+
if (!(el instanceof Element)) return null;
|
|
278
|
+
var rect = el.getBoundingClientRect();
|
|
279
|
+
var value = 'value' in el ? String(el.value || '') : '';
|
|
280
|
+
var text = (el.innerText || el.textContent || value || '').replace(new RegExp('\\\\s+', 'g'), ' ').trim();
|
|
281
|
+
return {
|
|
282
|
+
tag: el.tagName.toLowerCase(),
|
|
283
|
+
id: el.id || undefined,
|
|
284
|
+
selector: selectorFor(el),
|
|
285
|
+
text: text.slice(0, 160),
|
|
286
|
+
valuePreview: value ? value.slice(0, 160) : undefined,
|
|
287
|
+
visible: rect.width > 0 && rect.height > 0,
|
|
288
|
+
disabled: Boolean(el.disabled),
|
|
289
|
+
readOnly: Boolean(el.readOnly),
|
|
290
|
+
contentEditable: Boolean(el.isContentEditable),
|
|
291
|
+
bounds: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
var root = document.body;
|
|
295
|
+
if (scopeXpath) {
|
|
296
|
+
var xr = document.evaluate(scopeXpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
297
|
+
root = xr.singleNodeValue;
|
|
298
|
+
} else if (scopeSelector) {
|
|
299
|
+
var candidates = document.querySelectorAll(scopeSelector);
|
|
300
|
+
if (scopeText) {
|
|
301
|
+
for (var ci = 0; ci < candidates.length; ci++) {
|
|
302
|
+
if ((candidates[ci].textContent || '').includes(scopeText)) { root = candidates[ci]; break; }
|
|
303
|
+
}
|
|
304
|
+
} else {
|
|
305
|
+
root = candidates[0];
|
|
306
|
+
}
|
|
307
|
+
} else if (scopeText) {
|
|
308
|
+
var all = document.querySelectorAll('*');
|
|
309
|
+
for (var ai = 0; ai < all.length; ai++) {
|
|
310
|
+
if ((all[ai].textContent || '').includes(scopeText)) { root = all[ai]; break; }
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
var candidateElements = [];
|
|
314
|
+
if (root instanceof Element) {
|
|
315
|
+
candidateElements.push(root);
|
|
316
|
+
candidateElements = candidateElements.concat(Array.from(root.querySelectorAll('input, textarea, [contenteditable="true"], [role="textbox"], button, a')).slice(0, 12));
|
|
317
|
+
}
|
|
318
|
+
var active = document.activeElement;
|
|
319
|
+
var selection = window.getSelection();
|
|
320
|
+
return {
|
|
321
|
+
scope: {
|
|
322
|
+
target: scopeTarget,
|
|
323
|
+
nth: nth,
|
|
324
|
+
findText: findText,
|
|
325
|
+
selector: scopeSelector,
|
|
326
|
+
text: scopeText,
|
|
327
|
+
xpath: scopeXpath,
|
|
328
|
+
rootFound: Boolean(root)
|
|
329
|
+
},
|
|
330
|
+
activeElement: summarizeElement(active),
|
|
331
|
+
selection: {
|
|
332
|
+
text: selection ? selection.toString().slice(0, 160) : '',
|
|
333
|
+
collapsed: selection ? selection.isCollapsed : true,
|
|
334
|
+
anchorNode: selection && selection.anchorNode ? selection.anchorNode.nodeName : null,
|
|
335
|
+
focusNode: selection && selection.focusNode ? selection.focusNode.nodeName : null,
|
|
336
|
+
inputSelection: active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA') ? {
|
|
337
|
+
selectionStart: active.selectionStart,
|
|
338
|
+
selectionEnd: active.selectionEnd,
|
|
339
|
+
valuePreview: String(active.value || '').slice(0, 160)
|
|
340
|
+
} : undefined
|
|
341
|
+
},
|
|
342
|
+
candidates: candidateElements.map(summarizeElement).filter(Boolean)
|
|
343
|
+
};
|
|
344
|
+
}`, undefined, timeout, [findText, nth, scopeTarget ?? null, scopeSelector, scopeText, scopeXpath]);
|
|
345
|
+
}
|
|
171
346
|
async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeout) {
|
|
172
347
|
// 将 target 转为查询参数,传入注入脚本进行 DOM 查询
|
|
173
348
|
let scopeSelector = null;
|
|
@@ -301,12 +476,14 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
301
476
|
};
|
|
302
477
|
}`, undefined, timeout, [findText, nth, scopeSelector, scopeText, scopeXpath]);
|
|
303
478
|
if (!result || result.type === 'noscope') {
|
|
304
|
-
|
|
479
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
480
|
+
throw new StructuredToolError('TEXT_SCOPE_NOT_FOUND', `未找到目标元素: ${JSON.stringify(scopeTarget)}`, '请检查 target 是否能定位到包含目标文本的元素,或先用 extract(state) 查看可交互元素', diagnostics);
|
|
305
481
|
}
|
|
306
482
|
if (result.type === 'notfound') {
|
|
307
|
-
|
|
483
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
484
|
+
throw new StructuredToolError('TEXT_NOT_FOUND', scopeTarget
|
|
308
485
|
? `目标元素内未找到文本 "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`
|
|
309
|
-
: `未找到文本: "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}
|
|
486
|
+
: `未找到文本: "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`, '请检查 find 文本、nth 序号和 target 范围;context.candidates 提供了当前范围内的候选文本和 selector', diagnostics);
|
|
310
487
|
}
|
|
311
488
|
if (result.type === 'input') {
|
|
312
489
|
// 注入脚本已完成 focus + setSelectionRange(原子化,避免外层 mouseClick 聚焦不可靠)
|
|
@@ -331,336 +508,398 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
331
508
|
await unifiedSession.keyUp('Shift');
|
|
332
509
|
return coords.formatted;
|
|
333
510
|
}
|
|
511
|
+
async function executeEditingCommands(unifiedSession, commands, timeout) {
|
|
512
|
+
const result = await unifiedSession.evaluate(`function(commands) {
|
|
513
|
+
function summarizeSelection() {
|
|
514
|
+
var active = document.activeElement;
|
|
515
|
+
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA')) {
|
|
516
|
+
return {
|
|
517
|
+
type: active.tagName.toLowerCase(),
|
|
518
|
+
selectionStart: active.selectionStart,
|
|
519
|
+
selectionEnd: active.selectionEnd,
|
|
520
|
+
selectedText: String(active.value || '').slice(active.selectionStart || 0, active.selectionEnd || 0)
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
var selection = window.getSelection();
|
|
524
|
+
return {
|
|
525
|
+
type: 'document',
|
|
526
|
+
text: selection ? selection.toString() : '',
|
|
527
|
+
collapsed: selection ? selection.isCollapsed : true
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
function commandSelectAll() {
|
|
531
|
+
var active = document.activeElement;
|
|
532
|
+
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA') && typeof active.select === 'function') {
|
|
533
|
+
active.select();
|
|
534
|
+
active.dispatchEvent(new Event('select', {bubbles: true}));
|
|
535
|
+
return true;
|
|
536
|
+
}
|
|
537
|
+
if (active && active.isContentEditable) {
|
|
538
|
+
var range = document.createRange();
|
|
539
|
+
range.selectNodeContents(active);
|
|
540
|
+
var selection = window.getSelection();
|
|
541
|
+
if (!selection) return false;
|
|
542
|
+
selection.removeAllRanges();
|
|
543
|
+
selection.addRange(range);
|
|
544
|
+
return true;
|
|
545
|
+
}
|
|
546
|
+
return document.execCommand('selectAll');
|
|
547
|
+
}
|
|
548
|
+
var executed = commands.map(function(command) {
|
|
549
|
+
if (command === 'selectAll') {
|
|
550
|
+
return {command: command, success: commandSelectAll()};
|
|
551
|
+
}
|
|
552
|
+
if (['copy', 'cut', 'paste', 'undo', 'redo'].indexOf(command) !== -1) {
|
|
553
|
+
return {command: command, success: document.execCommand(command)};
|
|
554
|
+
}
|
|
555
|
+
return {command: command, success: false};
|
|
556
|
+
});
|
|
557
|
+
return {success: executed.every(function(item) { return item.success; }), executed: executed, selection: summarizeSelection()};
|
|
558
|
+
}`, undefined, timeout, [commands]);
|
|
559
|
+
if (!result.success) {
|
|
560
|
+
throw new StructuredToolError('EDIT_COMMAND_FAILED', `编辑命令执行失败: ${commands.join(', ')}`, '请确认当前页面已有可编辑焦点,或给 keydown 事件提供 target 参数', result);
|
|
561
|
+
}
|
|
562
|
+
return result;
|
|
563
|
+
}
|
|
334
564
|
/**
|
|
335
565
|
* 验证事件参数(两种执行模式共享),避免在 Extension/CDP 两个 switch 中重复校验
|
|
336
566
|
*/
|
|
567
|
+
function requiredEventParam(event, name) {
|
|
568
|
+
throw new Error(`events[].${String(name)} 是 ${event.type} 事件的必填参数`);
|
|
569
|
+
}
|
|
337
570
|
function validateEvent(event) {
|
|
338
571
|
if (event.commands && event.commands.length > 0 && event.type !== 'keydown') {
|
|
339
|
-
throw new Error(`commands
|
|
572
|
+
throw new Error(`events[].commands 只能用于 keydown 事件,当前事件类型为 ${event.type},如需触发编辑命令,请把 commands 放在 keydown 事件上`);
|
|
573
|
+
}
|
|
574
|
+
if (event.force && !event.forceReason) {
|
|
575
|
+
throw new Error('events[].forceReason 在 force=true 时必填');
|
|
340
576
|
}
|
|
341
577
|
switch (event.type) {
|
|
342
578
|
case 'keydown':
|
|
343
579
|
case 'keyup':
|
|
344
580
|
if (!event.key) {
|
|
345
|
-
|
|
581
|
+
requiredEventParam(event, 'key');
|
|
346
582
|
}
|
|
347
583
|
break;
|
|
348
|
-
case '
|
|
349
|
-
|
|
350
|
-
|
|
584
|
+
case 'mousemove':
|
|
585
|
+
case 'touchstart':
|
|
586
|
+
case 'touchmove':
|
|
587
|
+
if (!event.target) {
|
|
588
|
+
requiredEventParam(event, 'target');
|
|
351
589
|
}
|
|
352
590
|
break;
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
* Extension 模式:执行单个事件
|
|
357
|
-
*
|
|
358
|
-
* @returns 可选警告信息(如格式丢失提示)
|
|
359
|
-
*/
|
|
360
|
-
async function executeEventExtension(unifiedSession, event, humanize, timeout) {
|
|
361
|
-
validateEvent(event);
|
|
362
|
-
switch (event.type) {
|
|
363
|
-
case 'keydown': {
|
|
364
|
-
if (event.commands && event.commands.length > 0 && event.target) {
|
|
365
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
366
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
367
|
-
await unifiedSession.mouseDown('left');
|
|
368
|
-
await unifiedSession.mouseUp('left');
|
|
591
|
+
case 'type':
|
|
592
|
+
if (!event.text) {
|
|
593
|
+
requiredEventParam(event, 'text');
|
|
369
594
|
}
|
|
370
|
-
await unifiedSession.keyDown(event.key, event.commands);
|
|
371
595
|
break;
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
break;
|
|
376
|
-
}
|
|
377
|
-
case 'click': {
|
|
378
|
-
const button = event.button ?? 'left';
|
|
379
|
-
const clickCount = event.clickCount ?? 1;
|
|
380
|
-
if (event.target) {
|
|
381
|
-
// 坐标型 target:不过 actionableClick,但仍需 getTargetPointExtension 做 iframe offset 修正
|
|
382
|
-
if ('x' in event.target && 'y' in event.target) {
|
|
383
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
384
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
385
|
-
await unifiedSession.mouseClick(button, clickCount);
|
|
386
|
-
break;
|
|
387
|
-
}
|
|
388
|
-
// 左键单击:优先用 actionable click(带可操作性检查、自动滚动、遮挡检测)
|
|
389
|
-
// 非左键 / 多击:actionableClick 依赖 HTMLElement.click() 只能触发单次左键,必须走坐标路径
|
|
390
|
-
if (button === 'left' && clickCount === 1) {
|
|
391
|
-
const { selector, text: searchText, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
392
|
-
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
393
|
-
const nth = nthParam ?? 0;
|
|
394
|
-
if (elements.length > 0 && nth < elements.length) {
|
|
395
|
-
const refId = elements[nth].refId;
|
|
396
|
-
const result = await unifiedSession.actionableClick(refId, event.force);
|
|
397
|
-
if (!result.success) {
|
|
398
|
-
throw new Error(result.error || 'Click failed');
|
|
399
|
-
}
|
|
400
|
-
break;
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
// fallback: 找不到 refId 或需走坐标路径时
|
|
404
|
-
// refId 透传:stealth 模式下嵌套 iframe overlay 场景绕过 elementFromPoint
|
|
405
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
406
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
407
|
-
await unifiedSession.mouseClick(button, clickCount, typeof point.refId === 'string' ? point.refId : undefined);
|
|
408
|
-
break;
|
|
596
|
+
case 'select':
|
|
597
|
+
if (!event.find) {
|
|
598
|
+
requiredEventParam(event, 'find');
|
|
409
599
|
}
|
|
410
|
-
await unifiedSession.mouseClick(button, clickCount);
|
|
411
600
|
break;
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
601
|
+
case 'replace':
|
|
602
|
+
if (!event.find) {
|
|
603
|
+
requiredEventParam(event, 'find');
|
|
604
|
+
}
|
|
605
|
+
if (event.text === undefined) {
|
|
606
|
+
requiredEventParam(event, 'text');
|
|
417
607
|
}
|
|
418
|
-
await unifiedSession.mouseDown(event.button ?? 'left');
|
|
419
|
-
break;
|
|
420
|
-
}
|
|
421
|
-
case 'mouseup': {
|
|
422
|
-
await unifiedSession.mouseUp(event.button ?? 'left');
|
|
423
608
|
break;
|
|
424
|
-
|
|
425
|
-
case 'mousemove': {
|
|
609
|
+
case 'drag':
|
|
426
610
|
if (!event.target) {
|
|
427
|
-
|
|
611
|
+
requiredEventParam(event, 'target');
|
|
428
612
|
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
const path = generateBezierPath(unifiedSession.getMousePosition(), point, event.steps);
|
|
432
|
-
for (const p of path) {
|
|
433
|
-
await unifiedSession.mouseMove(p.x, p.y);
|
|
434
|
-
await randomDelay(getMouseMoveDelay(), getMouseMoveDelay() * 2);
|
|
435
|
-
}
|
|
613
|
+
if (!event.to) {
|
|
614
|
+
requiredEventParam(event, 'to');
|
|
436
615
|
}
|
|
437
|
-
|
|
438
|
-
|
|
616
|
+
break;
|
|
617
|
+
case 'editorInsert':
|
|
618
|
+
if (!event.text) {
|
|
619
|
+
requiredEventParam(event, 'text');
|
|
439
620
|
}
|
|
440
621
|
break;
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
const { selector, text, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
445
|
-
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
446
|
-
const nth = nthParam ?? 0;
|
|
447
|
-
if (elements.length > nth) {
|
|
448
|
-
// 用 refId 直接滚动目标元素(支持视口外元素)
|
|
449
|
-
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[nth].refId);
|
|
450
|
-
break;
|
|
451
|
-
}
|
|
452
|
-
// 找不到元素时 fallback 到坐标方式
|
|
453
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
454
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
622
|
+
case 'editorCommand':
|
|
623
|
+
if (!event.command) {
|
|
624
|
+
requiredEventParam(event, 'command');
|
|
455
625
|
}
|
|
456
|
-
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
457
626
|
break;
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
throw new Error('touchstart 事件需要 target 参数');
|
|
627
|
+
case 'wait':
|
|
628
|
+
if (event.ms === undefined) {
|
|
629
|
+
requiredEventParam(event, 'ms');
|
|
462
630
|
}
|
|
463
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
464
|
-
await unifiedSession.touchStart(point.x, point.y);
|
|
465
631
|
break;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
async function executeUnifiedEvent(unifiedSession, event, humanize, timeout) {
|
|
635
|
+
validateEvent(event);
|
|
636
|
+
const handler = unifiedInputHandlers[event.type];
|
|
637
|
+
if (!handler) {
|
|
638
|
+
throw new Error(`未知事件类型: ${event.type}`);
|
|
639
|
+
}
|
|
640
|
+
return handler({ unifiedSession, humanize, timeout }, event);
|
|
641
|
+
}
|
|
642
|
+
const unifiedInputHandlers = {
|
|
643
|
+
keydown: handleUnifiedKeyDown,
|
|
644
|
+
keyup: handleUnifiedKeyUp,
|
|
645
|
+
click: handleUnifiedClick,
|
|
646
|
+
mousedown: handleUnifiedMouseDown,
|
|
647
|
+
mouseup: handleUnifiedMouseUp,
|
|
648
|
+
mousemove: handleUnifiedMouseMove,
|
|
649
|
+
wheel: handleUnifiedWheel,
|
|
650
|
+
touchstart: handleUnifiedTouchStart,
|
|
651
|
+
touchmove: handleUnifiedTouchMove,
|
|
652
|
+
touchend: handleUnifiedTouchEnd,
|
|
653
|
+
type: handleUnifiedType,
|
|
654
|
+
wait: handleUnifiedWait,
|
|
655
|
+
select: handleUnifiedSelect,
|
|
656
|
+
replace: handleUnifiedReplace,
|
|
657
|
+
editorContext: handleUnifiedEditorContext,
|
|
658
|
+
editorInsert: handleUnifiedEditorInsert,
|
|
659
|
+
editorCommand: handleUnifiedEditorCommand,
|
|
660
|
+
drag: handleUnifiedDrag,
|
|
661
|
+
};
|
|
662
|
+
async function handleUnifiedKeyDown({ unifiedSession, timeout }, event) {
|
|
663
|
+
if (unifiedSession.getInputMode() === 'stealth' && event.commands && event.commands.length > 0) {
|
|
664
|
+
throw new Error('commands 参数不支持 stealth 输入模式,请先调用 manage action=inputMode inputMode=precise 切换后重试');
|
|
665
|
+
}
|
|
666
|
+
if (event.commands && event.commands.length > 0 && event.target) {
|
|
667
|
+
const focused = await focusTargetForCommands(unifiedSession, event.target, timeout);
|
|
668
|
+
if (!focused) {
|
|
669
|
+
throw new Error('commands 目标未找到或未成功聚焦');
|
|
466
670
|
}
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
671
|
+
}
|
|
672
|
+
await unifiedSession.keyDown(event.key, event.commands);
|
|
673
|
+
if (event.commands && event.commands.length > 0) {
|
|
674
|
+
return await executeEditingCommands(unifiedSession, event.commands, timeout);
|
|
675
|
+
}
|
|
676
|
+
return undefined;
|
|
677
|
+
}
|
|
678
|
+
async function handleUnifiedKeyUp({ unifiedSession }, event) {
|
|
679
|
+
await unifiedSession.keyUp(event.key);
|
|
680
|
+
return undefined;
|
|
681
|
+
}
|
|
682
|
+
async function handleUnifiedClick({ unifiedSession, timeout }, event) {
|
|
683
|
+
const button = event.button ?? 'left';
|
|
684
|
+
const clickCount = event.clickCount ?? 1;
|
|
685
|
+
if (!event.target) {
|
|
686
|
+
await unifiedSession.mouseClick(button, clickCount);
|
|
687
|
+
return undefined;
|
|
688
|
+
}
|
|
689
|
+
if ('x' in event.target && 'y' in event.target) {
|
|
690
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
691
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
692
|
+
await unifiedSession.mouseClick(button, clickCount);
|
|
693
|
+
return undefined;
|
|
694
|
+
}
|
|
695
|
+
if (button === 'left' && clickCount === 1) {
|
|
696
|
+
const { selector, text: searchText, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
697
|
+
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
698
|
+
const nth = nthParam ?? 0;
|
|
699
|
+
if (elements.length > 0 && nth < elements.length) {
|
|
700
|
+
const result = await unifiedSession.actionableClick(elements[nth].refId, event.force === true);
|
|
701
|
+
if (!result.success) {
|
|
702
|
+
throw new StructuredToolError('ACTIONABILITY_FAILED', result.error || 'Click failed', result.suggestions?.[0] ??
|
|
703
|
+
'请根据 context.rect、context.clickPoint 和 context.coveringElement 调整 target', result);
|
|
470
704
|
}
|
|
471
|
-
|
|
472
|
-
await unifiedSession.touchMove(point.x, point.y);
|
|
473
|
-
break;
|
|
705
|
+
return undefined;
|
|
474
706
|
}
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
707
|
+
}
|
|
708
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
709
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
710
|
+
await unifiedSession.mouseClick(button, clickCount, typeof point.refId === 'string' ? point.refId : undefined);
|
|
711
|
+
return undefined;
|
|
712
|
+
}
|
|
713
|
+
async function handleUnifiedMouseDown({ unifiedSession, timeout }, event) {
|
|
714
|
+
if (event.target) {
|
|
715
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
716
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
717
|
+
}
|
|
718
|
+
await unifiedSession.mouseDown(event.button ?? 'left');
|
|
719
|
+
return undefined;
|
|
720
|
+
}
|
|
721
|
+
async function handleUnifiedMouseUp({ unifiedSession }, event) {
|
|
722
|
+
await unifiedSession.mouseUp(event.button ?? 'left');
|
|
723
|
+
return undefined;
|
|
724
|
+
}
|
|
725
|
+
async function handleUnifiedMouseMove({ unifiedSession, humanize, timeout }, event) {
|
|
726
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
727
|
+
if (humanize && event.steps && event.steps > 1) {
|
|
728
|
+
const path = generateBezierPath(unifiedSession.getMousePosition(), point, event.steps);
|
|
729
|
+
for (const p of path) {
|
|
730
|
+
await unifiedSession.mouseMove(p.x, p.y);
|
|
731
|
+
await randomDelay(getMouseMoveDelay(), getMouseMoveDelay() * 2);
|
|
478
732
|
}
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
494
|
-
const nth = nthParam ?? 0;
|
|
495
|
-
if (elements.length === 0 || nth >= elements.length) {
|
|
496
|
-
throw new Error('目标元素未找到');
|
|
497
|
-
}
|
|
498
|
-
const refId = elements[nth].refId;
|
|
499
|
-
// 通过 Extension ISOLATED 世界执行 dispatch(访问 __mcpElementMap)
|
|
500
|
-
// 参考 Playwright fill():nativeInputValueSetter + dispatchEvent
|
|
501
|
-
const result = await unifiedSession.dispatchInput(refId, event.text);
|
|
502
|
-
if (!result.success) {
|
|
503
|
-
throw new Error(result.error || 'dispatch 输入失败');
|
|
504
|
-
}
|
|
505
|
-
break;
|
|
506
|
-
}
|
|
507
|
-
// 默认模式:键盘事件
|
|
508
|
-
// 如果有 target,先点击目标(聚焦)
|
|
509
|
-
if (event.target) {
|
|
510
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
511
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
512
|
-
await unifiedSession.mouseClick('left');
|
|
513
|
-
}
|
|
514
|
-
else {
|
|
515
|
-
const hasActiveFocus = await unifiedSession.evaluate('!!document.activeElement && ' +
|
|
516
|
-
'document.activeElement !== document.body && ' +
|
|
517
|
-
'document.activeElement !== document.documentElement');
|
|
518
|
-
if (!hasActiveFocus) {
|
|
519
|
-
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
const delay = event.delay ?? 0;
|
|
523
|
-
if (humanize) {
|
|
524
|
-
for (const char of event.text) {
|
|
525
|
-
await unifiedSession.typeText(char);
|
|
526
|
-
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
527
|
-
}
|
|
528
|
-
}
|
|
529
|
-
else {
|
|
530
|
-
await unifiedSession.typeText(event.text, delay);
|
|
531
|
-
}
|
|
532
|
-
break;
|
|
733
|
+
}
|
|
734
|
+
else {
|
|
735
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
736
|
+
}
|
|
737
|
+
return undefined;
|
|
738
|
+
}
|
|
739
|
+
async function handleUnifiedWheel({ unifiedSession, timeout }, event) {
|
|
740
|
+
if (event.target) {
|
|
741
|
+
const { selector, text, xpath, nth: nthParam } = targetToFindParams(event.target);
|
|
742
|
+
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
743
|
+
const nth = nthParam ?? 0;
|
|
744
|
+
if (elements.length > nth) {
|
|
745
|
+
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[nth].refId);
|
|
746
|
+
return undefined;
|
|
533
747
|
}
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
748
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
749
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
750
|
+
}
|
|
751
|
+
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
752
|
+
return undefined;
|
|
753
|
+
}
|
|
754
|
+
async function handleUnifiedTouchStart({ unifiedSession, timeout }, event) {
|
|
755
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
756
|
+
await unifiedSession.touchStart(point.x, point.y);
|
|
757
|
+
return undefined;
|
|
758
|
+
}
|
|
759
|
+
async function handleUnifiedTouchMove({ unifiedSession, timeout }, event) {
|
|
760
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
761
|
+
await unifiedSession.touchMove(point.x, point.y);
|
|
762
|
+
return undefined;
|
|
763
|
+
}
|
|
764
|
+
async function handleUnifiedTouchEnd({ unifiedSession }) {
|
|
765
|
+
await unifiedSession.touchEnd();
|
|
766
|
+
return undefined;
|
|
767
|
+
}
|
|
768
|
+
async function handleUnifiedType({ unifiedSession, humanize, timeout }, event) {
|
|
769
|
+
if (event.mode === 'controlled' || event.dispatch) {
|
|
770
|
+
await inputControlled(unifiedSession, event, timeout);
|
|
771
|
+
return undefined;
|
|
772
|
+
}
|
|
773
|
+
if (event.target) {
|
|
774
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
775
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
776
|
+
await unifiedSession.mouseClick('left');
|
|
777
|
+
}
|
|
778
|
+
else {
|
|
779
|
+
const hasActiveFocus = await unifiedSession.evaluate('!!document.activeElement && ' +
|
|
780
|
+
'document.activeElement !== document.body && ' +
|
|
781
|
+
'document.activeElement !== document.documentElement');
|
|
782
|
+
if (!hasActiveFocus) {
|
|
783
|
+
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
537
784
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
await
|
|
544
|
-
await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
545
|
-
break;
|
|
785
|
+
}
|
|
786
|
+
const delay = event.delay ?? 0;
|
|
787
|
+
if (humanize) {
|
|
788
|
+
for (const char of event.text) {
|
|
789
|
+
await unifiedSession.typeText(char);
|
|
790
|
+
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
546
791
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
})()`);
|
|
569
|
-
if (hasSelection) {
|
|
570
|
-
selectionConfirmed = true;
|
|
571
|
-
break;
|
|
572
|
-
}
|
|
573
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
574
|
-
}
|
|
575
|
-
if (!selectionConfirmed) {
|
|
576
|
-
throw new Error(`选区同步失败:文本 "${event.find}" 已定位但未能建立选区,无法执行替换`);
|
|
577
|
-
}
|
|
578
|
-
// Step 2: 检测可编辑性并替换
|
|
579
|
-
const replaceResult = await unifiedSession.evaluate(`function(replacementText) {
|
|
792
|
+
}
|
|
793
|
+
else {
|
|
794
|
+
await unifiedSession.typeText(event.text, delay);
|
|
795
|
+
}
|
|
796
|
+
return undefined;
|
|
797
|
+
}
|
|
798
|
+
async function handleUnifiedWait(_context, event) {
|
|
799
|
+
await new Promise((resolve) => setTimeout(resolve, event.ms));
|
|
800
|
+
return undefined;
|
|
801
|
+
}
|
|
802
|
+
async function handleUnifiedSelect({ unifiedSession, timeout }, event) {
|
|
803
|
+
await focusTargetIfNeeded(unifiedSession, event.target, event.nth, timeout);
|
|
804
|
+
await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
805
|
+
return undefined;
|
|
806
|
+
}
|
|
807
|
+
async function handleUnifiedReplace({ unifiedSession, timeout }, event) {
|
|
808
|
+
await focusTargetIfNeeded(unifiedSession, event.target, event.nth, timeout);
|
|
809
|
+
const formatted = await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
810
|
+
let selectionConfirmed = false;
|
|
811
|
+
for (let i = 0; i < 25; i++) {
|
|
812
|
+
const hasSelection = await unifiedSession.evaluate(`(function() {
|
|
580
813
|
var el = document.activeElement;
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
if (el.readOnly || el.disabled) return 'readonly';
|
|
584
|
-
el.setRangeText(replacementText, el.selectionStart, el.selectionEnd, 'end');
|
|
585
|
-
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
586
|
-
return 'ok';
|
|
814
|
+
if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA')) {
|
|
815
|
+
return el.selectionStart !== el.selectionEnd;
|
|
587
816
|
}
|
|
588
|
-
// 检测 contenteditable
|
|
589
817
|
var sel = window.getSelection();
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
editable = editable.parentElement;
|
|
595
|
-
}
|
|
596
|
-
if (!editable || !editable.isContentEditable) return 'readonly';
|
|
597
|
-
// contenteditable:execCommand 让浏览器自行产出事件序列
|
|
598
|
-
if (document.execCommand('insertText', false, replacementText)) return 'ok';
|
|
599
|
-
return 'fallback';
|
|
600
|
-
}`, undefined, timeout, [event.text]);
|
|
601
|
-
if (replaceResult === 'readonly') {
|
|
602
|
-
throw new Error(`目标元素不可编辑,已选中文本 "${event.find}" 但无法替换`);
|
|
603
|
-
}
|
|
604
|
-
if (replaceResult === 'fallback') {
|
|
605
|
-
// Fallback: 键盘输入覆盖选区
|
|
606
|
-
await unifiedSession.typeText(event.text);
|
|
607
|
-
}
|
|
608
|
-
if (formatted) {
|
|
609
|
-
return `替换的文本原在 <${formatted}> 标签内,替换后格式可能丢失`;
|
|
610
|
-
}
|
|
818
|
+
return sel && !sel.isCollapsed;
|
|
819
|
+
})()`);
|
|
820
|
+
if (hasSelection) {
|
|
821
|
+
selectionConfirmed = true;
|
|
611
822
|
break;
|
|
612
823
|
}
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
if ('x' in event.to || 'y' in event.to) {
|
|
626
|
-
throw new Error('drag 的 to 不支持坐标类型,请使用选择器(css/text/xpath/role 等)');
|
|
627
|
-
}
|
|
628
|
-
// 用 find 定位确认元素存在(支持 ARIA/testId 等高级定位),拿 refId 传入 extension 侧 dispatchEvent
|
|
629
|
-
const srcParams = targetToFindParams(event.target);
|
|
630
|
-
const dstParams = targetToFindParams(event.to);
|
|
631
|
-
const srcNth = srcParams.nth ?? 0;
|
|
632
|
-
const dstNth = dstParams.nth ?? 0;
|
|
633
|
-
// 执行 drag,失败时重试一次(React 重渲染可能导致 refId 失效)
|
|
634
|
-
const attemptDrag = async () => {
|
|
635
|
-
const srcEls = await unifiedSession.find(srcParams.selector, srcParams.text, srcParams.xpath, timeout);
|
|
636
|
-
const dstEls = await unifiedSession.find(dstParams.selector, dstParams.text, dstParams.xpath, timeout);
|
|
637
|
-
if (srcEls.length <= srcNth) {
|
|
638
|
-
throw new Error(`drag 源元素未找到: ${JSON.stringify(event.target)}`);
|
|
639
|
-
}
|
|
640
|
-
if (dstEls.length <= dstNth) {
|
|
641
|
-
throw new Error(`drag 目标元素未找到: ${JSON.stringify(event.to)}`);
|
|
642
|
-
}
|
|
643
|
-
return unifiedSession.dragAndDrop(srcEls[srcNth].refId, dstEls[dstNth].refId);
|
|
644
|
-
};
|
|
645
|
-
let dragResult = await attemptDrag();
|
|
646
|
-
let retried = false;
|
|
647
|
-
// 仅对 refId 失效(REF_STALE)重试:源/目标元素从 DOM 移除,典型是 React 重渲染
|
|
648
|
-
if (!dragResult.success && dragResult.code === 'REF_STALE') {
|
|
649
|
-
console.warn('[MCP] drag refId 失效,自动重试一次:', dragResult.error);
|
|
650
|
-
dragResult = await attemptDrag();
|
|
651
|
-
retried = true;
|
|
652
|
-
}
|
|
653
|
-
if (!dragResult.success) {
|
|
654
|
-
throw new Error(dragResult.error || 'drag 执行失败');
|
|
655
|
-
}
|
|
656
|
-
if (retried) {
|
|
657
|
-
return 'drag 因 refId 失效已自动重试一次(可能是 React 等框架重渲染导致)';
|
|
658
|
-
}
|
|
659
|
-
break;
|
|
824
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
825
|
+
}
|
|
826
|
+
if (!selectionConfirmed) {
|
|
827
|
+
throw new Error(`选区同步失败:文本 "${event.find}" 已定位但未能建立选区,无法执行替换`);
|
|
828
|
+
}
|
|
829
|
+
const replaceResult = await unifiedSession.evaluate(`function(replacementText) {
|
|
830
|
+
var el = document.activeElement;
|
|
831
|
+
if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') && el.setRangeText) {
|
|
832
|
+
if (el.readOnly || el.disabled) return 'readonly';
|
|
833
|
+
el.setRangeText(replacementText, el.selectionStart, el.selectionEnd, 'end');
|
|
834
|
+
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
835
|
+
return 'ok';
|
|
660
836
|
}
|
|
661
|
-
|
|
662
|
-
|
|
837
|
+
var sel = window.getSelection();
|
|
838
|
+
if (!sel || sel.isCollapsed) return 'readonly';
|
|
839
|
+
var anchor = sel.anchorNode;
|
|
840
|
+
var editable = anchor instanceof Element ? anchor : anchor && anchor.parentElement;
|
|
841
|
+
while (editable && !editable.isContentEditable && editable !== document.body) {
|
|
842
|
+
editable = editable.parentElement;
|
|
843
|
+
}
|
|
844
|
+
if (!editable || !editable.isContentEditable) return 'readonly';
|
|
845
|
+
if (document.execCommand('insertText', false, replacementText)) return 'ok';
|
|
846
|
+
return 'fallback';
|
|
847
|
+
}`, undefined, timeout, [event.text]);
|
|
848
|
+
if (replaceResult === 'readonly') {
|
|
849
|
+
throw new Error(`目标元素不可编辑,已选中文本 "${event.find}" 但无法替换`);
|
|
850
|
+
}
|
|
851
|
+
if (replaceResult === 'fallback') {
|
|
852
|
+
await unifiedSession.typeText(event.text);
|
|
663
853
|
}
|
|
854
|
+
if (formatted) {
|
|
855
|
+
return `替换的文本原在 <${formatted}> 标签内,替换后格式可能丢失`;
|
|
856
|
+
}
|
|
857
|
+
return undefined;
|
|
858
|
+
}
|
|
859
|
+
async function handleUnifiedEditorContext({ unifiedSession, timeout }, event) {
|
|
860
|
+
return (await editorAction(unifiedSession, 'context', event, timeout));
|
|
861
|
+
}
|
|
862
|
+
async function handleUnifiedEditorInsert({ unifiedSession, timeout }, event) {
|
|
863
|
+
await editorAction(unifiedSession, 'insert', event, timeout);
|
|
864
|
+
return undefined;
|
|
865
|
+
}
|
|
866
|
+
async function handleUnifiedEditorCommand({ unifiedSession, timeout }, event) {
|
|
867
|
+
await editorAction(unifiedSession, 'command', event, timeout);
|
|
868
|
+
return undefined;
|
|
869
|
+
}
|
|
870
|
+
async function handleUnifiedDrag({ unifiedSession, timeout }, event) {
|
|
871
|
+
if ('x' in event.target || 'y' in event.target) {
|
|
872
|
+
throw new Error('drag 的 target 不支持坐标类型,请使用选择器(css/text/xpath/role 等)');
|
|
873
|
+
}
|
|
874
|
+
if ('x' in event.to || 'y' in event.to) {
|
|
875
|
+
throw new Error('drag 的 to 不支持坐标类型,请使用选择器(css/text/xpath/role 等)');
|
|
876
|
+
}
|
|
877
|
+
const srcParams = targetToFindParams(event.target);
|
|
878
|
+
const dstParams = targetToFindParams(event.to);
|
|
879
|
+
const srcNth = srcParams.nth ?? 0;
|
|
880
|
+
const dstNth = dstParams.nth ?? 0;
|
|
881
|
+
const attemptDrag = async () => {
|
|
882
|
+
const srcEls = await unifiedSession.find(srcParams.selector, srcParams.text, srcParams.xpath, timeout);
|
|
883
|
+
const dstEls = await unifiedSession.find(dstParams.selector, dstParams.text, dstParams.xpath, timeout);
|
|
884
|
+
if (srcEls.length <= srcNth) {
|
|
885
|
+
throw new Error(`drag 源元素未找到: ${JSON.stringify(event.target)}`);
|
|
886
|
+
}
|
|
887
|
+
if (dstEls.length <= dstNth) {
|
|
888
|
+
throw new Error(`drag 目标元素未找到: ${JSON.stringify(event.to)}`);
|
|
889
|
+
}
|
|
890
|
+
return unifiedSession.dragAndDrop(srcEls[srcNth].refId, dstEls[dstNth].refId);
|
|
891
|
+
};
|
|
892
|
+
let dragResult = await attemptDrag();
|
|
893
|
+
let retried = false;
|
|
894
|
+
if (!dragResult.success && dragResult.code === 'REF_STALE') {
|
|
895
|
+
console.warn('[MCP] drag refId 失效,自动重试一次:', dragResult.error);
|
|
896
|
+
dragResult = await attemptDrag();
|
|
897
|
+
retried = true;
|
|
898
|
+
}
|
|
899
|
+
if (!dragResult.success) {
|
|
900
|
+
throw new Error(dragResult.error || 'drag 执行失败');
|
|
901
|
+
}
|
|
902
|
+
return retried ? 'drag 因 refId 失效已自动重试一次(可能是 React 等框架重渲染导致)' : undefined;
|
|
664
903
|
}
|
|
665
904
|
/**
|
|
666
905
|
* Extension 模式:获取目标点坐标
|
|
@@ -673,6 +912,135 @@ async function executeEventExtension(unifiedSession, event, humanize, timeout) {
|
|
|
673
912
|
* - precise 模式直接使用
|
|
674
913
|
* - stealth 模式需减 offset 转为 iframe 相对坐标
|
|
675
914
|
*/
|
|
915
|
+
async function inputControlled(unifiedSession, event, timeout) {
|
|
916
|
+
if (!event.target) {
|
|
917
|
+
throw new Error('controlled 输入需要 target 参数定位输入元素');
|
|
918
|
+
}
|
|
919
|
+
if ('x' in event.target && 'y' in event.target) {
|
|
920
|
+
throw new Error('controlled 输入不支持坐标型 target,请使用 CSS 选择器、role 或文本定位');
|
|
921
|
+
}
|
|
922
|
+
const { selector, text: searchText, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
923
|
+
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
924
|
+
const nth = nthParam ?? 0;
|
|
925
|
+
if (elements.length === 0 || nth >= elements.length) {
|
|
926
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', `controlled 输入目标未找到: ${JSON.stringify(event.target)}`, '请检查 target 是否能定位到 input、textarea、select 或 contenteditable 元素;context.candidates 提供当前页面候选元素', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout));
|
|
927
|
+
}
|
|
928
|
+
const result = await unifiedSession.dispatchInput(elements[nth].refId, event.text ?? '');
|
|
929
|
+
if (!result.success) {
|
|
930
|
+
throw new StructuredToolError('CONTROLLED_INPUT_FAILED', result.error || 'controlled 输入失败', '请检查目标元素是否可编辑、未 disabled,并确认当前 frame 与 target 匹配', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout, result.error || 'controlled 输入失败'));
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
async function buildTargetDiagnosticContext(unifiedSession, target, matchCount, nth, timeout, reason = '目标元素未找到') {
|
|
934
|
+
let page;
|
|
935
|
+
try {
|
|
936
|
+
page = await unifiedSession.evaluate(`(() => {
|
|
937
|
+
const active = document.activeElement;
|
|
938
|
+
const selection = window.getSelection();
|
|
939
|
+
const candidates = Array.from(document.querySelectorAll('input, textarea, select, [contenteditable="true"], button, a, [role="button"], [role="textbox"], [role="combobox"]')).slice(0, 20);
|
|
940
|
+
return {
|
|
941
|
+
activeElement: active ? {
|
|
942
|
+
tag: active.tagName.toLowerCase(),
|
|
943
|
+
id: active.id || undefined,
|
|
944
|
+
className: typeof active.className === 'string' ? active.className : undefined,
|
|
945
|
+
text: (active.textContent || '').trim().slice(0, 80),
|
|
946
|
+
value: 'value' in active ? active.value : undefined,
|
|
947
|
+
selectionStart: 'selectionStart' in active ? active.selectionStart : undefined,
|
|
948
|
+
selectionEnd: 'selectionEnd' in active ? active.selectionEnd : undefined
|
|
949
|
+
} : null,
|
|
950
|
+
selection: {
|
|
951
|
+
text: selection ? selection.toString().slice(0, 160) : '',
|
|
952
|
+
collapsed: selection ? selection.isCollapsed : true,
|
|
953
|
+
anchorNode: selection && selection.anchorNode ? selection.anchorNode.nodeName : null,
|
|
954
|
+
focusNode: selection && selection.focusNode ? selection.focusNode.nodeName : null
|
|
955
|
+
},
|
|
956
|
+
candidates: candidates.map(function(el) {
|
|
957
|
+
const rect = el.getBoundingClientRect();
|
|
958
|
+
return {
|
|
959
|
+
tag: el.tagName.toLowerCase(),
|
|
960
|
+
id: el.id || undefined,
|
|
961
|
+
role: el.getAttribute('role') || undefined,
|
|
962
|
+
text: (el.textContent || el.getAttribute('aria-label') || el.getAttribute('placeholder') || '').trim().slice(0, 80),
|
|
963
|
+
visible: rect.width > 0 && rect.height > 0,
|
|
964
|
+
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
|
|
965
|
+
bounds: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}
|
|
966
|
+
};
|
|
967
|
+
})
|
|
968
|
+
};
|
|
969
|
+
})()`, undefined, timeout);
|
|
970
|
+
}
|
|
971
|
+
catch (err) {
|
|
972
|
+
page = { diagnosticError: err instanceof Error ? err.message : String(err) };
|
|
973
|
+
}
|
|
974
|
+
return { reason, target, matchCount, nth, page };
|
|
975
|
+
}
|
|
976
|
+
async function editorAction(unifiedSession, action, event, timeout) {
|
|
977
|
+
if (event.target) {
|
|
978
|
+
const focused = await focusTargetForCommands(unifiedSession, event.target, timeout);
|
|
979
|
+
if (!focused) {
|
|
980
|
+
throw new Error('editor 目标未找到或未成功聚焦');
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
return unifiedSession.evaluate(`function(action, text, command) {
|
|
984
|
+
var active = document.activeElement;
|
|
985
|
+
var sel = window.getSelection();
|
|
986
|
+
var editable = active;
|
|
987
|
+
if (!editable || editable === document.body || editable === document.documentElement) {
|
|
988
|
+
var anchor = sel && sel.anchorNode;
|
|
989
|
+
editable = anchor instanceof Element ? anchor : anchor && anchor.parentElement;
|
|
990
|
+
while (editable && !editable.isContentEditable && editable !== document.body) {
|
|
991
|
+
editable = editable.parentElement;
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
var isInput = !!active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA');
|
|
995
|
+
var isEditable = !!editable && editable.isContentEditable;
|
|
996
|
+
if (!isInput && !isEditable) {
|
|
997
|
+
throw new Error('editor 事件需要已聚焦 input、textarea 或 contenteditable 元素');
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
if (action === 'context') {
|
|
1001
|
+
var selectedText = '';
|
|
1002
|
+
if (isInput) {
|
|
1003
|
+
selectedText = active.value.slice(active.selectionStart || 0, active.selectionEnd || 0);
|
|
1004
|
+
} else if (sel) {
|
|
1005
|
+
selectedText = sel.toString();
|
|
1006
|
+
}
|
|
1007
|
+
return {
|
|
1008
|
+
activeElement: active ? {
|
|
1009
|
+
tag: active.tagName.toLowerCase(),
|
|
1010
|
+
id: active.id || undefined,
|
|
1011
|
+
isContentEditable: !!active.isContentEditable,
|
|
1012
|
+
selectionStart: isInput ? active.selectionStart : undefined,
|
|
1013
|
+
selectionEnd: isInput ? active.selectionEnd : undefined
|
|
1014
|
+
} : null,
|
|
1015
|
+
editableElement: editable ? {
|
|
1016
|
+
tag: editable.tagName.toLowerCase(),
|
|
1017
|
+
id: editable.id || undefined,
|
|
1018
|
+
isContentEditable: !!editable.isContentEditable
|
|
1019
|
+
} : null,
|
|
1020
|
+
selectedText: selectedText,
|
|
1021
|
+
selectionCollapsed: isInput ? active.selectionStart === active.selectionEnd : !sel || sel.isCollapsed
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
if (action === 'insert') {
|
|
1026
|
+
if (isInput) {
|
|
1027
|
+
active.setRangeText(text, active.selectionStart || 0, active.selectionEnd || 0, 'end');
|
|
1028
|
+
active.dispatchEvent(new Event('input', {bubbles: true}));
|
|
1029
|
+
active.dispatchEvent(new Event('change', {bubbles: true}));
|
|
1030
|
+
return {success: true};
|
|
1031
|
+
}
|
|
1032
|
+
if (document.execCommand('insertText', false, text)) {
|
|
1033
|
+
return {success: true};
|
|
1034
|
+
}
|
|
1035
|
+
throw new Error('editorInsert 无法插入文本');
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
if (!document.execCommand(command, false, text || null)) {
|
|
1039
|
+
throw new Error('editorCommand 执行失败: ' + command);
|
|
1040
|
+
}
|
|
1041
|
+
return {success: true, command: command};
|
|
1042
|
+
}`, undefined, timeout, [action, event.text ?? '', event.command ?? '']);
|
|
1043
|
+
}
|
|
676
1044
|
async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
677
1045
|
const frameOffset = unifiedSession.getFrameOffset();
|
|
678
1046
|
const isStealth = unifiedSession.getInputMode() === 'stealth';
|
|
@@ -686,12 +1054,12 @@ async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
|
686
1054
|
}
|
|
687
1055
|
const { selector, text, xpath, nth: nthParam } = targetToFindParams(target);
|
|
688
1056
|
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
1057
|
+
const nth = nthParam ?? 0;
|
|
689
1058
|
if (elements.length === 0) {
|
|
690
|
-
throw new
|
|
1059
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', `未找到目标元素: ${JSON.stringify(target)}`, '请检查 target 是否正确,或先用 extract type=state 查看 interactiveElements 候选', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout));
|
|
691
1060
|
}
|
|
692
|
-
const nth = nthParam ?? 0;
|
|
693
1061
|
if (nth >= elements.length) {
|
|
694
|
-
throw new
|
|
1062
|
+
throw new StructuredToolError('TARGET_INDEX_OUT_OF_RANGE', `第 ${nth} 个匹配元素不存在(共 ${elements.length} 个)`, '请降低 nth,或先用 extract type=state 查看当前匹配到的候选元素', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout, '目标元素序号越界'));
|
|
695
1063
|
}
|
|
696
1064
|
// 视口外时滚动后重新取 rect:与 actionableClick (left+single) 行为对齐,
|
|
697
1065
|
// 否则非左键 / 多击的坐标路径在视口外坐标 dispatch,浏览器找不到元素,事件丢失
|
|
@@ -732,142 +1100,125 @@ async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
|
732
1100
|
refId,
|
|
733
1101
|
};
|
|
734
1102
|
}
|
|
735
|
-
// iframe + precise:消费者(chrome.debugger
|
|
1103
|
+
// iframe + precise:消费者(chrome.debugger)需要父视口绝对,
|
|
736
1104
|
// scrollIntoView({block:'center'}) 会 cascade 到父框架,导致 frameOffset 与父绝对 rect 都过期,
|
|
737
1105
|
// refetch find() 让 content-handler 重新计算 frameOffset 并返回最新父绝对 rect
|
|
738
1106
|
const refreshed = await unifiedSession.find(selector, text, xpath, timeout);
|
|
739
1107
|
const rect = refreshed[nth]?.rect ?? elements[nth].rect;
|
|
740
1108
|
return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2, refId };
|
|
741
1109
|
}
|
|
742
|
-
|
|
743
|
-
* CDP 模式:执行单个事件
|
|
744
|
-
*/
|
|
745
|
-
async function executeEvent(session, event, humanize, timeout) {
|
|
1110
|
+
async function executeCdpEvent(session, event, humanize, timeout) {
|
|
746
1111
|
validateEvent(event);
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
}
|
|
816
|
-
}
|
|
817
|
-
else {
|
|
818
|
-
await session.touchMove(point.x, point.y);
|
|
819
|
-
}
|
|
820
|
-
break;
|
|
821
|
-
}
|
|
822
|
-
case 'touchend': {
|
|
823
|
-
await session.touchEnd();
|
|
824
|
-
break;
|
|
1112
|
+
const handler = cdpInputHandlers[event.type];
|
|
1113
|
+
if (!handler) {
|
|
1114
|
+
throw new Error(`未知事件类型: ${event.type}`);
|
|
1115
|
+
}
|
|
1116
|
+
await handler({ session, humanize, timeout }, event);
|
|
1117
|
+
}
|
|
1118
|
+
const cdpInputHandlers = {
|
|
1119
|
+
keydown: handleCdpKeyDown,
|
|
1120
|
+
keyup: handleCdpKeyUp,
|
|
1121
|
+
click: handleCdpClick,
|
|
1122
|
+
mousedown: handleCdpMouseDown,
|
|
1123
|
+
mouseup: handleCdpMouseUp,
|
|
1124
|
+
mousemove: handleCdpMouseMove,
|
|
1125
|
+
wheel: handleCdpWheel,
|
|
1126
|
+
touchstart: handleCdpTouchStart,
|
|
1127
|
+
touchmove: handleCdpTouchMove,
|
|
1128
|
+
touchend: handleCdpTouchEnd,
|
|
1129
|
+
type: handleCdpType,
|
|
1130
|
+
wait: handleCdpWait,
|
|
1131
|
+
drag: handleCdpDrag,
|
|
1132
|
+
};
|
|
1133
|
+
async function handleCdpKeyDown({ session }, event) {
|
|
1134
|
+
await session.keyDown(event.key, event.commands);
|
|
1135
|
+
}
|
|
1136
|
+
async function handleCdpKeyUp({ session }, event) {
|
|
1137
|
+
await session.keyUp(event.key);
|
|
1138
|
+
}
|
|
1139
|
+
async function handleCdpClick({ session, humanize, timeout }, event) {
|
|
1140
|
+
if (event.target) {
|
|
1141
|
+
await moveToTarget(session, event.target, humanize, timeout);
|
|
1142
|
+
}
|
|
1143
|
+
const cdpButton = event.button ?? 'left';
|
|
1144
|
+
const cdpClickCount = event.clickCount ?? 1;
|
|
1145
|
+
for (let i = 1; i <= cdpClickCount; i++) {
|
|
1146
|
+
await session.mouseDown(cdpButton, i);
|
|
1147
|
+
await session.mouseUp(cdpButton, i);
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
async function handleCdpMouseDown({ session, humanize, timeout }, event) {
|
|
1151
|
+
if (event.target) {
|
|
1152
|
+
await moveToTarget(session, event.target, humanize, timeout);
|
|
1153
|
+
}
|
|
1154
|
+
await session.mouseDown(event.button ?? 'left');
|
|
1155
|
+
}
|
|
1156
|
+
async function handleCdpMouseUp({ session }, event) {
|
|
1157
|
+
await session.mouseUp(event.button ?? 'left');
|
|
1158
|
+
}
|
|
1159
|
+
async function handleCdpMouseMove({ session, humanize, timeout }, event) {
|
|
1160
|
+
await moveToTarget(session, event.target, humanize, timeout, event.steps);
|
|
1161
|
+
}
|
|
1162
|
+
async function handleCdpWheel({ session, humanize, timeout }, event) {
|
|
1163
|
+
if (event.target) {
|
|
1164
|
+
await moveToTarget(session, event.target, humanize, timeout);
|
|
1165
|
+
}
|
|
1166
|
+
await session.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
1167
|
+
}
|
|
1168
|
+
async function handleCdpTouchStart({ session, timeout }, event) {
|
|
1169
|
+
const point = await getTargetPoint(session, event.target, timeout);
|
|
1170
|
+
await session.touchStart(point.x, point.y);
|
|
1171
|
+
}
|
|
1172
|
+
async function handleCdpTouchMove({ session, humanize, timeout }, event) {
|
|
1173
|
+
const point = await getTargetPoint(session, event.target, timeout);
|
|
1174
|
+
if (humanize && event.steps && event.steps > 1) {
|
|
1175
|
+
const current = session.getBehaviorSimulator().getCurrentPosition();
|
|
1176
|
+
const path = generateBezierPath(current, point, event.steps);
|
|
1177
|
+
for (const p of path) {
|
|
1178
|
+
await session.touchMove(p.x, p.y);
|
|
1179
|
+
await randomDelay(5, 15);
|
|
825
1180
|
}
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
// 人类化打字
|
|
850
|
-
for (const char of event.text) {
|
|
851
|
-
await session.type(char);
|
|
852
|
-
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
853
|
-
}
|
|
854
|
-
}
|
|
855
|
-
else {
|
|
856
|
-
await session.type(event.text, delay);
|
|
857
|
-
}
|
|
858
|
-
break;
|
|
1181
|
+
}
|
|
1182
|
+
else {
|
|
1183
|
+
await session.touchMove(point.x, point.y);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
async function handleCdpTouchEnd({ session }) {
|
|
1187
|
+
await session.touchEnd();
|
|
1188
|
+
}
|
|
1189
|
+
async function handleCdpType({ session, humanize, timeout }, event) {
|
|
1190
|
+
if (event.dispatch || event.mode === 'controlled') {
|
|
1191
|
+
throw new Error('controlled/dispatch 输入需要 Extension 连接,当前为 CDP 模式');
|
|
1192
|
+
}
|
|
1193
|
+
if (event.target) {
|
|
1194
|
+
await moveToTarget(session, event.target, humanize, timeout, undefined, 'input');
|
|
1195
|
+
await session.mouseDown('left');
|
|
1196
|
+
await session.mouseUp('left');
|
|
1197
|
+
}
|
|
1198
|
+
else {
|
|
1199
|
+
const hasActiveFocus = await session.evaluate('!!document.activeElement && ' +
|
|
1200
|
+
'document.activeElement !== document.body && ' +
|
|
1201
|
+
'document.activeElement !== document.documentElement');
|
|
1202
|
+
if (!hasActiveFocus) {
|
|
1203
|
+
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
859
1204
|
}
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
1205
|
+
}
|
|
1206
|
+
const delay = event.delay ?? 0;
|
|
1207
|
+
if (humanize) {
|
|
1208
|
+
for (const char of event.text) {
|
|
1209
|
+
await session.type(char);
|
|
1210
|
+
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
863
1211
|
}
|
|
864
|
-
default:
|
|
865
|
-
// drag 仅在 Extension 模式可用,给出明确错误而非通用"未知事件类型"
|
|
866
|
-
if (event.type === 'drag') {
|
|
867
|
-
throw new Error('drag 事件仅在 Extension 模式下可用,当前为 CDP 模式');
|
|
868
|
-
}
|
|
869
|
-
throw new Error(`未知事件类型: ${event.type}`);
|
|
870
1212
|
}
|
|
1213
|
+
else {
|
|
1214
|
+
await session.type(event.text, delay);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
async function handleCdpWait(_context, event) {
|
|
1218
|
+
await new Promise((resolve) => setTimeout(resolve, event.ms));
|
|
1219
|
+
}
|
|
1220
|
+
async function handleCdpDrag() {
|
|
1221
|
+
throw new Error('drag 事件仅在 Extension 模式下可用,当前为 CDP 模式');
|
|
871
1222
|
}
|
|
872
1223
|
/**
|
|
873
1224
|
* 移动到目标位置
|
|
@@ -946,6 +1297,12 @@ export function registerInputTool(server) {
|
|
|
946
1297
|
{type: "keydown", key: "a", commands: ["selectAll"]},
|
|
947
1298
|
{type: "keyup", key: "a"}
|
|
948
1299
|
]
|
|
1300
|
+
|
|
1301
|
+
最短示例:
|
|
1302
|
+
click: {events:[{type:"click",target:{css:"button[type=submit]"}}]}
|
|
1303
|
+
type: {events:[{type:"type",target:{css:"input[name=q]"},text:"hello"}]}
|
|
1304
|
+
controlled type: {events:[{type:"type",mode:"controlled",target:{css:"input[name=q]"},text:"hello"}]}
|
|
1305
|
+
replace: {events:[{type:"replace",target:{css:"textarea"},find:"old",text:"new"}]}
|
|
949
1306
|
注意:纯键盘事件(不带 commands)仅保证 JS keyboard event 可被监听,不保证触发浏览器原生编辑行为;
|
|
950
1307
|
全选/复制/粘贴等语义用 commands;"全选并替换文本"用 select/replace 事件更简洁;
|
|
951
1308
|
commands 仅支持 inputMode=precise,stealth 模式下会报错`,
|