@pyrokine/mcp-chrome 2.0.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -35
- package/dist/core/browser-driver.d.ts +118 -1
- package/dist/core/browser-driver.d.ts.map +1 -1
- package/dist/core/browser-driver.js +11 -0
- package/dist/core/browser-driver.js.map +1 -1
- package/dist/core/error-sanitizer.js +2 -2
- package/dist/core/error-sanitizer.js.map +1 -1
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +17 -4
- package/dist/core/errors.js.map +1 -1
- package/dist/core/index.d.ts +1 -1
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/locator.d.ts.map +1 -1
- package/dist/core/locator.js +4 -3
- package/dist/core/locator.js.map +1 -1
- package/dist/core/session.d.ts +5 -2
- package/dist/core/session.d.ts.map +1 -1
- package/dist/core/session.js +107 -4
- package/dist/core/session.js.map +1 -1
- package/dist/core/types.d.ts +17 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +57 -4
- package/dist/core/types.js.map +1 -1
- package/dist/core/unified-session.d.ts +29 -9
- package/dist/core/unified-session.d.ts.map +1 -1
- package/dist/core/unified-session.js +226 -18
- package/dist/core/unified-session.js.map +1 -1
- package/dist/core/utils.d.ts +1 -0
- package/dist/core/utils.d.ts.map +1 -1
- package/dist/core/utils.js +17 -2
- package/dist/core/utils.js.map +1 -1
- package/dist/extension/bridge.d.ts +37 -22
- package/dist/extension/bridge.d.ts.map +1 -1
- package/dist/extension/bridge.js +223 -63
- package/dist/extension/bridge.js.map +1 -1
- package/dist/extension/http-server.d.ts.map +1 -1
- package/dist/extension/http-server.js +39 -10
- package/dist/extension/http-server.js.map +1 -1
- package/dist/tools/browse.d.ts.map +1 -1
- package/dist/tools/browse.js +153 -54
- package/dist/tools/browse.js.map +1 -1
- package/dist/tools/cookies.d.ts.map +1 -1
- package/dist/tools/cookies.js +2 -4
- package/dist/tools/cookies.js.map +1 -1
- package/dist/tools/evaluate.d.ts.map +1 -1
- package/dist/tools/evaluate.js +37 -10
- package/dist/tools/evaluate.js.map +1 -1
- package/dist/tools/extract.d.ts +1 -0
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +466 -227
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/input.d.ts.map +1 -1
- package/dist/tools/input.js +781 -444
- package/dist/tools/input.js.map +1 -1
- package/dist/tools/logs.d.ts.map +1 -1
- package/dist/tools/logs.js +75 -23
- package/dist/tools/logs.js.map +1 -1
- package/dist/tools/manage.d.ts +0 -3
- package/dist/tools/manage.d.ts.map +1 -1
- package/dist/tools/manage.js +339 -285
- package/dist/tools/manage.js.map +1 -1
- package/dist/tools/png.d.ts +16 -0
- package/dist/tools/png.d.ts.map +1 -0
- package/dist/tools/png.js +197 -0
- package/dist/tools/png.js.map +1 -0
- package/dist/tools/schema.d.ts +14 -112
- package/dist/tools/schema.d.ts.map +1 -1
- package/dist/tools/schema.js +21 -12
- package/dist/tools/schema.js.map +1 -1
- package/dist/tools/wait.d.ts.map +1 -1
- package/dist/tools/wait.js +73 -13
- package/dist/tools/wait.js.map +1 -1
- package/package.json +11 -11
package/dist/tools/input.js
CHANGED
|
@@ -33,6 +33,9 @@ const inputEventSchema = z.object({
|
|
|
33
33
|
'select',
|
|
34
34
|
'replace',
|
|
35
35
|
'drag',
|
|
36
|
+
'editorContext',
|
|
37
|
+
'editorInsert',
|
|
38
|
+
'editorCommand',
|
|
36
39
|
])
|
|
37
40
|
.describe('事件类型'),
|
|
38
41
|
key: z.string().optional().describe('按键(keydown/keyup)'),
|
|
@@ -60,6 +63,11 @@ const inputEventSchema = z.object({
|
|
|
60
63
|
ms: z.number().optional().describe('等待毫秒'),
|
|
61
64
|
find: z.string().optional().describe('要查找并选中的文本(select/replace)'),
|
|
62
65
|
nth: z.number().optional().describe('第 N 个匹配(select/replace,从 0 开始,默认 0 即第一个)'),
|
|
66
|
+
command: z.string().optional().describe('浏览器编辑命令(editorCommand),如 bold、italic、insertOrderedList'),
|
|
67
|
+
mode: z
|
|
68
|
+
.enum(['keyboard', 'controlled'])
|
|
69
|
+
.optional()
|
|
70
|
+
.describe('输入模式(type),keyboard=键盘事件,controlled=直接设置 value 并触发 input/change 事件'),
|
|
63
71
|
dispatch: z
|
|
64
72
|
.boolean()
|
|
65
73
|
.optional()
|
|
@@ -68,6 +76,7 @@ const inputEventSchema = z.object({
|
|
|
68
76
|
.boolean()
|
|
69
77
|
.optional()
|
|
70
78
|
.describe('强制执行(click),跳过可操作性检查(可见性、遮挡检测等),直接在目标元素上触发事件,用于已知需要绕过检查的场景'),
|
|
79
|
+
forceReason: z.string().optional().describe('force=true 时必填,说明为什么需要跳过可操作性检查'),
|
|
71
80
|
});
|
|
72
81
|
/**
|
|
73
82
|
* input 参数 schema
|
|
@@ -75,6 +84,7 @@ const inputEventSchema = z.object({
|
|
|
75
84
|
const inputSchema = z.object({
|
|
76
85
|
events: z.array(inputEventSchema).describe('事件序列'),
|
|
77
86
|
humanize: z.boolean().optional().describe('启用人类行为模拟(贝塞尔曲线移动、随机延迟)'),
|
|
87
|
+
diagnostics: z.boolean().optional().describe('执行后返回新增 console error/warning 和失败网络请求摘要'),
|
|
78
88
|
tabId: z
|
|
79
89
|
.string()
|
|
80
90
|
.optional()
|
|
@@ -85,6 +95,28 @@ const inputSchema = z.object({
|
|
|
85
95
|
.optional()
|
|
86
96
|
.describe('iframe 定位(可选,仅 Extension 模式),CSS 选择器(如 "iframe#main")或索引(如 0),不指定则在主框架操作'),
|
|
87
97
|
});
|
|
98
|
+
class StructuredToolError extends Error {
|
|
99
|
+
code;
|
|
100
|
+
suggestion;
|
|
101
|
+
context;
|
|
102
|
+
constructor(code, message, suggestion, context) {
|
|
103
|
+
super(message);
|
|
104
|
+
this.code = code;
|
|
105
|
+
this.suggestion = suggestion;
|
|
106
|
+
this.context = context;
|
|
107
|
+
this.name = 'StructuredToolError';
|
|
108
|
+
}
|
|
109
|
+
toJSON() {
|
|
110
|
+
return {
|
|
111
|
+
error: {
|
|
112
|
+
code: this.code,
|
|
113
|
+
message: this.message,
|
|
114
|
+
suggestion: this.suggestion,
|
|
115
|
+
context: this.context,
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
}
|
|
88
120
|
/**
|
|
89
121
|
* input 工具处理器
|
|
90
122
|
*/
|
|
@@ -95,31 +127,17 @@ async function handleInput(args) {
|
|
|
95
127
|
const humanize = args.humanize ?? false;
|
|
96
128
|
return await unifiedSession.withTabId(args.tabId, async () => {
|
|
97
129
|
return await unifiedSession.withFrame(args.frame, async () => {
|
|
98
|
-
|
|
130
|
+
const diagnosticsStart = args.diagnostics ? await captureDiagnosticsStart(unifiedSession) : undefined;
|
|
99
131
|
const warnings = [];
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
}
|
|
132
|
+
const eventResults = [];
|
|
133
|
+
const session = mode === 'extension' ? undefined : getSession();
|
|
134
|
+
for (const event of args.events) {
|
|
135
|
+
const result = await executeInputEvent({ unifiedSession, session, mode, humanize, timeout: args.timeout }, event);
|
|
136
|
+
if (typeof result === 'string') {
|
|
137
|
+
warnings.push(result);
|
|
107
138
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
// CDP 模式:逐事件分发(无 Extension bridge)
|
|
111
|
-
const session = getSession();
|
|
112
|
-
for (const event of args.events) {
|
|
113
|
-
if (event.type === 'select' || event.type === 'replace') {
|
|
114
|
-
// select/replace 通过 unifiedSession(内部自适应双模式)
|
|
115
|
-
const w = await executeEventExtension(unifiedSession, event, humanize, args.timeout);
|
|
116
|
-
if (w) {
|
|
117
|
-
warnings.push(w);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
else {
|
|
121
|
-
await executeEvent(session, event, humanize, args.timeout);
|
|
122
|
-
}
|
|
139
|
+
else if (result) {
|
|
140
|
+
eventResults.push(result);
|
|
123
141
|
}
|
|
124
142
|
}
|
|
125
143
|
const result = {
|
|
@@ -130,6 +148,12 @@ async function handleInput(args) {
|
|
|
130
148
|
if (warnings.length > 0) {
|
|
131
149
|
result.warnings = warnings;
|
|
132
150
|
}
|
|
151
|
+
if (eventResults.length > 0) {
|
|
152
|
+
result.eventResults = eventResults;
|
|
153
|
+
}
|
|
154
|
+
if (diagnosticsStart) {
|
|
155
|
+
result.diagnostics = await captureDiagnosticsDelta(unifiedSession, diagnosticsStart);
|
|
156
|
+
}
|
|
133
157
|
return formatResponse(result);
|
|
134
158
|
}); // withFrame
|
|
135
159
|
}); // withTabId
|
|
@@ -138,6 +162,44 @@ async function handleInput(args) {
|
|
|
138
162
|
return formatErrorResponse(error);
|
|
139
163
|
}
|
|
140
164
|
}
|
|
165
|
+
const unifiedOnlyEventTypes = new Set([
|
|
166
|
+
'select',
|
|
167
|
+
'replace',
|
|
168
|
+
'editorContext',
|
|
169
|
+
'editorInsert',
|
|
170
|
+
'editorCommand',
|
|
171
|
+
]);
|
|
172
|
+
async function executeInputEvent(context, event) {
|
|
173
|
+
if (context.mode === 'extension' || unifiedOnlyEventTypes.has(event.type)) {
|
|
174
|
+
return executeUnifiedEvent(context.unifiedSession, event, context.humanize, context.timeout);
|
|
175
|
+
}
|
|
176
|
+
if (!context.session) {
|
|
177
|
+
throw new Error('CDP 输入事件缺少 session');
|
|
178
|
+
}
|
|
179
|
+
await executeCdpEvent(context.session, event, context.humanize, context.timeout);
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
async function captureDiagnosticsStart(unifiedSession) {
|
|
183
|
+
await unifiedSession.enableConsole();
|
|
184
|
+
await unifiedSession.enableNetwork();
|
|
185
|
+
const consoleLogs = await unifiedSession.getConsoleLogs();
|
|
186
|
+
const network = await unifiedSession.getNetworkRequests();
|
|
187
|
+
return { consoleCount: consoleLogs.length, networkCount: network.length };
|
|
188
|
+
}
|
|
189
|
+
async function captureDiagnosticsDelta(unifiedSession, start) {
|
|
190
|
+
const consoleLogs = await unifiedSession.getConsoleLogs();
|
|
191
|
+
const network = await unifiedSession.getNetworkRequests();
|
|
192
|
+
return {
|
|
193
|
+
console: consoleLogs
|
|
194
|
+
.slice(start.consoleCount)
|
|
195
|
+
.filter((item) => ['error', 'warning', 'warn'].includes(item.level))
|
|
196
|
+
.slice(-20),
|
|
197
|
+
failedRequests: network
|
|
198
|
+
.slice(start.networkCount)
|
|
199
|
+
.filter((item) => item.errorText || (item.status !== undefined && item.status >= 400))
|
|
200
|
+
.slice(-20),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
141
203
|
/**
|
|
142
204
|
* 通过真实鼠标事件选中页面文本
|
|
143
205
|
*
|
|
@@ -153,15 +215,15 @@ async function handleInput(args) {
|
|
|
153
215
|
*/
|
|
154
216
|
async function focusTargetForCommands(unifiedSession, target, timeout) {
|
|
155
217
|
if ('x' in target || 'y' in target) {
|
|
156
|
-
return;
|
|
218
|
+
return true;
|
|
157
219
|
}
|
|
158
220
|
const params = targetToFindParams(target);
|
|
159
221
|
const els = await unifiedSession.find(params.selector, params.text, params.xpath, timeout);
|
|
160
222
|
const nth0 = params.nth ?? 0;
|
|
161
223
|
if (els.length <= nth0) {
|
|
162
|
-
return;
|
|
224
|
+
return false;
|
|
163
225
|
}
|
|
164
|
-
await unifiedSession.evaluate(`(() => {
|
|
226
|
+
const focused = await unifiedSession.evaluate(`(() => {
|
|
165
227
|
const ref = window.__mcpElementMap?.[${JSON.stringify(els[nth0].refId)}]
|
|
166
228
|
const el = ref?.deref()
|
|
167
229
|
if (!(el instanceof HTMLElement)) {
|
|
@@ -170,6 +232,7 @@ async function focusTargetForCommands(unifiedSession, target, timeout) {
|
|
|
170
232
|
el.focus()
|
|
171
233
|
return document.activeElement === el
|
|
172
234
|
})()`, undefined, timeout);
|
|
235
|
+
return focused === true;
|
|
173
236
|
}
|
|
174
237
|
async function focusTargetIfNeeded(unifiedSession, target, nth, timeout) {
|
|
175
238
|
if (!target || 'x' in target || 'y' in target) {
|
|
@@ -188,6 +251,98 @@ async function focusTargetIfNeeded(unifiedSession, target, nth, timeout) {
|
|
|
188
251
|
}
|
|
189
252
|
}
|
|
190
253
|
}
|
|
254
|
+
async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout) {
|
|
255
|
+
return await unifiedSession.evaluate(`function(findText, nth, scopeTarget, scopeSelector, scopeText, scopeXpath) {
|
|
256
|
+
function selectorFor(el) {
|
|
257
|
+
if (!(el instanceof Element)) return null;
|
|
258
|
+
if (el.id) return '#' + CSS.escape(el.id);
|
|
259
|
+
var parts = [];
|
|
260
|
+
var current = el;
|
|
261
|
+
while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 4) {
|
|
262
|
+
var part = current.tagName.toLowerCase();
|
|
263
|
+
if (current.classList && current.classList.length) {
|
|
264
|
+
part += '.' + Array.from(current.classList).slice(0, 2).map(function(cls) { return CSS.escape(cls); }).join('.');
|
|
265
|
+
}
|
|
266
|
+
var parent = current.parentElement;
|
|
267
|
+
if (parent) {
|
|
268
|
+
var siblings = Array.from(parent.children).filter(function(child) { return child.tagName === current.tagName; });
|
|
269
|
+
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
|
270
|
+
}
|
|
271
|
+
parts.unshift(part);
|
|
272
|
+
current = parent;
|
|
273
|
+
}
|
|
274
|
+
return parts.join(' > ');
|
|
275
|
+
}
|
|
276
|
+
function summarizeElement(el) {
|
|
277
|
+
if (!(el instanceof Element)) return null;
|
|
278
|
+
var rect = el.getBoundingClientRect();
|
|
279
|
+
var value = 'value' in el ? String(el.value || '') : '';
|
|
280
|
+
var text = (el.innerText || el.textContent || value || '').replace(new RegExp('\\\\s+', 'g'), ' ').trim();
|
|
281
|
+
return {
|
|
282
|
+
tag: el.tagName.toLowerCase(),
|
|
283
|
+
id: el.id || undefined,
|
|
284
|
+
selector: selectorFor(el),
|
|
285
|
+
text: text.slice(0, 160),
|
|
286
|
+
valuePreview: value ? value.slice(0, 160) : undefined,
|
|
287
|
+
visible: rect.width > 0 && rect.height > 0,
|
|
288
|
+
disabled: Boolean(el.disabled),
|
|
289
|
+
readOnly: Boolean(el.readOnly),
|
|
290
|
+
contentEditable: Boolean(el.isContentEditable),
|
|
291
|
+
bounds: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
var root = document.body;
|
|
295
|
+
if (scopeXpath) {
|
|
296
|
+
var xr = document.evaluate(scopeXpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
297
|
+
root = xr.singleNodeValue;
|
|
298
|
+
} else if (scopeSelector) {
|
|
299
|
+
var candidates = document.querySelectorAll(scopeSelector);
|
|
300
|
+
if (scopeText) {
|
|
301
|
+
for (var ci = 0; ci < candidates.length; ci++) {
|
|
302
|
+
if ((candidates[ci].textContent || '').includes(scopeText)) { root = candidates[ci]; break; }
|
|
303
|
+
}
|
|
304
|
+
} else {
|
|
305
|
+
root = candidates[0];
|
|
306
|
+
}
|
|
307
|
+
} else if (scopeText) {
|
|
308
|
+
var all = document.querySelectorAll('*');
|
|
309
|
+
for (var ai = 0; ai < all.length; ai++) {
|
|
310
|
+
if ((all[ai].textContent || '').includes(scopeText)) { root = all[ai]; break; }
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
var candidateElements = [];
|
|
314
|
+
if (root instanceof Element) {
|
|
315
|
+
candidateElements.push(root);
|
|
316
|
+
candidateElements = candidateElements.concat(Array.from(root.querySelectorAll('input, textarea, [contenteditable="true"], [role="textbox"], button, a')).slice(0, 12));
|
|
317
|
+
}
|
|
318
|
+
var active = document.activeElement;
|
|
319
|
+
var selection = window.getSelection();
|
|
320
|
+
return {
|
|
321
|
+
scope: {
|
|
322
|
+
target: scopeTarget,
|
|
323
|
+
nth: nth,
|
|
324
|
+
findText: findText,
|
|
325
|
+
selector: scopeSelector,
|
|
326
|
+
text: scopeText,
|
|
327
|
+
xpath: scopeXpath,
|
|
328
|
+
rootFound: Boolean(root)
|
|
329
|
+
},
|
|
330
|
+
activeElement: summarizeElement(active),
|
|
331
|
+
selection: {
|
|
332
|
+
text: selection ? selection.toString().slice(0, 160) : '',
|
|
333
|
+
collapsed: selection ? selection.isCollapsed : true,
|
|
334
|
+
anchorNode: selection && selection.anchorNode ? selection.anchorNode.nodeName : null,
|
|
335
|
+
focusNode: selection && selection.focusNode ? selection.focusNode.nodeName : null,
|
|
336
|
+
inputSelection: active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA') ? {
|
|
337
|
+
selectionStart: active.selectionStart,
|
|
338
|
+
selectionEnd: active.selectionEnd,
|
|
339
|
+
valuePreview: String(active.value || '').slice(0, 160)
|
|
340
|
+
} : undefined
|
|
341
|
+
},
|
|
342
|
+
candidates: candidateElements.map(summarizeElement).filter(Boolean)
|
|
343
|
+
};
|
|
344
|
+
}`, undefined, timeout, [findText, nth, scopeTarget ?? null, scopeSelector, scopeText, scopeXpath]);
|
|
345
|
+
}
|
|
191
346
|
async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeout) {
|
|
192
347
|
// 将 target 转为查询参数,传入注入脚本进行 DOM 查询
|
|
193
348
|
let scopeSelector = null;
|
|
@@ -321,12 +476,14 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
321
476
|
};
|
|
322
477
|
}`, undefined, timeout, [findText, nth, scopeSelector, scopeText, scopeXpath]);
|
|
323
478
|
if (!result || result.type === 'noscope') {
|
|
324
|
-
|
|
479
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
480
|
+
throw new StructuredToolError('TEXT_SCOPE_NOT_FOUND', `未找到目标元素: ${JSON.stringify(scopeTarget)}`, '请检查 target 是否能定位到包含目标文本的元素,或先用 extract(state) 查看可交互元素', diagnostics);
|
|
325
481
|
}
|
|
326
482
|
if (result.type === 'notfound') {
|
|
327
|
-
|
|
483
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
484
|
+
throw new StructuredToolError('TEXT_NOT_FOUND', scopeTarget
|
|
328
485
|
? `目标元素内未找到文本 "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`
|
|
329
|
-
: `未找到文本: "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}
|
|
486
|
+
: `未找到文本: "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`, '请检查 find 文本、nth 序号和 target 范围;context.candidates 提供了当前范围内的候选文本和 selector', diagnostics);
|
|
330
487
|
}
|
|
331
488
|
if (result.type === 'input') {
|
|
332
489
|
// 注入脚本已完成 focus + setSelectionRange(原子化,避免外层 mouseClick 聚焦不可靠)
|
|
@@ -351,336 +508,398 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
351
508
|
await unifiedSession.keyUp('Shift');
|
|
352
509
|
return coords.formatted;
|
|
353
510
|
}
|
|
511
|
+
async function executeEditingCommands(unifiedSession, commands, timeout) {
|
|
512
|
+
const result = await unifiedSession.evaluate(`function(commands) {
|
|
513
|
+
function summarizeSelection() {
|
|
514
|
+
var active = document.activeElement;
|
|
515
|
+
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA')) {
|
|
516
|
+
return {
|
|
517
|
+
type: active.tagName.toLowerCase(),
|
|
518
|
+
selectionStart: active.selectionStart,
|
|
519
|
+
selectionEnd: active.selectionEnd,
|
|
520
|
+
selectedText: String(active.value || '').slice(active.selectionStart || 0, active.selectionEnd || 0)
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
var selection = window.getSelection();
|
|
524
|
+
return {
|
|
525
|
+
type: 'document',
|
|
526
|
+
text: selection ? selection.toString() : '',
|
|
527
|
+
collapsed: selection ? selection.isCollapsed : true
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
function commandSelectAll() {
|
|
531
|
+
var active = document.activeElement;
|
|
532
|
+
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA') && typeof active.select === 'function') {
|
|
533
|
+
active.select();
|
|
534
|
+
active.dispatchEvent(new Event('select', {bubbles: true}));
|
|
535
|
+
return true;
|
|
536
|
+
}
|
|
537
|
+
if (active && active.isContentEditable) {
|
|
538
|
+
var range = document.createRange();
|
|
539
|
+
range.selectNodeContents(active);
|
|
540
|
+
var selection = window.getSelection();
|
|
541
|
+
if (!selection) return false;
|
|
542
|
+
selection.removeAllRanges();
|
|
543
|
+
selection.addRange(range);
|
|
544
|
+
return true;
|
|
545
|
+
}
|
|
546
|
+
return document.execCommand('selectAll');
|
|
547
|
+
}
|
|
548
|
+
var executed = commands.map(function(command) {
|
|
549
|
+
if (command === 'selectAll') {
|
|
550
|
+
return {command: command, success: commandSelectAll()};
|
|
551
|
+
}
|
|
552
|
+
if (['copy', 'cut', 'paste', 'undo', 'redo'].indexOf(command) !== -1) {
|
|
553
|
+
return {command: command, success: document.execCommand(command)};
|
|
554
|
+
}
|
|
555
|
+
return {command: command, success: false};
|
|
556
|
+
});
|
|
557
|
+
return {success: executed.every(function(item) { return item.success; }), executed: executed, selection: summarizeSelection()};
|
|
558
|
+
}`, undefined, timeout, [commands]);
|
|
559
|
+
if (!result.success) {
|
|
560
|
+
throw new StructuredToolError('EDIT_COMMAND_FAILED', `编辑命令执行失败: ${commands.join(', ')}`, '请确认当前页面已有可编辑焦点,或给 keydown 事件提供 target 参数', result);
|
|
561
|
+
}
|
|
562
|
+
return result;
|
|
563
|
+
}
|
|
354
564
|
/**
|
|
355
565
|
* 验证事件参数(两种执行模式共享),避免在 Extension/CDP 两个 switch 中重复校验
|
|
356
566
|
*/
|
|
567
|
+
function requiredEventParam(event, name) {
|
|
568
|
+
throw new Error(`events[].${String(name)} 是 ${event.type} 事件的必填参数`);
|
|
569
|
+
}
|
|
357
570
|
function validateEvent(event) {
|
|
358
571
|
if (event.commands && event.commands.length > 0 && event.type !== 'keydown') {
|
|
359
|
-
throw new Error(`commands
|
|
572
|
+
throw new Error(`events[].commands 只能用于 keydown 事件,当前事件类型为 ${event.type},如需触发编辑命令,请把 commands 放在 keydown 事件上`);
|
|
573
|
+
}
|
|
574
|
+
if (event.force && !event.forceReason) {
|
|
575
|
+
throw new Error('events[].forceReason 在 force=true 时必填');
|
|
360
576
|
}
|
|
361
577
|
switch (event.type) {
|
|
362
578
|
case 'keydown':
|
|
363
579
|
case 'keyup':
|
|
364
580
|
if (!event.key) {
|
|
365
|
-
|
|
581
|
+
requiredEventParam(event, 'key');
|
|
366
582
|
}
|
|
367
583
|
break;
|
|
368
|
-
case '
|
|
369
|
-
|
|
370
|
-
|
|
584
|
+
case 'mousemove':
|
|
585
|
+
case 'touchstart':
|
|
586
|
+
case 'touchmove':
|
|
587
|
+
if (!event.target) {
|
|
588
|
+
requiredEventParam(event, 'target');
|
|
371
589
|
}
|
|
372
590
|
break;
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
* Extension 模式:执行单个事件
|
|
377
|
-
*
|
|
378
|
-
* @returns 可选警告信息(如格式丢失提示)
|
|
379
|
-
*/
|
|
380
|
-
async function executeEventExtension(unifiedSession, event, humanize, timeout) {
|
|
381
|
-
validateEvent(event);
|
|
382
|
-
switch (event.type) {
|
|
383
|
-
case 'keydown': {
|
|
384
|
-
if (unifiedSession.getInputMode() === 'stealth' && event.commands && event.commands.length > 0) {
|
|
385
|
-
throw new Error('commands 参数不支持 stealth 输入模式,请先调用 manage action=inputMode inputMode=precise 切换后重试');
|
|
386
|
-
}
|
|
387
|
-
if (event.commands && event.commands.length > 0 && event.target) {
|
|
388
|
-
await focusTargetForCommands(unifiedSession, event.target, timeout);
|
|
591
|
+
case 'type':
|
|
592
|
+
if (!event.text) {
|
|
593
|
+
requiredEventParam(event, 'text');
|
|
389
594
|
}
|
|
390
|
-
await unifiedSession.keyDown(event.key, event.commands);
|
|
391
|
-
break;
|
|
392
|
-
}
|
|
393
|
-
case 'keyup': {
|
|
394
|
-
await unifiedSession.keyUp(event.key);
|
|
395
595
|
break;
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
const clickCount = event.clickCount ?? 1;
|
|
400
|
-
if (event.target) {
|
|
401
|
-
// 坐标型 target:不过 actionableClick,但仍需 getTargetPointExtension 做 iframe offset 修正
|
|
402
|
-
if ('x' in event.target && 'y' in event.target) {
|
|
403
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
404
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
405
|
-
await unifiedSession.mouseClick(button, clickCount);
|
|
406
|
-
break;
|
|
407
|
-
}
|
|
408
|
-
// 左键单击:优先用 actionable click(带可操作性检查、自动滚动、遮挡检测)
|
|
409
|
-
// 非左键 / 多击:actionableClick 依赖 HTMLElement.click() 只能触发单次左键,必须走坐标路径
|
|
410
|
-
if (button === 'left' && clickCount === 1) {
|
|
411
|
-
const { selector, text: searchText, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
412
|
-
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
413
|
-
const nth = nthParam ?? 0;
|
|
414
|
-
if (elements.length > 0 && nth < elements.length) {
|
|
415
|
-
const refId = elements[nth].refId;
|
|
416
|
-
const result = await unifiedSession.actionableClick(refId, event.force);
|
|
417
|
-
if (!result.success) {
|
|
418
|
-
throw new Error(result.error || 'Click failed');
|
|
419
|
-
}
|
|
420
|
-
break;
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
// fallback: 找不到 refId 或需走坐标路径时
|
|
424
|
-
// refId 透传:stealth 模式下嵌套 iframe overlay 场景绕过 elementFromPoint
|
|
425
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
426
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
427
|
-
await unifiedSession.mouseClick(button, clickCount, typeof point.refId === 'string' ? point.refId : undefined);
|
|
428
|
-
break;
|
|
596
|
+
case 'select':
|
|
597
|
+
if (!event.find) {
|
|
598
|
+
requiredEventParam(event, 'find');
|
|
429
599
|
}
|
|
430
|
-
await unifiedSession.mouseClick(button, clickCount);
|
|
431
600
|
break;
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
601
|
+
case 'replace':
|
|
602
|
+
if (!event.find) {
|
|
603
|
+
requiredEventParam(event, 'find');
|
|
604
|
+
}
|
|
605
|
+
if (event.text === undefined) {
|
|
606
|
+
requiredEventParam(event, 'text');
|
|
437
607
|
}
|
|
438
|
-
await unifiedSession.mouseDown(event.button ?? 'left');
|
|
439
|
-
break;
|
|
440
|
-
}
|
|
441
|
-
case 'mouseup': {
|
|
442
|
-
await unifiedSession.mouseUp(event.button ?? 'left');
|
|
443
608
|
break;
|
|
444
|
-
|
|
445
|
-
case 'mousemove': {
|
|
609
|
+
case 'drag':
|
|
446
610
|
if (!event.target) {
|
|
447
|
-
|
|
611
|
+
requiredEventParam(event, 'target');
|
|
448
612
|
}
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
const path = generateBezierPath(unifiedSession.getMousePosition(), point, event.steps);
|
|
452
|
-
for (const p of path) {
|
|
453
|
-
await unifiedSession.mouseMove(p.x, p.y);
|
|
454
|
-
await randomDelay(getMouseMoveDelay(), getMouseMoveDelay() * 2);
|
|
455
|
-
}
|
|
613
|
+
if (!event.to) {
|
|
614
|
+
requiredEventParam(event, 'to');
|
|
456
615
|
}
|
|
457
|
-
|
|
458
|
-
|
|
616
|
+
break;
|
|
617
|
+
case 'editorInsert':
|
|
618
|
+
if (!event.text) {
|
|
619
|
+
requiredEventParam(event, 'text');
|
|
459
620
|
}
|
|
460
621
|
break;
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
const { selector, text, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
465
|
-
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
466
|
-
const nth = nthParam ?? 0;
|
|
467
|
-
if (elements.length > nth) {
|
|
468
|
-
// 用 refId 直接滚动目标元素(支持视口外元素)
|
|
469
|
-
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[nth].refId);
|
|
470
|
-
break;
|
|
471
|
-
}
|
|
472
|
-
// 找不到元素时 fallback 到坐标方式
|
|
473
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
474
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
622
|
+
case 'editorCommand':
|
|
623
|
+
if (!event.command) {
|
|
624
|
+
requiredEventParam(event, 'command');
|
|
475
625
|
}
|
|
476
|
-
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
477
626
|
break;
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
throw new Error('touchstart 事件需要 target 参数');
|
|
627
|
+
case 'wait':
|
|
628
|
+
if (event.ms === undefined) {
|
|
629
|
+
requiredEventParam(event, 'ms');
|
|
482
630
|
}
|
|
483
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
484
|
-
await unifiedSession.touchStart(point.x, point.y);
|
|
485
631
|
break;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
async function executeUnifiedEvent(unifiedSession, event, humanize, timeout) {
|
|
635
|
+
validateEvent(event);
|
|
636
|
+
const handler = unifiedInputHandlers[event.type];
|
|
637
|
+
if (!handler) {
|
|
638
|
+
throw new Error(`未知事件类型: ${event.type}`);
|
|
639
|
+
}
|
|
640
|
+
return handler({ unifiedSession, humanize, timeout }, event);
|
|
641
|
+
}
|
|
642
|
+
const unifiedInputHandlers = {
|
|
643
|
+
keydown: handleUnifiedKeyDown,
|
|
644
|
+
keyup: handleUnifiedKeyUp,
|
|
645
|
+
click: handleUnifiedClick,
|
|
646
|
+
mousedown: handleUnifiedMouseDown,
|
|
647
|
+
mouseup: handleUnifiedMouseUp,
|
|
648
|
+
mousemove: handleUnifiedMouseMove,
|
|
649
|
+
wheel: handleUnifiedWheel,
|
|
650
|
+
touchstart: handleUnifiedTouchStart,
|
|
651
|
+
touchmove: handleUnifiedTouchMove,
|
|
652
|
+
touchend: handleUnifiedTouchEnd,
|
|
653
|
+
type: handleUnifiedType,
|
|
654
|
+
wait: handleUnifiedWait,
|
|
655
|
+
select: handleUnifiedSelect,
|
|
656
|
+
replace: handleUnifiedReplace,
|
|
657
|
+
editorContext: handleUnifiedEditorContext,
|
|
658
|
+
editorInsert: handleUnifiedEditorInsert,
|
|
659
|
+
editorCommand: handleUnifiedEditorCommand,
|
|
660
|
+
drag: handleUnifiedDrag,
|
|
661
|
+
};
|
|
662
|
+
async function handleUnifiedKeyDown({ unifiedSession, timeout }, event) {
|
|
663
|
+
if (unifiedSession.getInputMode() === 'stealth' && event.commands && event.commands.length > 0) {
|
|
664
|
+
throw new Error('commands 参数不支持 stealth 输入模式,请先调用 manage action=inputMode inputMode=precise 切换后重试');
|
|
665
|
+
}
|
|
666
|
+
if (event.commands && event.commands.length > 0 && event.target) {
|
|
667
|
+
const focused = await focusTargetForCommands(unifiedSession, event.target, timeout);
|
|
668
|
+
if (!focused) {
|
|
669
|
+
throw new Error('commands 目标未找到或未成功聚焦');
|
|
486
670
|
}
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
671
|
+
}
|
|
672
|
+
await unifiedSession.keyDown(event.key, event.commands);
|
|
673
|
+
if (event.commands && event.commands.length > 0) {
|
|
674
|
+
return await executeEditingCommands(unifiedSession, event.commands, timeout);
|
|
675
|
+
}
|
|
676
|
+
return undefined;
|
|
677
|
+
}
|
|
678
|
+
async function handleUnifiedKeyUp({ unifiedSession }, event) {
|
|
679
|
+
await unifiedSession.keyUp(event.key);
|
|
680
|
+
return undefined;
|
|
681
|
+
}
|
|
682
|
+
async function handleUnifiedClick({ unifiedSession, timeout }, event) {
|
|
683
|
+
const button = event.button ?? 'left';
|
|
684
|
+
const clickCount = event.clickCount ?? 1;
|
|
685
|
+
if (!event.target) {
|
|
686
|
+
await unifiedSession.mouseClick(button, clickCount);
|
|
687
|
+
return undefined;
|
|
688
|
+
}
|
|
689
|
+
if ('x' in event.target && 'y' in event.target) {
|
|
690
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
691
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
692
|
+
await unifiedSession.mouseClick(button, clickCount);
|
|
693
|
+
return undefined;
|
|
694
|
+
}
|
|
695
|
+
if (button === 'left' && clickCount === 1) {
|
|
696
|
+
const { selector, text: searchText, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
697
|
+
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
698
|
+
const nth = nthParam ?? 0;
|
|
699
|
+
if (elements.length > 0 && nth < elements.length) {
|
|
700
|
+
const result = await unifiedSession.actionableClick(elements[nth].refId, event.force === true);
|
|
701
|
+
if (!result.success) {
|
|
702
|
+
throw new StructuredToolError('ACTIONABILITY_FAILED', result.error || 'Click failed', result.suggestions?.[0] ??
|
|
703
|
+
'请根据 context.rect、context.clickPoint 和 context.coveringElement 调整 target', result);
|
|
490
704
|
}
|
|
491
|
-
|
|
492
|
-
await unifiedSession.touchMove(point.x, point.y);
|
|
493
|
-
break;
|
|
705
|
+
return undefined;
|
|
494
706
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
707
|
+
}
|
|
708
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
709
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
710
|
+
await unifiedSession.mouseClick(button, clickCount, typeof point.refId === 'string' ? point.refId : undefined);
|
|
711
|
+
return undefined;
|
|
712
|
+
}
|
|
713
|
+
async function handleUnifiedMouseDown({ unifiedSession, timeout }, event) {
|
|
714
|
+
if (event.target) {
|
|
715
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
716
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
717
|
+
}
|
|
718
|
+
await unifiedSession.mouseDown(event.button ?? 'left');
|
|
719
|
+
return undefined;
|
|
720
|
+
}
|
|
721
|
+
async function handleUnifiedMouseUp({ unifiedSession }, event) {
|
|
722
|
+
await unifiedSession.mouseUp(event.button ?? 'left');
|
|
723
|
+
return undefined;
|
|
724
|
+
}
|
|
725
|
+
async function handleUnifiedMouseMove({ unifiedSession, humanize, timeout }, event) {
|
|
726
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
727
|
+
if (humanize && event.steps && event.steps > 1) {
|
|
728
|
+
const path = generateBezierPath(unifiedSession.getMousePosition(), point, event.steps);
|
|
729
|
+
for (const p of path) {
|
|
730
|
+
await unifiedSession.mouseMove(p.x, p.y);
|
|
731
|
+
await randomDelay(getMouseMoveDelay(), getMouseMoveDelay() * 2);
|
|
498
732
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
514
|
-
const nth = nthParam ?? 0;
|
|
515
|
-
if (elements.length === 0 || nth >= elements.length) {
|
|
516
|
-
throw new Error('目标元素未找到');
|
|
517
|
-
}
|
|
518
|
-
const refId = elements[nth].refId;
|
|
519
|
-
// 通过 Extension ISOLATED 世界执行 dispatch(访问 __mcpElementMap)
|
|
520
|
-
// 参考 Playwright fill():nativeInputValueSetter + dispatchEvent
|
|
521
|
-
const result = await unifiedSession.dispatchInput(refId, event.text);
|
|
522
|
-
if (!result.success) {
|
|
523
|
-
throw new Error(result.error || 'dispatch 输入失败');
|
|
524
|
-
}
|
|
525
|
-
break;
|
|
526
|
-
}
|
|
527
|
-
// 默认模式:键盘事件
|
|
528
|
-
// 如果有 target,先点击目标(聚焦)
|
|
529
|
-
if (event.target) {
|
|
530
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
531
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
532
|
-
await unifiedSession.mouseClick('left');
|
|
533
|
-
}
|
|
534
|
-
else {
|
|
535
|
-
const hasActiveFocus = await unifiedSession.evaluate('!!document.activeElement && ' +
|
|
536
|
-
'document.activeElement !== document.body && ' +
|
|
537
|
-
'document.activeElement !== document.documentElement');
|
|
538
|
-
if (!hasActiveFocus) {
|
|
539
|
-
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
const delay = event.delay ?? 0;
|
|
543
|
-
if (humanize) {
|
|
544
|
-
for (const char of event.text) {
|
|
545
|
-
await unifiedSession.typeText(char);
|
|
546
|
-
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
else {
|
|
550
|
-
await unifiedSession.typeText(event.text, delay);
|
|
551
|
-
}
|
|
552
|
-
break;
|
|
733
|
+
}
|
|
734
|
+
else {
|
|
735
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
736
|
+
}
|
|
737
|
+
return undefined;
|
|
738
|
+
}
|
|
739
|
+
async function handleUnifiedWheel({ unifiedSession, timeout }, event) {
|
|
740
|
+
if (event.target) {
|
|
741
|
+
const { selector, text, xpath, nth: nthParam } = targetToFindParams(event.target);
|
|
742
|
+
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
743
|
+
const nth = nthParam ?? 0;
|
|
744
|
+
if (elements.length > nth) {
|
|
745
|
+
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[nth].refId);
|
|
746
|
+
return undefined;
|
|
553
747
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
748
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
749
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
750
|
+
}
|
|
751
|
+
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
752
|
+
return undefined;
|
|
753
|
+
}
|
|
754
|
+
async function handleUnifiedTouchStart({ unifiedSession, timeout }, event) {
|
|
755
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
756
|
+
await unifiedSession.touchStart(point.x, point.y);
|
|
757
|
+
return undefined;
|
|
758
|
+
}
|
|
759
|
+
async function handleUnifiedTouchMove({ unifiedSession, timeout }, event) {
|
|
760
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
761
|
+
await unifiedSession.touchMove(point.x, point.y);
|
|
762
|
+
return undefined;
|
|
763
|
+
}
|
|
764
|
+
async function handleUnifiedTouchEnd({ unifiedSession }) {
|
|
765
|
+
await unifiedSession.touchEnd();
|
|
766
|
+
return undefined;
|
|
767
|
+
}
|
|
768
|
+
async function handleUnifiedType({ unifiedSession, humanize, timeout }, event) {
|
|
769
|
+
if (event.mode === 'controlled' || event.dispatch) {
|
|
770
|
+
await inputControlled(unifiedSession, event, timeout);
|
|
771
|
+
return undefined;
|
|
772
|
+
}
|
|
773
|
+
if (event.target) {
|
|
774
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
775
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
776
|
+
await unifiedSession.mouseClick('left');
|
|
777
|
+
}
|
|
778
|
+
else {
|
|
779
|
+
const hasActiveFocus = await unifiedSession.evaluate('!!document.activeElement && ' +
|
|
780
|
+
'document.activeElement !== document.body && ' +
|
|
781
|
+
'document.activeElement !== document.documentElement');
|
|
782
|
+
if (!hasActiveFocus) {
|
|
783
|
+
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
557
784
|
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
await
|
|
564
|
-
await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
565
|
-
break;
|
|
785
|
+
}
|
|
786
|
+
const delay = event.delay ?? 0;
|
|
787
|
+
if (humanize) {
|
|
788
|
+
for (const char of event.text) {
|
|
789
|
+
await unifiedSession.typeText(char);
|
|
790
|
+
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
566
791
|
}
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
})()`);
|
|
589
|
-
if (hasSelection) {
|
|
590
|
-
selectionConfirmed = true;
|
|
591
|
-
break;
|
|
592
|
-
}
|
|
593
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
594
|
-
}
|
|
595
|
-
if (!selectionConfirmed) {
|
|
596
|
-
throw new Error(`选区同步失败:文本 "${event.find}" 已定位但未能建立选区,无法执行替换`);
|
|
597
|
-
}
|
|
598
|
-
// Step 2: 检测可编辑性并替换
|
|
599
|
-
const replaceResult = await unifiedSession.evaluate(`function(replacementText) {
|
|
792
|
+
}
|
|
793
|
+
else {
|
|
794
|
+
await unifiedSession.typeText(event.text, delay);
|
|
795
|
+
}
|
|
796
|
+
return undefined;
|
|
797
|
+
}
|
|
798
|
+
async function handleUnifiedWait(_context, event) {
|
|
799
|
+
await new Promise((resolve) => setTimeout(resolve, event.ms));
|
|
800
|
+
return undefined;
|
|
801
|
+
}
|
|
802
|
+
async function handleUnifiedSelect({ unifiedSession, timeout }, event) {
|
|
803
|
+
await focusTargetIfNeeded(unifiedSession, event.target, event.nth, timeout);
|
|
804
|
+
await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
805
|
+
return undefined;
|
|
806
|
+
}
|
|
807
|
+
async function handleUnifiedReplace({ unifiedSession, timeout }, event) {
|
|
808
|
+
await focusTargetIfNeeded(unifiedSession, event.target, event.nth, timeout);
|
|
809
|
+
const formatted = await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
810
|
+
let selectionConfirmed = false;
|
|
811
|
+
for (let i = 0; i < 25; i++) {
|
|
812
|
+
const hasSelection = await unifiedSession.evaluate(`(function() {
|
|
600
813
|
var el = document.activeElement;
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
if (el.readOnly || el.disabled) return 'readonly';
|
|
604
|
-
el.setRangeText(replacementText, el.selectionStart, el.selectionEnd, 'end');
|
|
605
|
-
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
606
|
-
return 'ok';
|
|
814
|
+
if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA')) {
|
|
815
|
+
return el.selectionStart !== el.selectionEnd;
|
|
607
816
|
}
|
|
608
|
-
// 检测 contenteditable
|
|
609
817
|
var sel = window.getSelection();
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
editable = editable.parentElement;
|
|
615
|
-
}
|
|
616
|
-
if (!editable || !editable.isContentEditable) return 'readonly';
|
|
617
|
-
// contenteditable:execCommand 让浏览器自行产出事件序列
|
|
618
|
-
if (document.execCommand('insertText', false, replacementText)) return 'ok';
|
|
619
|
-
return 'fallback';
|
|
620
|
-
}`, undefined, timeout, [event.text]);
|
|
621
|
-
if (replaceResult === 'readonly') {
|
|
622
|
-
throw new Error(`目标元素不可编辑,已选中文本 "${event.find}" 但无法替换`);
|
|
623
|
-
}
|
|
624
|
-
if (replaceResult === 'fallback') {
|
|
625
|
-
// Fallback: 键盘输入覆盖选区
|
|
626
|
-
await unifiedSession.typeText(event.text);
|
|
627
|
-
}
|
|
628
|
-
if (formatted) {
|
|
629
|
-
return `替换的文本原在 <${formatted}> 标签内,替换后格式可能丢失`;
|
|
630
|
-
}
|
|
818
|
+
return sel && !sel.isCollapsed;
|
|
819
|
+
})()`);
|
|
820
|
+
if (hasSelection) {
|
|
821
|
+
selectionConfirmed = true;
|
|
631
822
|
break;
|
|
632
823
|
}
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
824
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
825
|
+
}
|
|
826
|
+
if (!selectionConfirmed) {
|
|
827
|
+
throw new Error(`选区同步失败:文本 "${event.find}" 已定位但未能建立选区,无法执行替换`);
|
|
828
|
+
}
|
|
829
|
+
const replaceResult = await unifiedSession.evaluate(`function(replacementText) {
|
|
830
|
+
var el = document.activeElement;
|
|
831
|
+
if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') && el.setRangeText) {
|
|
832
|
+
if (el.readOnly || el.disabled) return 'readonly';
|
|
833
|
+
el.setRangeText(replacementText, el.selectionStart, el.selectionEnd, 'end');
|
|
834
|
+
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
835
|
+
return 'ok';
|
|
836
|
+
}
|
|
837
|
+
var sel = window.getSelection();
|
|
838
|
+
if (!sel || sel.isCollapsed) return 'readonly';
|
|
839
|
+
var anchor = sel.anchorNode;
|
|
840
|
+
var editable = anchor instanceof Element ? anchor : anchor && anchor.parentElement;
|
|
841
|
+
while (editable && !editable.isContentEditable && editable !== document.body) {
|
|
842
|
+
editable = editable.parentElement;
|
|
843
|
+
}
|
|
844
|
+
if (!editable || !editable.isContentEditable) return 'readonly';
|
|
845
|
+
if (document.execCommand('insertText', false, replacementText)) return 'ok';
|
|
846
|
+
return 'fallback';
|
|
847
|
+
}`, undefined, timeout, [event.text]);
|
|
848
|
+
if (replaceResult === 'readonly') {
|
|
849
|
+
throw new Error(`目标元素不可编辑,已选中文本 "${event.find}" 但无法替换`);
|
|
850
|
+
}
|
|
851
|
+
if (replaceResult === 'fallback') {
|
|
852
|
+
await unifiedSession.typeText(event.text);
|
|
853
|
+
}
|
|
854
|
+
if (formatted) {
|
|
855
|
+
return `替换的文本原在 <${formatted}> 标签内,替换后格式可能丢失`;
|
|
856
|
+
}
|
|
857
|
+
return undefined;
|
|
858
|
+
}
|
|
859
|
+
async function handleUnifiedEditorContext({ unifiedSession, timeout }, event) {
|
|
860
|
+
return (await editorAction(unifiedSession, 'context', event, timeout));
|
|
861
|
+
}
|
|
862
|
+
async function handleUnifiedEditorInsert({ unifiedSession, timeout }, event) {
|
|
863
|
+
await editorAction(unifiedSession, 'insert', event, timeout);
|
|
864
|
+
return undefined;
|
|
865
|
+
}
|
|
866
|
+
async function handleUnifiedEditorCommand({ unifiedSession, timeout }, event) {
|
|
867
|
+
await editorAction(unifiedSession, 'command', event, timeout);
|
|
868
|
+
return undefined;
|
|
869
|
+
}
|
|
870
|
+
async function handleUnifiedDrag({ unifiedSession, timeout }, event) {
|
|
871
|
+
if ('x' in event.target || 'y' in event.target) {
|
|
872
|
+
throw new Error('drag 的 target 不支持坐标类型,请使用选择器(css/text/xpath/role 等)');
|
|
873
|
+
}
|
|
874
|
+
if ('x' in event.to || 'y' in event.to) {
|
|
875
|
+
throw new Error('drag 的 to 不支持坐标类型,请使用选择器(css/text/xpath/role 等)');
|
|
876
|
+
}
|
|
877
|
+
const srcParams = targetToFindParams(event.target);
|
|
878
|
+
const dstParams = targetToFindParams(event.to);
|
|
879
|
+
const srcNth = srcParams.nth ?? 0;
|
|
880
|
+
const dstNth = dstParams.nth ?? 0;
|
|
881
|
+
const attemptDrag = async () => {
|
|
882
|
+
const srcEls = await unifiedSession.find(srcParams.selector, srcParams.text, srcParams.xpath, timeout);
|
|
883
|
+
const dstEls = await unifiedSession.find(dstParams.selector, dstParams.text, dstParams.xpath, timeout);
|
|
884
|
+
if (srcEls.length <= srcNth) {
|
|
885
|
+
throw new Error(`drag 源元素未找到: ${JSON.stringify(event.target)}`);
|
|
886
|
+
}
|
|
887
|
+
if (dstEls.length <= dstNth) {
|
|
888
|
+
throw new Error(`drag 目标元素未找到: ${JSON.stringify(event.to)}`);
|
|
680
889
|
}
|
|
681
|
-
|
|
682
|
-
|
|
890
|
+
return unifiedSession.dragAndDrop(srcEls[srcNth].refId, dstEls[dstNth].refId);
|
|
891
|
+
};
|
|
892
|
+
let dragResult = await attemptDrag();
|
|
893
|
+
let retried = false;
|
|
894
|
+
if (!dragResult.success && dragResult.code === 'REF_STALE') {
|
|
895
|
+
console.warn('[MCP] drag refId 失效,自动重试一次:', dragResult.error);
|
|
896
|
+
dragResult = await attemptDrag();
|
|
897
|
+
retried = true;
|
|
683
898
|
}
|
|
899
|
+
if (!dragResult.success) {
|
|
900
|
+
throw new Error(dragResult.error || 'drag 执行失败');
|
|
901
|
+
}
|
|
902
|
+
return retried ? 'drag 因 refId 失效已自动重试一次(可能是 React 等框架重渲染导致)' : undefined;
|
|
684
903
|
}
|
|
685
904
|
/**
|
|
686
905
|
* Extension 模式:获取目标点坐标
|
|
@@ -693,6 +912,135 @@ async function executeEventExtension(unifiedSession, event, humanize, timeout) {
|
|
|
693
912
|
* - precise 模式直接使用
|
|
694
913
|
* - stealth 模式需减 offset 转为 iframe 相对坐标
|
|
695
914
|
*/
|
|
915
|
+
async function inputControlled(unifiedSession, event, timeout) {
|
|
916
|
+
if (!event.target) {
|
|
917
|
+
throw new Error('controlled 输入需要 target 参数定位输入元素');
|
|
918
|
+
}
|
|
919
|
+
if ('x' in event.target && 'y' in event.target) {
|
|
920
|
+
throw new Error('controlled 输入不支持坐标型 target,请使用 CSS 选择器、role 或文本定位');
|
|
921
|
+
}
|
|
922
|
+
const { selector, text: searchText, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
923
|
+
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
924
|
+
const nth = nthParam ?? 0;
|
|
925
|
+
if (elements.length === 0 || nth >= elements.length) {
|
|
926
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', `controlled 输入目标未找到: ${JSON.stringify(event.target)}`, '请检查 target 是否能定位到 input、textarea、select 或 contenteditable 元素;context.candidates 提供当前页面候选元素', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout));
|
|
927
|
+
}
|
|
928
|
+
const result = await unifiedSession.dispatchInput(elements[nth].refId, event.text ?? '');
|
|
929
|
+
if (!result.success) {
|
|
930
|
+
throw new StructuredToolError('CONTROLLED_INPUT_FAILED', result.error || 'controlled 输入失败', '请检查目标元素是否可编辑、未 disabled,并确认当前 frame 与 target 匹配', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout, result.error || 'controlled 输入失败'));
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
async function buildTargetDiagnosticContext(unifiedSession, target, matchCount, nth, timeout, reason = '目标元素未找到') {
|
|
934
|
+
let page;
|
|
935
|
+
try {
|
|
936
|
+
page = await unifiedSession.evaluate(`(() => {
|
|
937
|
+
const active = document.activeElement;
|
|
938
|
+
const selection = window.getSelection();
|
|
939
|
+
const candidates = Array.from(document.querySelectorAll('input, textarea, select, [contenteditable="true"], button, a, [role="button"], [role="textbox"], [role="combobox"]')).slice(0, 20);
|
|
940
|
+
return {
|
|
941
|
+
activeElement: active ? {
|
|
942
|
+
tag: active.tagName.toLowerCase(),
|
|
943
|
+
id: active.id || undefined,
|
|
944
|
+
className: typeof active.className === 'string' ? active.className : undefined,
|
|
945
|
+
text: (active.textContent || '').trim().slice(0, 80),
|
|
946
|
+
value: 'value' in active ? active.value : undefined,
|
|
947
|
+
selectionStart: 'selectionStart' in active ? active.selectionStart : undefined,
|
|
948
|
+
selectionEnd: 'selectionEnd' in active ? active.selectionEnd : undefined
|
|
949
|
+
} : null,
|
|
950
|
+
selection: {
|
|
951
|
+
text: selection ? selection.toString().slice(0, 160) : '',
|
|
952
|
+
collapsed: selection ? selection.isCollapsed : true,
|
|
953
|
+
anchorNode: selection && selection.anchorNode ? selection.anchorNode.nodeName : null,
|
|
954
|
+
focusNode: selection && selection.focusNode ? selection.focusNode.nodeName : null
|
|
955
|
+
},
|
|
956
|
+
candidates: candidates.map(function(el) {
|
|
957
|
+
const rect = el.getBoundingClientRect();
|
|
958
|
+
return {
|
|
959
|
+
tag: el.tagName.toLowerCase(),
|
|
960
|
+
id: el.id || undefined,
|
|
961
|
+
role: el.getAttribute('role') || undefined,
|
|
962
|
+
text: (el.textContent || el.getAttribute('aria-label') || el.getAttribute('placeholder') || '').trim().slice(0, 80),
|
|
963
|
+
visible: rect.width > 0 && rect.height > 0,
|
|
964
|
+
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
|
|
965
|
+
bounds: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}
|
|
966
|
+
};
|
|
967
|
+
})
|
|
968
|
+
};
|
|
969
|
+
})()`, undefined, timeout);
|
|
970
|
+
}
|
|
971
|
+
catch (err) {
|
|
972
|
+
page = { diagnosticError: err instanceof Error ? err.message : String(err) };
|
|
973
|
+
}
|
|
974
|
+
return { reason, target, matchCount, nth, page };
|
|
975
|
+
}
|
|
976
|
+
async function editorAction(unifiedSession, action, event, timeout) {
|
|
977
|
+
if (event.target) {
|
|
978
|
+
const focused = await focusTargetForCommands(unifiedSession, event.target, timeout);
|
|
979
|
+
if (!focused) {
|
|
980
|
+
throw new Error('editor 目标未找到或未成功聚焦');
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
return unifiedSession.evaluate(`function(action, text, command) {
|
|
984
|
+
var active = document.activeElement;
|
|
985
|
+
var sel = window.getSelection();
|
|
986
|
+
var editable = active;
|
|
987
|
+
if (!editable || editable === document.body || editable === document.documentElement) {
|
|
988
|
+
var anchor = sel && sel.anchorNode;
|
|
989
|
+
editable = anchor instanceof Element ? anchor : anchor && anchor.parentElement;
|
|
990
|
+
while (editable && !editable.isContentEditable && editable !== document.body) {
|
|
991
|
+
editable = editable.parentElement;
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
var isInput = !!active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA');
|
|
995
|
+
var isEditable = !!editable && editable.isContentEditable;
|
|
996
|
+
if (!isInput && !isEditable) {
|
|
997
|
+
throw new Error('editor 事件需要已聚焦 input、textarea 或 contenteditable 元素');
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
if (action === 'context') {
|
|
1001
|
+
var selectedText = '';
|
|
1002
|
+
if (isInput) {
|
|
1003
|
+
selectedText = active.value.slice(active.selectionStart || 0, active.selectionEnd || 0);
|
|
1004
|
+
} else if (sel) {
|
|
1005
|
+
selectedText = sel.toString();
|
|
1006
|
+
}
|
|
1007
|
+
return {
|
|
1008
|
+
activeElement: active ? {
|
|
1009
|
+
tag: active.tagName.toLowerCase(),
|
|
1010
|
+
id: active.id || undefined,
|
|
1011
|
+
isContentEditable: !!active.isContentEditable,
|
|
1012
|
+
selectionStart: isInput ? active.selectionStart : undefined,
|
|
1013
|
+
selectionEnd: isInput ? active.selectionEnd : undefined
|
|
1014
|
+
} : null,
|
|
1015
|
+
editableElement: editable ? {
|
|
1016
|
+
tag: editable.tagName.toLowerCase(),
|
|
1017
|
+
id: editable.id || undefined,
|
|
1018
|
+
isContentEditable: !!editable.isContentEditable
|
|
1019
|
+
} : null,
|
|
1020
|
+
selectedText: selectedText,
|
|
1021
|
+
selectionCollapsed: isInput ? active.selectionStart === active.selectionEnd : !sel || sel.isCollapsed
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
if (action === 'insert') {
|
|
1026
|
+
if (isInput) {
|
|
1027
|
+
active.setRangeText(text, active.selectionStart || 0, active.selectionEnd || 0, 'end');
|
|
1028
|
+
active.dispatchEvent(new Event('input', {bubbles: true}));
|
|
1029
|
+
active.dispatchEvent(new Event('change', {bubbles: true}));
|
|
1030
|
+
return {success: true};
|
|
1031
|
+
}
|
|
1032
|
+
if (document.execCommand('insertText', false, text)) {
|
|
1033
|
+
return {success: true};
|
|
1034
|
+
}
|
|
1035
|
+
throw new Error('editorInsert 无法插入文本');
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
if (!document.execCommand(command, false, text || null)) {
|
|
1039
|
+
throw new Error('editorCommand 执行失败: ' + command);
|
|
1040
|
+
}
|
|
1041
|
+
return {success: true, command: command};
|
|
1042
|
+
}`, undefined, timeout, [action, event.text ?? '', event.command ?? '']);
|
|
1043
|
+
}
|
|
696
1044
|
async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
697
1045
|
const frameOffset = unifiedSession.getFrameOffset();
|
|
698
1046
|
const isStealth = unifiedSession.getInputMode() === 'stealth';
|
|
@@ -706,12 +1054,12 @@ async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
|
706
1054
|
}
|
|
707
1055
|
const { selector, text, xpath, nth: nthParam } = targetToFindParams(target);
|
|
708
1056
|
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
1057
|
+
const nth = nthParam ?? 0;
|
|
709
1058
|
if (elements.length === 0) {
|
|
710
|
-
throw new
|
|
1059
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', `未找到目标元素: ${JSON.stringify(target)}`, '请检查 target 是否正确,或先用 extract type=state 查看 interactiveElements 候选', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout));
|
|
711
1060
|
}
|
|
712
|
-
const nth = nthParam ?? 0;
|
|
713
1061
|
if (nth >= elements.length) {
|
|
714
|
-
throw new
|
|
1062
|
+
throw new StructuredToolError('TARGET_INDEX_OUT_OF_RANGE', `第 ${nth} 个匹配元素不存在(共 ${elements.length} 个)`, '请降低 nth,或先用 extract type=state 查看当前匹配到的候选元素', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout, '目标元素序号越界'));
|
|
715
1063
|
}
|
|
716
1064
|
// 视口外时滚动后重新取 rect:与 actionableClick (left+single) 行为对齐,
|
|
717
1065
|
// 否则非左键 / 多击的坐标路径在视口外坐标 dispatch,浏览器找不到元素,事件丢失
|
|
@@ -752,142 +1100,125 @@ async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
|
752
1100
|
refId,
|
|
753
1101
|
};
|
|
754
1102
|
}
|
|
755
|
-
// iframe + precise:消费者(chrome.debugger
|
|
1103
|
+
// iframe + precise:消费者(chrome.debugger)需要父视口绝对,
|
|
756
1104
|
// scrollIntoView({block:'center'}) 会 cascade 到父框架,导致 frameOffset 与父绝对 rect 都过期,
|
|
757
1105
|
// refetch find() 让 content-handler 重新计算 frameOffset 并返回最新父绝对 rect
|
|
758
1106
|
const refreshed = await unifiedSession.find(selector, text, xpath, timeout);
|
|
759
1107
|
const rect = refreshed[nth]?.rect ?? elements[nth].rect;
|
|
760
1108
|
return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2, refId };
|
|
761
1109
|
}
|
|
762
|
-
|
|
763
|
-
* CDP 模式:执行单个事件
|
|
764
|
-
*/
|
|
765
|
-
async function executeEvent(session, event, humanize, timeout) {
|
|
1110
|
+
async function executeCdpEvent(session, event, humanize, timeout) {
|
|
766
1111
|
validateEvent(event);
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
else {
|
|
838
|
-
await session.touchMove(point.x, point.y);
|
|
839
|
-
}
|
|
840
|
-
break;
|
|
841
|
-
}
|
|
842
|
-
case 'touchend': {
|
|
843
|
-
await session.touchEnd();
|
|
844
|
-
break;
|
|
1112
|
+
const handler = cdpInputHandlers[event.type];
|
|
1113
|
+
if (!handler) {
|
|
1114
|
+
throw new Error(`未知事件类型: ${event.type}`);
|
|
1115
|
+
}
|
|
1116
|
+
await handler({ session, humanize, timeout }, event);
|
|
1117
|
+
}
|
|
1118
|
+
const cdpInputHandlers = {
|
|
1119
|
+
keydown: handleCdpKeyDown,
|
|
1120
|
+
keyup: handleCdpKeyUp,
|
|
1121
|
+
click: handleCdpClick,
|
|
1122
|
+
mousedown: handleCdpMouseDown,
|
|
1123
|
+
mouseup: handleCdpMouseUp,
|
|
1124
|
+
mousemove: handleCdpMouseMove,
|
|
1125
|
+
wheel: handleCdpWheel,
|
|
1126
|
+
touchstart: handleCdpTouchStart,
|
|
1127
|
+
touchmove: handleCdpTouchMove,
|
|
1128
|
+
touchend: handleCdpTouchEnd,
|
|
1129
|
+
type: handleCdpType,
|
|
1130
|
+
wait: handleCdpWait,
|
|
1131
|
+
drag: handleCdpDrag,
|
|
1132
|
+
};
|
|
1133
|
+
async function handleCdpKeyDown({ session }, event) {
|
|
1134
|
+
await session.keyDown(event.key, event.commands);
|
|
1135
|
+
}
|
|
1136
|
+
async function handleCdpKeyUp({ session }, event) {
|
|
1137
|
+
await session.keyUp(event.key);
|
|
1138
|
+
}
|
|
1139
|
+
async function handleCdpClick({ session, humanize, timeout }, event) {
|
|
1140
|
+
if (event.target) {
|
|
1141
|
+
await moveToTarget(session, event.target, humanize, timeout);
|
|
1142
|
+
}
|
|
1143
|
+
const cdpButton = event.button ?? 'left';
|
|
1144
|
+
const cdpClickCount = event.clickCount ?? 1;
|
|
1145
|
+
for (let i = 1; i <= cdpClickCount; i++) {
|
|
1146
|
+
await session.mouseDown(cdpButton, i);
|
|
1147
|
+
await session.mouseUp(cdpButton, i);
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
async function handleCdpMouseDown({ session, humanize, timeout }, event) {
|
|
1151
|
+
if (event.target) {
|
|
1152
|
+
await moveToTarget(session, event.target, humanize, timeout);
|
|
1153
|
+
}
|
|
1154
|
+
await session.mouseDown(event.button ?? 'left');
|
|
1155
|
+
}
|
|
1156
|
+
async function handleCdpMouseUp({ session }, event) {
|
|
1157
|
+
await session.mouseUp(event.button ?? 'left');
|
|
1158
|
+
}
|
|
1159
|
+
async function handleCdpMouseMove({ session, humanize, timeout }, event) {
|
|
1160
|
+
await moveToTarget(session, event.target, humanize, timeout, event.steps);
|
|
1161
|
+
}
|
|
1162
|
+
async function handleCdpWheel({ session, humanize, timeout }, event) {
|
|
1163
|
+
if (event.target) {
|
|
1164
|
+
await moveToTarget(session, event.target, humanize, timeout);
|
|
1165
|
+
}
|
|
1166
|
+
await session.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
1167
|
+
}
|
|
1168
|
+
async function handleCdpTouchStart({ session, timeout }, event) {
|
|
1169
|
+
const point = await getTargetPoint(session, event.target, timeout);
|
|
1170
|
+
await session.touchStart(point.x, point.y);
|
|
1171
|
+
}
|
|
1172
|
+
async function handleCdpTouchMove({ session, humanize, timeout }, event) {
|
|
1173
|
+
const point = await getTargetPoint(session, event.target, timeout);
|
|
1174
|
+
if (humanize && event.steps && event.steps > 1) {
|
|
1175
|
+
const current = session.getBehaviorSimulator().getCurrentPosition();
|
|
1176
|
+
const path = generateBezierPath(current, point, event.steps);
|
|
1177
|
+
for (const p of path) {
|
|
1178
|
+
await session.touchMove(p.x, p.y);
|
|
1179
|
+
await randomDelay(5, 15);
|
|
845
1180
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
// 人类化打字
|
|
870
|
-
for (const char of event.text) {
|
|
871
|
-
await session.type(char);
|
|
872
|
-
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
else {
|
|
876
|
-
await session.type(event.text, delay);
|
|
877
|
-
}
|
|
878
|
-
break;
|
|
1181
|
+
}
|
|
1182
|
+
else {
|
|
1183
|
+
await session.touchMove(point.x, point.y);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
async function handleCdpTouchEnd({ session }) {
|
|
1187
|
+
await session.touchEnd();
|
|
1188
|
+
}
|
|
1189
|
+
async function handleCdpType({ session, humanize, timeout }, event) {
|
|
1190
|
+
if (event.dispatch || event.mode === 'controlled') {
|
|
1191
|
+
throw new Error('controlled/dispatch 输入需要 Extension 连接,当前为 CDP 模式');
|
|
1192
|
+
}
|
|
1193
|
+
if (event.target) {
|
|
1194
|
+
await moveToTarget(session, event.target, humanize, timeout, undefined, 'input');
|
|
1195
|
+
await session.mouseDown('left');
|
|
1196
|
+
await session.mouseUp('left');
|
|
1197
|
+
}
|
|
1198
|
+
else {
|
|
1199
|
+
const hasActiveFocus = await session.evaluate('!!document.activeElement && ' +
|
|
1200
|
+
'document.activeElement !== document.body && ' +
|
|
1201
|
+
'document.activeElement !== document.documentElement');
|
|
1202
|
+
if (!hasActiveFocus) {
|
|
1203
|
+
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
879
1204
|
}
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
1205
|
+
}
|
|
1206
|
+
const delay = event.delay ?? 0;
|
|
1207
|
+
if (humanize) {
|
|
1208
|
+
for (const char of event.text) {
|
|
1209
|
+
await session.type(char);
|
|
1210
|
+
await randomDelay(getTypingDelay(delay), getTypingDelay(delay) * 1.5);
|
|
883
1211
|
}
|
|
884
|
-
default:
|
|
885
|
-
// drag 仅在 Extension 模式可用,给出明确错误而非通用"未知事件类型"
|
|
886
|
-
if (event.type === 'drag') {
|
|
887
|
-
throw new Error('drag 事件仅在 Extension 模式下可用,当前为 CDP 模式');
|
|
888
|
-
}
|
|
889
|
-
throw new Error(`未知事件类型: ${event.type}`);
|
|
890
1212
|
}
|
|
1213
|
+
else {
|
|
1214
|
+
await session.type(event.text, delay);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
async function handleCdpWait(_context, event) {
|
|
1218
|
+
await new Promise((resolve) => setTimeout(resolve, event.ms));
|
|
1219
|
+
}
|
|
1220
|
+
async function handleCdpDrag() {
|
|
1221
|
+
throw new Error('drag 事件仅在 Extension 模式下可用,当前为 CDP 模式');
|
|
891
1222
|
}
|
|
892
1223
|
/**
|
|
893
1224
|
* 移动到目标位置
|
|
@@ -966,6 +1297,12 @@ export function registerInputTool(server) {
|
|
|
966
1297
|
{type: "keydown", key: "a", commands: ["selectAll"]},
|
|
967
1298
|
{type: "keyup", key: "a"}
|
|
968
1299
|
]
|
|
1300
|
+
|
|
1301
|
+
最短示例:
|
|
1302
|
+
click: {events:[{type:"click",target:{css:"button[type=submit]"}}]}
|
|
1303
|
+
type: {events:[{type:"type",target:{css:"input[name=q]"},text:"hello"}]}
|
|
1304
|
+
controlled type: {events:[{type:"type",mode:"controlled",target:{css:"input[name=q]"},text:"hello"}]}
|
|
1305
|
+
replace: {events:[{type:"replace",target:{css:"textarea"},find:"old",text:"new"}]}
|
|
969
1306
|
注意:纯键盘事件(不带 commands)仅保证 JS keyboard event 可被监听,不保证触发浏览器原生编辑行为;
|
|
970
1307
|
全选/复制/粘贴等语义用 commands;"全选并替换文本"用 select/replace 事件更简洁;
|
|
971
1308
|
commands 仅支持 inputMode=precise,stealth 模式下会报错`,
|