@pyrokine/mcp-chrome 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +721 -303
- package/README_zh.md +750 -292
- package/dist/cdp/client.d.ts +7 -3
- package/dist/cdp/client.d.ts.map +1 -1
- package/dist/cdp/client.js +252 -63
- package/dist/cdp/client.js.map +1 -1
- package/dist/cdp/launcher.d.ts +1 -1
- package/dist/cdp/launcher.d.ts.map +1 -1
- package/dist/cdp/launcher.js +33 -12
- package/dist/cdp/launcher.js.map +1 -1
- package/dist/core/auto-wait.d.ts.map +1 -1
- package/dist/core/auto-wait.js +88 -52
- package/dist/core/auto-wait.js.map +1 -1
- package/dist/core/browser-driver.d.ts +76 -18
- package/dist/core/browser-driver.d.ts.map +1 -1
- package/dist/core/browser-driver.js.map +1 -1
- package/dist/core/browser-event-schema.d.ts +287 -0
- package/dist/core/browser-event-schema.d.ts.map +1 -0
- package/dist/core/browser-event-schema.js +104 -0
- package/dist/core/browser-event-schema.js.map +1 -0
- package/dist/core/browser-events.d.ts +99 -0
- package/dist/core/browser-events.d.ts.map +1 -0
- package/dist/core/browser-events.js +1312 -0
- package/dist/core/browser-events.js.map +1 -0
- package/dist/core/challenge-page.d.ts +39 -0
- package/dist/core/challenge-page.d.ts.map +1 -0
- package/dist/core/challenge-page.js +226 -0
- package/dist/core/challenge-page.js.map +1 -0
- package/dist/core/dom-helpers.d.ts +28 -0
- package/dist/core/dom-helpers.d.ts.map +1 -0
- package/dist/core/dom-helpers.js +311 -0
- package/dist/core/dom-helpers.js.map +1 -0
- package/dist/core/errors.d.ts +10 -4
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +63 -13
- package/dist/core/errors.js.map +1 -1
- package/dist/core/file-upload.d.ts +22 -0
- package/dist/core/file-upload.d.ts.map +1 -0
- package/dist/core/file-upload.js +194 -0
- package/dist/core/file-upload.js.map +1 -0
- package/dist/core/locator.d.ts +17 -2
- package/dist/core/locator.d.ts.map +1 -1
- package/dist/core/locator.js +156 -212
- package/dist/core/locator.js.map +1 -1
- package/dist/core/os-mouse.d.ts +11 -0
- package/dist/core/os-mouse.d.ts.map +1 -1
- package/dist/core/os-mouse.js +102 -29
- package/dist/core/os-mouse.js.map +1 -1
- package/dist/core/retry.d.ts.map +1 -1
- package/dist/core/retry.js +4 -1
- package/dist/core/retry.js.map +1 -1
- package/dist/core/session.d.ts +65 -14
- package/dist/core/session.d.ts.map +1 -1
- package/dist/core/session.js +1089 -273
- package/dist/core/session.js.map +1 -1
- package/dist/core/types.d.ts +5 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +16 -1
- package/dist/core/types.js.map +1 -1
- package/dist/core/unified-session.d.ts +50 -41
- package/dist/core/unified-session.d.ts.map +1 -1
- package/dist/core/unified-session.js +938 -285
- package/dist/core/unified-session.js.map +1 -1
- package/dist/core/utils.d.ts +5 -4
- package/dist/core/utils.d.ts.map +1 -1
- package/dist/core/utils.js +73 -18
- package/dist/core/utils.js.map +1 -1
- package/dist/extension/bridge.d.ts +50 -19
- package/dist/extension/bridge.d.ts.map +1 -1
- package/dist/extension/bridge.js +289 -57
- package/dist/extension/bridge.js.map +1 -1
- package/dist/extension/http-server.d.ts +7 -2
- package/dist/extension/http-server.d.ts.map +1 -1
- package/dist/extension/http-server.js +219 -48
- package/dist/extension/http-server.js.map +1 -1
- package/dist/index.js +14 -1
- package/dist/index.js.map +1 -1
- package/dist/tools/browse.d.ts.map +1 -1
- package/dist/tools/browse.js +66 -32
- package/dist/tools/browse.js.map +1 -1
- package/dist/tools/browser-event-output.d.ts +7 -0
- package/dist/tools/browser-event-output.d.ts.map +1 -0
- package/dist/tools/browser-event-output.js +19 -0
- package/dist/tools/browser-event-output.js.map +1 -0
- package/dist/tools/challenge.d.ts +17 -5
- package/dist/tools/challenge.d.ts.map +1 -1
- package/dist/tools/challenge.js +113 -195
- package/dist/tools/challenge.js.map +1 -1
- package/dist/tools/cookies.d.ts.map +1 -1
- package/dist/tools/cookies.js +33 -8
- package/dist/tools/cookies.js.map +1 -1
- package/dist/tools/diagnostics.d.ts.map +1 -1
- package/dist/tools/diagnostics.js +14 -0
- package/dist/tools/diagnostics.js.map +1 -1
- package/dist/tools/evaluate.d.ts +6 -5
- package/dist/tools/evaluate.d.ts.map +1 -1
- package/dist/tools/evaluate.js +102 -52
- package/dist/tools/evaluate.js.map +1 -1
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +178 -124
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/input.d.ts.map +1 -1
- package/dist/tools/input.js +623 -271
- package/dist/tools/input.js.map +1 -1
- package/dist/tools/logs.d.ts.map +1 -1
- package/dist/tools/logs.js +50 -11
- package/dist/tools/logs.js.map +1 -1
- package/dist/tools/manage.d.ts.map +1 -1
- package/dist/tools/manage.js +127 -46
- package/dist/tools/manage.js.map +1 -1
- package/dist/tools/network-sanitizer.d.ts +1 -0
- package/dist/tools/network-sanitizer.d.ts.map +1 -1
- package/dist/tools/network-sanitizer.js +30 -1
- package/dist/tools/network-sanitizer.js.map +1 -1
- package/dist/tools/page-context.d.ts +4 -3
- package/dist/tools/page-context.d.ts.map +1 -1
- package/dist/tools/page-context.js +56 -14
- package/dist/tools/page-context.js.map +1 -1
- package/dist/tools/post-condition.d.ts +1 -1
- package/dist/tools/post-condition.d.ts.map +1 -1
- package/dist/tools/post-condition.js +153 -47
- package/dist/tools/post-condition.js.map +1 -1
- package/dist/tools/schema.d.ts +7 -4
- package/dist/tools/schema.d.ts.map +1 -1
- package/dist/tools/schema.js +32 -122
- package/dist/tools/schema.js.map +1 -1
- package/dist/tools/target-diagnostics.d.ts +10 -2
- package/dist/tools/target-diagnostics.d.ts.map +1 -1
- package/dist/tools/target-diagnostics.js +74 -21
- package/dist/tools/target-diagnostics.js.map +1 -1
- package/dist/tools/wait.d.ts.map +1 -1
- package/dist/tools/wait.js +108 -42
- package/dist/tools/wait.js.map +1 -1
- package/extension/dist/assets/{content.ts-C4imZ6A9.js → content.ts-BAMOcF3R.js} +1 -1
- package/extension/dist/assets/{content.ts-loader-CnJuCGhl.js → content.ts-loader-4F9a1EWZ.js} +1 -1
- package/extension/dist/assets/index.ts-B79g2ikq.js +19820 -0
- package/extension/dist/assets/{page-scripts-CMIY1zhd.js → page-scripts-CSxn58h1.js} +300 -410
- package/extension/dist/assets/{popup.html-CgtWL6Py.js → popup.html-DwwHo00N.js} +8 -6
- package/extension/dist/manifest.json +4 -4
- package/extension/dist/service-worker-loader.js +1 -1
- package/extension/dist/src/popup/popup.html +3 -3
- package/package.json +15 -7
- package/scripts/start-chrome.ps1 +9 -1
- package/extension/dist/assets/index.ts-GeHNZBml.js +0 -8972
package/dist/tools/input.js
CHANGED
|
@@ -8,17 +8,20 @@
|
|
|
8
8
|
* - 支持任意组合:Ctrl+Alt+A+左键+右键+拖拽
|
|
9
9
|
* - humanize 可选:行为模拟是可选功能
|
|
10
10
|
*/
|
|
11
|
+
import { randomUUID } from 'node:crypto';
|
|
12
|
+
import { prepareUploadFiles, uploadInPage } from '../core/file-upload.js';
|
|
11
13
|
import { z } from 'zod';
|
|
12
14
|
import { generateBezierPath, getMouseMoveDelay, getTypingDelay, randomDelay } from '../anti-detection/index.js';
|
|
15
|
+
import { BrowserError, ElementNotFoundError, getErrorPageContext, isNativeExecutionResultUnknown, StructuredToolError, TimeoutError, } from '../core/errors.js';
|
|
13
16
|
import { formatErrorResponse, formatResponse, getSession, getUnifiedSession } from '../core/index.js';
|
|
14
|
-
import { StructuredToolError, TimeoutError } from '../core/errors.js';
|
|
15
17
|
import { DEFAULT_TIMEOUT } from '../core/types.js';
|
|
16
|
-
import { ChallengeDeniedError, ChallengeTimeoutError, resolveChallenge } from './challenge.js';
|
|
18
|
+
import { ChallengeDeniedError, challengeResultFields, ChallengeTimeoutError, resolveChallenge } from './challenge.js';
|
|
17
19
|
import { appendDiagnostics, finishDiagnostics, startDiagnostics } from './diagnostics.js';
|
|
18
20
|
import { withPageContext } from './page-context.js';
|
|
19
21
|
import { postConditionSchema, waitForPostCondition } from './post-condition.js';
|
|
20
|
-
import { pageConditionSchema, pageTargetIdSchema, resolvePageTargetId, targetToFindParams, targetZodSchema, } from './schema.js';
|
|
21
|
-
import { buildTargetDiagnostics } from './target-diagnostics.js';
|
|
22
|
+
import { frameZodSchema, pageConditionSchema, pageTargetIdSchema, resolvePageTargetId, targetToFindParams, targetZodSchema, } from './schema.js';
|
|
23
|
+
import { buildTargetDiagnostics, enrichTargetError, targetLookupTimeout } from './target-diagnostics.js';
|
|
24
|
+
import { installDomHelpers } from '../core/dom-helpers.js';
|
|
22
25
|
const INPUT_WAIT_MAX_MS = 60_000;
|
|
23
26
|
const TEXT_SELECTION_INPUT_TYPES = new Set(['text', 'search', 'tel', 'url', 'password']);
|
|
24
27
|
function throwInputTimeoutError(timeout) {
|
|
@@ -76,6 +79,7 @@ const inputEventSchema = z.object({
|
|
|
76
79
|
'touchmove',
|
|
77
80
|
'touchend',
|
|
78
81
|
'type',
|
|
82
|
+
'upload',
|
|
79
83
|
'wait',
|
|
80
84
|
'select',
|
|
81
85
|
'replace',
|
|
@@ -101,12 +105,17 @@ const inputEventSchema = z.object({
|
|
|
101
105
|
.describe('鼠标点击次数(click,默认 1,设为 2 触发双击事件,设为 3 触发三击事件)'),
|
|
102
106
|
target: targetZodSchema
|
|
103
107
|
.optional()
|
|
104
|
-
.describe('目标元素(mousemove/touchstart/touchmove 必填;click/mousedown/wheel/type/drag 可选;
|
|
108
|
+
.describe('目标元素(upload/mousemove/touchstart/touchmove 必填;click/mousedown/wheel/type/drag 可选;' +
|
|
109
|
+
'select/replace 可选;drag 时为拖拽源;upload 支持隐藏文件控件)'),
|
|
105
110
|
to: targetZodSchema.optional().describe('拖拽目标元素(drag 事件必填)'),
|
|
106
111
|
steps: z.number().optional().describe('移动步数(mousemove/touchmove)'),
|
|
107
112
|
deltaX: z.number().optional().describe('水平滚动量'),
|
|
108
113
|
deltaY: z.number().optional().describe('垂直滚动量'),
|
|
109
114
|
text: z.string().max(10000).optional().describe('输入文本(type,最大 10000 字符)或替换文本(replace)'),
|
|
115
|
+
files: z
|
|
116
|
+
.array(z.string().min(1))
|
|
117
|
+
.optional()
|
|
118
|
+
.describe('upload 的 Server 本地文件路径,支持绝对、相对和 ~/ 路径;总大小须小于 50 MiB,空数组清空选择'),
|
|
110
119
|
delay: z.number().min(0).max(100).optional().describe('按键间隔毫秒(type 事件最大 100ms,避免长时延 DoS)'),
|
|
111
120
|
ms: z
|
|
112
121
|
.number()
|
|
@@ -121,11 +130,12 @@ const inputEventSchema = z.object({
|
|
|
121
130
|
mode: z
|
|
122
131
|
.enum(['keyboard', 'controlled'])
|
|
123
132
|
.optional()
|
|
124
|
-
.describe('输入模式(type
|
|
133
|
+
.describe('输入模式(type),默认 keyboard;precise 键盘换行会派发 Enter 并触发页面 Enter 行为;' +
|
|
134
|
+
'controlled 替换完整文本并触发 input/change,不派发 Enter,需 Extension 和 target;选区插入用 editorInsert'),
|
|
125
135
|
dispatch: z
|
|
126
136
|
.boolean()
|
|
127
137
|
.optional()
|
|
128
|
-
.describe('
|
|
138
|
+
.describe('controlled 模式别名(type),替换完整文本并触发 input/change,不派发 Enter;默认 false 使用键盘事件,precise 换行会派发 Enter'),
|
|
129
139
|
force: z
|
|
130
140
|
.boolean()
|
|
131
141
|
.optional()
|
|
@@ -152,17 +162,14 @@ const inputSchema = z.object({
|
|
|
152
162
|
.number()
|
|
153
163
|
.optional()
|
|
154
164
|
.describe('超时毫秒,含 Cloudflare Challenge 等待、目标准备、鼠标移动、字符输入和延迟;到期后停止后续动作,不自动重放'),
|
|
155
|
-
frame:
|
|
156
|
-
.union([z.string(), z.number()])
|
|
157
|
-
.optional()
|
|
158
|
-
.describe('iframe 定位(可选,仅 Extension 模式),CSS 选择器(如 "iframe#main")或索引(如 0),不指定则在主框架操作'),
|
|
165
|
+
frame: frameZodSchema.describe('iframe selector/索引或逐层路径(如 ["iframe#outer", 0]),省略时使用主文档'),
|
|
159
166
|
});
|
|
160
167
|
function sanitizeInputResponse(response, events, sensitiveInput) {
|
|
161
168
|
if (!response.isError && !sensitiveInput) {
|
|
162
169
|
return response;
|
|
163
170
|
}
|
|
164
171
|
const sensitiveValues = events
|
|
165
|
-
.flatMap((event) => [event.find, event.text])
|
|
172
|
+
.flatMap((event) => [event.find, event.text, ...(event.files ?? []).filter((path) => /[/\\]/.test(path))])
|
|
166
173
|
.filter((value) => !!value);
|
|
167
174
|
for (const item of response.content) {
|
|
168
175
|
try {
|
|
@@ -191,6 +198,7 @@ function sanitizeInputResponse(response, events, sensitiveInput) {
|
|
|
191
198
|
'message',
|
|
192
199
|
'suggestion',
|
|
193
200
|
'context',
|
|
201
|
+
'logs',
|
|
194
202
|
'contextError',
|
|
195
203
|
'url',
|
|
196
204
|
'title',
|
|
@@ -221,9 +229,13 @@ function sanitizeInputResponse(response, events, sensitiveInput) {
|
|
|
221
229
|
/**
|
|
222
230
|
* input 工具处理器
|
|
223
231
|
*/
|
|
224
|
-
async function handleInput(args) {
|
|
232
|
+
async function handleInput(args, signal) {
|
|
233
|
+
const inputStartedAt = Date.now();
|
|
234
|
+
const deadline = args.timeout === undefined ? undefined : inputStartedAt + args.timeout;
|
|
225
235
|
let eventsExecuted = 0;
|
|
226
236
|
let actionMayHaveExecuted = false;
|
|
237
|
+
let actionExecutedUnknown = false;
|
|
238
|
+
let challengeFields = {};
|
|
227
239
|
const privacy = { sensitive: false };
|
|
228
240
|
const progress = {
|
|
229
241
|
actionStatus: 'not_started',
|
|
@@ -253,45 +265,55 @@ async function handleInput(args) {
|
|
|
253
265
|
const response = await withPageContext(unifiedSession, resolvePageTargetId(args), async () => {
|
|
254
266
|
return unifiedSession.withPageCondition(args, () => unifiedSession.withFrame(args.frame, async () => {
|
|
255
267
|
const mode = unifiedSession.getMode();
|
|
256
|
-
const inputStartedAt = Date.now();
|
|
257
268
|
if (args.expectedUrl === undefined && args.expectedRevision === undefined) {
|
|
258
|
-
await resolveChallenge(unifiedSession, 'input', 'not_started',
|
|
269
|
+
challengeFields = challengeResultFields(await resolveChallenge(unifiedSession, 'input', 'not_started', deadline === undefined ? undefined : remainingInputTimeout(deadline)));
|
|
259
270
|
}
|
|
260
|
-
const diagnostics = await startDiagnostics(unifiedSession, args.diagnostics);
|
|
271
|
+
const diagnostics = await startDiagnostics(unifiedSession, args.diagnostics, deadline);
|
|
261
272
|
const warnings = [];
|
|
262
273
|
const eventResults = [];
|
|
263
274
|
const session = mode === 'cdp' ? getSession() : undefined;
|
|
264
275
|
try {
|
|
265
|
-
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
276
|
+
await unifiedSession.withInputCleanup(async () => {
|
|
277
|
+
for (const event of args.events) {
|
|
278
|
+
unifiedSession.assertRequestActive();
|
|
279
|
+
const eventTimeout = args.timeout === undefined
|
|
280
|
+
? undefined
|
|
281
|
+
: Math.max(0, args.timeout - (Date.now() - inputStartedAt));
|
|
282
|
+
if (eventTimeout !== undefined && eventTimeout <= 0) {
|
|
283
|
+
throwInputTimeoutError(args.timeout);
|
|
284
|
+
}
|
|
285
|
+
progress.actionStatus = 'failed';
|
|
286
|
+
const result = await unifiedSession.withUserAction('input', () => executeInputEvent({
|
|
287
|
+
unifiedSession,
|
|
288
|
+
session,
|
|
289
|
+
mode,
|
|
290
|
+
humanize,
|
|
291
|
+
timeout: eventTimeout,
|
|
292
|
+
frame: args.frame,
|
|
293
|
+
signal,
|
|
294
|
+
privacy,
|
|
295
|
+
}, event), event.type === 'keydown' &&
|
|
296
|
+
Boolean(event.commands?.length) &&
|
|
297
|
+
event.target !== undefined &&
|
|
298
|
+
'x' in event.target &&
|
|
299
|
+
'y' in event.target);
|
|
300
|
+
++eventsExecuted;
|
|
301
|
+
if (typeof result === 'string') {
|
|
302
|
+
warnings.push(result);
|
|
303
|
+
}
|
|
304
|
+
else if (result) {
|
|
305
|
+
eventResults.push(result);
|
|
306
|
+
}
|
|
288
307
|
}
|
|
289
|
-
|
|
308
|
+
progress.actionStatus = 'completed';
|
|
309
|
+
});
|
|
290
310
|
}
|
|
291
311
|
catch (error) {
|
|
292
312
|
const response = formatErrorResponse(error);
|
|
293
313
|
const text = response.content[0]?.text;
|
|
294
|
-
if ((error instanceof ChallengeTimeoutError ||
|
|
314
|
+
if ((error instanceof ChallengeTimeoutError ||
|
|
315
|
+
error instanceof ChallengeDeniedError) &&
|
|
316
|
+
text) {
|
|
295
317
|
try {
|
|
296
318
|
const payload = JSON.parse(text);
|
|
297
319
|
Object.assign(payload, {
|
|
@@ -299,7 +321,7 @@ async function handleInput(args) {
|
|
|
299
321
|
remainingEventsNotExecuted: args.events.length - eventsExecuted,
|
|
300
322
|
verificationRequested: Boolean(args.postCondition),
|
|
301
323
|
});
|
|
302
|
-
appendDiagnostics(payload, await finishDiagnostics(unifiedSession, diagnostics));
|
|
324
|
+
appendDiagnostics(payload, await finishDiagnostics(unifiedSession, diagnostics, deadline));
|
|
303
325
|
response.content[0].text = JSON.stringify(payload, null, 2);
|
|
304
326
|
}
|
|
305
327
|
catch {
|
|
@@ -312,30 +334,80 @@ async function handleInput(args) {
|
|
|
312
334
|
const payload = JSON.parse(text);
|
|
313
335
|
const message = error instanceof Error ? error.message : String(error);
|
|
314
336
|
const timedOut = /timeout|超时/i.test(message);
|
|
315
|
-
const preActionTimeout = /timeout
|
|
316
|
-
actionMayHaveExecuted =
|
|
337
|
+
const preActionTimeout = /timeout.*before|超过 input 剩余 timeout/i.test(message);
|
|
338
|
+
actionMayHaveExecuted =
|
|
339
|
+
isNativeExecutionResultUnknown(error) || (timedOut && !preActionTimeout);
|
|
317
340
|
const preconditionFailed = error !== null &&
|
|
318
341
|
typeof error === 'object' &&
|
|
319
342
|
'code' in error &&
|
|
320
343
|
error.code === 'ACTION_PRECONDITION_FAILED';
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
344
|
+
const executionContext = error instanceof StructuredToolError || error instanceof BrowserError
|
|
345
|
+
? error.context
|
|
346
|
+
: undefined;
|
|
347
|
+
const contextPhase = getErrorPageContext(error)?.contextPhase;
|
|
348
|
+
const preparationFailure = contextPhase === 'preparation' || contextPhase === 'precondition';
|
|
349
|
+
const explicitNotStarted = preparationFailure || executionContext?.actionExecuted === false;
|
|
350
|
+
actionExecutedUnknown =
|
|
351
|
+
!explicitNotStarted && executionContext?.actionExecuted === 'unknown';
|
|
352
|
+
actionMayHaveExecuted =
|
|
353
|
+
(actionMayHaveExecuted ||
|
|
354
|
+
actionExecutedUnknown ||
|
|
355
|
+
executionContext?.actionExecuted === true) &&
|
|
356
|
+
!preconditionFailed &&
|
|
357
|
+
!explicitNotStarted;
|
|
358
|
+
const cleanupFailure = progress.actionStatus === 'completed';
|
|
359
|
+
const keyboardFocusCleanup = getErrorPageContext(error)?.keyboardFocusCleanup ??
|
|
360
|
+
executionContext?.keyboardFocusCleanup;
|
|
361
|
+
if (keyboardFocusCleanup) {
|
|
362
|
+
payload.keyboardFocusCleanup = keyboardFocusCleanup;
|
|
363
|
+
}
|
|
364
|
+
const keyboardWarning = getErrorPageContext(error)?.keyboardFocusWarning;
|
|
365
|
+
if (typeof keyboardWarning === 'string' &&
|
|
366
|
+
!warnings.includes(keyboardWarning)) {
|
|
367
|
+
warnings.push(keyboardWarning);
|
|
368
|
+
}
|
|
369
|
+
if (warnings.length) {
|
|
370
|
+
payload.warnings = warnings;
|
|
371
|
+
}
|
|
372
|
+
if (cleanupFailure) {
|
|
373
|
+
actionMayHaveExecuted = true;
|
|
374
|
+
}
|
|
375
|
+
progress.actionStatus = cleanupFailure
|
|
376
|
+
? 'completed'
|
|
377
|
+
: (preconditionFailed ||
|
|
378
|
+
preparationFailure ||
|
|
379
|
+
(explicitNotStarted &&
|
|
380
|
+
executionContext?.actionStatus === 'not_started')) &&
|
|
381
|
+
eventsExecuted === 0
|
|
324
382
|
? 'not_started'
|
|
325
|
-
: actionMayHaveExecuted
|
|
383
|
+
: actionMayHaveExecuted && executionContext?.actionStatus !== 'failed'
|
|
326
384
|
? 'unknown'
|
|
327
385
|
: 'failed';
|
|
328
386
|
payload.eventsExecuted = eventsExecuted;
|
|
329
|
-
payload.actionExecuted =
|
|
387
|
+
payload.actionExecuted =
|
|
388
|
+
eventsExecuted > 0
|
|
389
|
+
? true
|
|
390
|
+
: actionExecutedUnknown
|
|
391
|
+
? 'unknown'
|
|
392
|
+
: actionMayHaveExecuted;
|
|
330
393
|
payload.actionStatus = progress.actionStatus;
|
|
331
394
|
payload.verificationRequested = Boolean(args.postCondition);
|
|
332
395
|
payload.verificationStatus = 'unavailable';
|
|
333
|
-
|
|
396
|
+
if (preparationFailure) {
|
|
397
|
+
payload.remainingEventsNotExecuted = args.events.length - eventsExecuted;
|
|
398
|
+
}
|
|
399
|
+
payload.failureStage = preparationFailure
|
|
400
|
+
? contextPhase === 'precondition'
|
|
401
|
+
? 'precondition'
|
|
402
|
+
: 'context'
|
|
403
|
+
: (executionContext?.failureStage ?? 'action');
|
|
334
404
|
payload.retryable =
|
|
335
|
-
|
|
405
|
+
!preparationFailure &&
|
|
406
|
+
executionContext?.retryable !== false &&
|
|
407
|
+
eventsExecuted === 0 &&
|
|
336
408
|
!actionMayHaveExecuted &&
|
|
337
409
|
/timeout|超时|disconnect|未连接|context|debugger/i.test(message);
|
|
338
|
-
appendDiagnostics(payload, await finishDiagnostics(unifiedSession, diagnostics));
|
|
410
|
+
appendDiagnostics(payload, await finishDiagnostics(unifiedSession, diagnostics, deadline));
|
|
339
411
|
response.content[0].text = JSON.stringify(payload, null, 2);
|
|
340
412
|
}
|
|
341
413
|
catch {
|
|
@@ -345,8 +417,19 @@ async function handleInput(args) {
|
|
|
345
417
|
return response;
|
|
346
418
|
}
|
|
347
419
|
progress.actionStatus = 'completed';
|
|
420
|
+
failureStage = 'verification';
|
|
421
|
+
if (deadline === undefined || Date.now() < deadline) {
|
|
422
|
+
Object.assign(challengeFields, challengeResultFields(await resolveChallenge(unifiedSession, 'input', 'completed', deadline === undefined ? undefined : remainingInputTimeout(deadline))));
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
Object.assign(challengeFields, {
|
|
426
|
+
challengeVerificationStatus: 'unavailable',
|
|
427
|
+
challengeVerificationError: '输入已确认完成,剩余 timeout 已用尽',
|
|
428
|
+
});
|
|
429
|
+
}
|
|
348
430
|
failureStage = 'context';
|
|
349
431
|
const result = {
|
|
432
|
+
...challengeFields,
|
|
350
433
|
success: true,
|
|
351
434
|
eventsExecuted,
|
|
352
435
|
mode,
|
|
@@ -364,20 +447,19 @@ async function handleInput(args) {
|
|
|
364
447
|
}
|
|
365
448
|
if (args.postCondition) {
|
|
366
449
|
failureStage = 'verification';
|
|
367
|
-
const postCondition = await waitForPostCondition(unifiedSession, args.postCondition, 'input');
|
|
450
|
+
const postCondition = await waitForPostCondition(unifiedSession, args.postCondition, 'input', undefined, deadline, signal);
|
|
368
451
|
failureStage = 'context';
|
|
369
452
|
result.postCondition = postCondition;
|
|
370
453
|
result.verificationStatus = postCondition.verificationStatus;
|
|
371
|
-
result.retryable = postCondition.retryable;
|
|
372
454
|
if (postCondition.verificationStatus !== 'matched') {
|
|
373
455
|
result.success = false;
|
|
374
456
|
result.failureStage = 'verification';
|
|
375
457
|
}
|
|
376
458
|
}
|
|
377
|
-
appendDiagnostics(result, await finishDiagnostics(unifiedSession, diagnostics));
|
|
459
|
+
appendDiagnostics(result, await finishDiagnostics(unifiedSession, diagnostics, deadline));
|
|
378
460
|
return formatResponse(result);
|
|
379
|
-
})); // withPageCondition, withFrame
|
|
380
|
-
}); // withTabId
|
|
461
|
+
}, deadline), deadline); // withPageCondition, withFrame, withInputCleanup
|
|
462
|
+
}, deadline, signal, 'input'); // withTabId
|
|
381
463
|
return sanitizeInputResponse(response, args.events, privacy.sensitive);
|
|
382
464
|
}
|
|
383
465
|
catch (error) {
|
|
@@ -400,18 +482,46 @@ async function handleInput(args) {
|
|
|
400
482
|
typeof error === 'object' &&
|
|
401
483
|
'code' in error &&
|
|
402
484
|
error.code === 'ACTION_PRECONDITION_FAILED');
|
|
485
|
+
const executionContext = error instanceof StructuredToolError || error instanceof BrowserError ? error.context : undefined;
|
|
486
|
+
const challengeFailure = error instanceof ChallengeTimeoutError || error instanceof ChallengeDeniedError ? error : null;
|
|
487
|
+
const contextPhase = getErrorPageContext(error)?.contextPhase;
|
|
488
|
+
const preparationFailure = progress.actionStatus === 'not_started' &&
|
|
489
|
+
(contextPhase === 'before' || contextPhase === 'precondition' || contextPhase === 'preparation');
|
|
403
490
|
Object.assign(payload, {
|
|
491
|
+
...challengeFields,
|
|
404
492
|
eventsExecuted,
|
|
405
|
-
|
|
406
|
-
|
|
493
|
+
remainingEventsNotExecuted: args.events.length - eventsExecuted,
|
|
494
|
+
actionExecuted: preparationFailure
|
|
495
|
+
? false
|
|
496
|
+
: eventsExecuted > 0
|
|
497
|
+
? true
|
|
498
|
+
: actionExecutedUnknown || executionContext?.actionExecuted === 'unknown'
|
|
499
|
+
? 'unknown'
|
|
500
|
+
: executionContext?.actionExecuted === true || actionMayHaveExecuted,
|
|
501
|
+
actionStatus: preparationFailure
|
|
407
502
|
? 'not_started'
|
|
408
|
-
: progress.actionStatus === '
|
|
409
|
-
? '
|
|
410
|
-
:
|
|
503
|
+
: progress.actionStatus === 'completed'
|
|
504
|
+
? 'completed'
|
|
505
|
+
: preconditionFailed && eventsExecuted === 0
|
|
506
|
+
? 'not_started'
|
|
507
|
+
: challengeFailure
|
|
508
|
+
? challengeFailure.actionStatus
|
|
509
|
+
: (executionContext?.actionStatus ??
|
|
510
|
+
(progress.actionStatus === 'not_started' ? 'failed' : progress.actionStatus)),
|
|
411
511
|
verificationRequested: Boolean(args.postCondition),
|
|
412
|
-
verificationStatus:
|
|
413
|
-
|
|
414
|
-
|
|
512
|
+
verificationStatus: challengeFailure
|
|
513
|
+
? payload.verificationStatus
|
|
514
|
+
: (executionContext?.verificationStatus ?? 'unavailable'),
|
|
515
|
+
failureStage: preparationFailure
|
|
516
|
+
? contextPhase === 'precondition'
|
|
517
|
+
? 'precondition'
|
|
518
|
+
: 'context'
|
|
519
|
+
: unavailableCondition
|
|
520
|
+
? 'precondition'
|
|
521
|
+
: (executionContext?.failureStage ?? failureStage),
|
|
522
|
+
retryable: !preparationFailure &&
|
|
523
|
+
executionContext?.retryable !== false &&
|
|
524
|
+
progress.actionStatus !== 'completed' &&
|
|
415
525
|
eventsExecuted === 0 &&
|
|
416
526
|
!actionMayHaveExecuted &&
|
|
417
527
|
!preconditionFailed &&
|
|
@@ -434,6 +544,7 @@ async function handleInput(args) {
|
|
|
434
544
|
}
|
|
435
545
|
}
|
|
436
546
|
const unifiedOnlyEventTypes = new Set([
|
|
547
|
+
'upload',
|
|
437
548
|
'keydown',
|
|
438
549
|
'keyup',
|
|
439
550
|
'select',
|
|
@@ -443,22 +554,35 @@ const unifiedOnlyEventTypes = new Set([
|
|
|
443
554
|
'editorCommand',
|
|
444
555
|
]);
|
|
445
556
|
async function executeInputEvent(context, event) {
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
557
|
+
const timeout = context.timeout ?? DEFAULT_TIMEOUT;
|
|
558
|
+
const deadline = Date.now() + timeout;
|
|
559
|
+
try {
|
|
560
|
+
if (context.mode === 'extension' || unifiedOnlyEventTypes.has(event.type)) {
|
|
561
|
+
return await executeUnifiedEvent({
|
|
562
|
+
unifiedSession: context.unifiedSession,
|
|
563
|
+
session: context.session,
|
|
564
|
+
mode: context.mode,
|
|
565
|
+
humanize: context.humanize,
|
|
566
|
+
timeout: context.timeout,
|
|
567
|
+
frame: context.frame,
|
|
568
|
+
signal: context.signal,
|
|
569
|
+
privacy: context.privacy,
|
|
570
|
+
}, event);
|
|
571
|
+
}
|
|
572
|
+
if (!context.session) {
|
|
573
|
+
// noinspection ExceptionCaughtLocallyJS -- Report failure
|
|
574
|
+
throw new Error('CDP 输入事件缺少 session');
|
|
575
|
+
}
|
|
576
|
+
await executeCdpEvent(context.session, event, context.humanize, context.timeout);
|
|
577
|
+
return undefined;
|
|
578
|
+
}
|
|
579
|
+
catch (error) {
|
|
580
|
+
if (context.mode === 'cdp' && error instanceof ElementNotFoundError) {
|
|
581
|
+
const target = error.context.target;
|
|
582
|
+
await enrichTargetError(error, context.unifiedSession, target, { timeout, deadline, frame: context.frame });
|
|
583
|
+
}
|
|
584
|
+
throw error;
|
|
585
|
+
}
|
|
462
586
|
}
|
|
463
587
|
/**
|
|
464
588
|
* 通过真实鼠标事件选中页面文本
|
|
@@ -470,55 +594,62 @@ async function executeInputEvent(context, event) {
|
|
|
470
594
|
* @returns 格式化标签名(如 "code"),若被替换文本在格式化节点内
|
|
471
595
|
*/
|
|
472
596
|
/**
|
|
473
|
-
*
|
|
474
|
-
* select/replace 事件用,保证选区建立前 activeElement 就是目标
|
|
597
|
+
* 命令与 editor 目标只定位并聚焦,不派发鼠标事件
|
|
475
598
|
*/
|
|
476
599
|
async function focusTargetForCommands(unifiedSession, target, timeout, frame) {
|
|
477
600
|
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
478
|
-
if ('x' in target || 'y' in target) {
|
|
479
|
-
await handleUnifiedClick({
|
|
480
|
-
unifiedSession,
|
|
481
|
-
mode: unifiedSession.getMode(),
|
|
482
|
-
humanize: false,
|
|
483
|
-
timeout: remainingInputTimeout(deadline),
|
|
484
|
-
frame,
|
|
485
|
-
}, { type: 'click', target });
|
|
486
|
-
return;
|
|
487
|
-
}
|
|
488
601
|
if (unifiedSession.getMode() === 'cdp') {
|
|
489
602
|
await focusCdpTarget(getSession(), target, remainingInputTimeout(deadline));
|
|
490
603
|
return;
|
|
491
604
|
}
|
|
492
605
|
const params = targetToFindParams(target);
|
|
493
|
-
const els = await unifiedSession.find(params.selector, params.text, params.xpath, remainingInputTimeout(deadline));
|
|
494
606
|
const nth0 = params.nth ?? 0;
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
607
|
+
const els = 'x' in target && 'y' in target
|
|
608
|
+
? await unifiedSession.findAtPoint(target.x, target.y, nth0, remainingInputTimeout(deadline))
|
|
609
|
+
: await unifiedSession.find(params.selector, params.text, params.xpath, remainingInputTimeout(deadline), params.semantic, nth0);
|
|
610
|
+
if (els.length === 0) {
|
|
611
|
+
throw new StructuredToolError(nth0 === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', `第 ${nth0} 个命令目标不存在: ${JSON.stringify(target)}`, '请检查 target、frame 和 target.nth,或先用 extract type=state 查看当前候选元素', await buildTargetDiagnosticContext(unifiedSession, target, null, nth0, remainingInputTimeout(deadline), '命令目标未找到', frame));
|
|
499
612
|
}
|
|
500
613
|
const focused = await unifiedSession.evaluate(`(() => {
|
|
501
|
-
|
|
614
|
+
if (Date.now() >= ${deadline}) throw new Error('input timeout before command focus')
|
|
615
|
+
const ref = window.__mcpElementMap?.[${JSON.stringify(els[0].refId)}]
|
|
502
616
|
const el = ref?.deref()
|
|
503
617
|
if (!(el instanceof HTMLElement)) {
|
|
504
|
-
|
|
618
|
+
return false
|
|
505
619
|
}
|
|
506
|
-
|
|
507
|
-
|
|
620
|
+
(${installDomHelpers.toString()})();
|
|
621
|
+
el.focus({preventScroll: true})
|
|
622
|
+
return globalThis.__mcpDomHelpers.activeElement() === el
|
|
508
623
|
})()`, undefined, remainingInputTimeout(deadline));
|
|
509
624
|
if (focused !== true) {
|
|
510
|
-
throw new StructuredToolError('TARGET_FOCUS_FAILED', '命令目标未成功聚焦', '请检查目标是否可编辑、未 disabled 并处于正确 frame', await buildTargetDiagnosticContext(unifiedSession, target,
|
|
625
|
+
throw new StructuredToolError('TARGET_FOCUS_FAILED', '命令目标未成功聚焦', '请检查目标是否可编辑、未 disabled 并处于正确 frame', await buildTargetDiagnosticContext(unifiedSession, target, null, nth0, remainingInputTimeout(deadline), '命令目标聚焦失败', frame));
|
|
511
626
|
}
|
|
512
627
|
}
|
|
513
628
|
export async function focusCdpTarget(session, target, timeout) {
|
|
514
|
-
const
|
|
515
|
-
const
|
|
629
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
630
|
+
const focus = `function() {
|
|
631
|
+
if (Date.now() >= ${deadline}) throw new Error('input timeout before command focus');
|
|
516
632
|
if (!(this instanceof HTMLElement)) return false;
|
|
633
|
+
(${installDomHelpers.toString()})();
|
|
517
634
|
this.focus({preventScroll: true});
|
|
518
|
-
return
|
|
519
|
-
}
|
|
635
|
+
return globalThis.__mcpDomHelpers.activeElement() === this;
|
|
636
|
+
}`;
|
|
637
|
+
const focused = 'x' in target && 'y' in target
|
|
638
|
+
? await session.evaluate(`(() => {
|
|
639
|
+
(${installDomHelpers.toString()})();
|
|
640
|
+
const el = globalThis.__mcpDomHelpers.deepElementFromPoint(${target.x}, ${target.y});
|
|
641
|
+
return el ? (${focus}).call(el) : null;
|
|
642
|
+
})()`, undefined, remainingInputTimeout(deadline))
|
|
643
|
+
: await session
|
|
644
|
+
.createLocator(target, { timeout: targetLookupTimeout(remainingInputTimeout(deadline)) })
|
|
645
|
+
.evaluateOn(focus);
|
|
646
|
+
if (focused === null) {
|
|
647
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', '目标坐标未命中元素', '请检查目标坐标是否在当前页面视口内', {
|
|
648
|
+
target,
|
|
649
|
+
});
|
|
650
|
+
}
|
|
520
651
|
if (!focused) {
|
|
521
|
-
throw new StructuredToolError('TARGET_FOCUS_FAILED', '目标元素未成功聚焦', '
|
|
652
|
+
throw new StructuredToolError('TARGET_FOCUS_FAILED', '目标元素未成功聚焦', '请检查目标是否可聚焦、未 disabled,并确认 target 指向当前页面元素', { target });
|
|
522
653
|
}
|
|
523
654
|
}
|
|
524
655
|
async function focusTargetIfNeeded(unifiedSession, session, mode, target, timeout, frame) {
|
|
@@ -544,24 +675,24 @@ async function focusTargetIfNeeded(unifiedSession, session, mode, target, timeou
|
|
|
544
675
|
return;
|
|
545
676
|
}
|
|
546
677
|
const params = targetToFindParams(target);
|
|
547
|
-
const els = await unifiedSession.find(params.selector, params.text, params.xpath, remainingInputTimeout(deadline));
|
|
548
678
|
const nth0 = params.nth ?? 0;
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
: `第 ${nth0} 个匹配元素不存在(共 ${els.length} 个)`, '请检查 target、frame 和 target.nth,或先用 extract type=state 查看当前候选元素', await buildTargetDiagnosticContext(unifiedSession, target, els.length, nth0, timeout, '聚焦目标未找到', frame));
|
|
679
|
+
const els = await unifiedSession.find(params.selector, params.text, params.xpath, remainingInputTimeout(deadline), params.semantic, nth0);
|
|
680
|
+
if (els.length === 0) {
|
|
681
|
+
throw new StructuredToolError(nth0 === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', `第 ${nth0} 个匹配元素不存在: ${JSON.stringify(target)}`, '请检查 target、frame 和 target.nth,或先用 extract type=state 查看当前候选元素', await buildTargetDiagnosticContext(unifiedSession, target, null, nth0, timeout, '聚焦目标未找到', frame));
|
|
553
682
|
}
|
|
554
683
|
remainingInputTimeout(deadline);
|
|
555
|
-
const result = await unifiedSession.actionableClick(els[
|
|
684
|
+
const result = await unifiedSession.actionableClick(els[0].refId);
|
|
556
685
|
if (!result.success) {
|
|
557
686
|
throw new StructuredToolError('ACTIONABILITY_FAILED', result.error || '目标元素无法聚焦', result.suggestions?.[0] ?? '请检查元素是否可见、未被遮挡并处于正确 frame', {
|
|
558
687
|
...result,
|
|
559
|
-
...(await buildTargetDiagnosticContext(unifiedSession, target,
|
|
688
|
+
...(await buildTargetDiagnosticContext(unifiedSession, target, null, nth0, timeout, result.error || '目标元素无法聚焦', frame)),
|
|
560
689
|
});
|
|
561
690
|
}
|
|
562
691
|
}
|
|
563
|
-
async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, occurrenceNth, scopeNth, scopeSelector, scopeText, scopeXpath, timeout) {
|
|
564
|
-
return await unifiedSession.evaluate(`function(findText, occurrenceNth, scopeNth, scopeTarget, scopeSelector, scopeText, scopeXpath) {
|
|
692
|
+
async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, occurrenceNth, scopeNth, scopeSelector, scopeText, scopeXpath, scopeSemantic, timeout) {
|
|
693
|
+
return await unifiedSession.evaluate(`function(findText, occurrenceNth, scopeNth, scopeTarget, scopeSelector, scopeText, scopeXpath, scopeSemantic) {
|
|
694
|
+
(${installDomHelpers.toString()})();
|
|
695
|
+
var helpers = globalThis.__mcpDomHelpers;
|
|
565
696
|
function selectorFor(el) {
|
|
566
697
|
if (!(el instanceof Element)) return null;
|
|
567
698
|
if (el.id) return '#' + CSS.escape(el.id);
|
|
@@ -615,22 +746,10 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
615
746
|
};
|
|
616
747
|
}
|
|
617
748
|
var root = document.body;
|
|
618
|
-
if (scopeXpath) {
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
var candidates = Array.from(document.querySelectorAll(scopeSelector));
|
|
623
|
-
if (scopeText) {
|
|
624
|
-
candidates = candidates.filter(function(candidate) {
|
|
625
|
-
return (candidate.textContent || '').includes(scopeText);
|
|
626
|
-
});
|
|
627
|
-
}
|
|
628
|
-
root = candidates[scopeNth];
|
|
629
|
-
} else if (scopeText) {
|
|
630
|
-
var all = Array.from(document.querySelectorAll('*')).filter(function(candidate) {
|
|
631
|
-
return (candidate.textContent || '').includes(scopeText);
|
|
632
|
-
});
|
|
633
|
-
root = all[scopeNth];
|
|
749
|
+
if (scopeXpath || scopeSelector || scopeText || scopeSemantic) {
|
|
750
|
+
root = helpers.query({
|
|
751
|
+
selector:scopeSelector, text:scopeText, xpath:scopeXpath, semantic:scopeSemantic
|
|
752
|
+
})[scopeNth];
|
|
634
753
|
}
|
|
635
754
|
var candidateElements = [];
|
|
636
755
|
if (root instanceof Element) {
|
|
@@ -638,10 +757,10 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
638
757
|
var candidateSelector =
|
|
639
758
|
'input, textarea, [contenteditable="true"], [role="textbox"], button, a';
|
|
640
759
|
candidateElements = candidateElements.concat(
|
|
641
|
-
|
|
760
|
+
helpers.query({selector:candidateSelector}, root).slice(0, 12)
|
|
642
761
|
);
|
|
643
762
|
}
|
|
644
|
-
var active =
|
|
763
|
+
var active = helpers.activeElement();
|
|
645
764
|
var activeIsPassword =
|
|
646
765
|
active instanceof HTMLInputElement &&
|
|
647
766
|
(active.type || 'text').toLowerCase() === 'password';
|
|
@@ -675,7 +794,7 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
675
794
|
},
|
|
676
795
|
candidates: candidateElements.map(summarizeElement).filter(Boolean)
|
|
677
796
|
};
|
|
678
|
-
}`, undefined, timeout, [findText, occurrenceNth, scopeNth, scopeTarget ?? null, scopeSelector, scopeText, scopeXpath]);
|
|
797
|
+
}`, undefined, timeout, [findText, occurrenceNth, scopeNth, scopeTarget ?? null, scopeSelector, scopeText, scopeXpath, scopeSemantic]);
|
|
679
798
|
}
|
|
680
799
|
async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeout, privacy) {
|
|
681
800
|
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
@@ -687,35 +806,27 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
687
806
|
let scopeSelector = null;
|
|
688
807
|
let scopeText = null;
|
|
689
808
|
let scopeXpath = null;
|
|
809
|
+
let scopeSemantic = null;
|
|
690
810
|
let scopeNth = 0;
|
|
691
811
|
if (scopeTarget && !('x' in scopeTarget) && !('y' in scopeTarget)) {
|
|
692
812
|
const params = targetToFindParams(scopeTarget);
|
|
693
813
|
scopeSelector = params.selector ?? null;
|
|
694
814
|
scopeText = params.text ?? null;
|
|
695
815
|
scopeXpath = params.xpath ?? null;
|
|
816
|
+
scopeSemantic = params.semantic ?? null;
|
|
696
817
|
scopeNth = params.nth ?? 0;
|
|
697
818
|
}
|
|
698
819
|
// Step 1: 注入脚本定位文本
|
|
699
820
|
//noinspection SpellCheckingInspection -- 模板中的坐标字段名
|
|
700
|
-
const result = await unifiedSession.evaluate(`function(findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath) {
|
|
821
|
+
const result = await unifiedSession.evaluate(`function(findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath, scopeSemantic) {
|
|
822
|
+
(${installDomHelpers.toString()})();
|
|
823
|
+
var helpers = globalThis.__mcpDomHelpers;
|
|
701
824
|
// 确定搜索根节点
|
|
702
825
|
var root = document.body;
|
|
703
|
-
if (scopeXpath) {
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
var candidates = Array.from(document.querySelectorAll(scopeSelector));
|
|
708
|
-
if (scopeText) {
|
|
709
|
-
candidates = candidates.filter(function(candidate) {
|
|
710
|
-
return (candidate.textContent || '').includes(scopeText);
|
|
711
|
-
});
|
|
712
|
-
}
|
|
713
|
-
root = candidates[scopeNth];
|
|
714
|
-
} else if (scopeText) {
|
|
715
|
-
var all = Array.from(document.querySelectorAll('*')).filter(function(candidate) {
|
|
716
|
-
return (candidate.textContent || '').includes(scopeText);
|
|
717
|
-
});
|
|
718
|
-
root = all[scopeNth];
|
|
826
|
+
if (scopeXpath || scopeSelector || scopeText || scopeSemantic) {
|
|
827
|
+
root = helpers.query({
|
|
828
|
+
selector:scopeSelector, text:scopeText, xpath:scopeXpath, semantic:scopeSemantic
|
|
829
|
+
})[scopeNth];
|
|
719
830
|
}
|
|
720
831
|
if (!root) return {type: 'noscope'};
|
|
721
832
|
|
|
@@ -747,7 +858,7 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
747
858
|
}
|
|
748
859
|
|
|
749
860
|
// 在子树中查找 input/textarea
|
|
750
|
-
var inputs =
|
|
861
|
+
var inputs = helpers.query({selector:'input, textarea'}, root);
|
|
751
862
|
for (var k = 0; k < inputs.length; k++) {
|
|
752
863
|
var inp = inputs[k];
|
|
753
864
|
var v = inp.value || '';
|
|
@@ -778,14 +889,20 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
778
889
|
// DOM 文本节点:TreeWalker 遍历(限制规模防止大 DOM 阻塞)
|
|
779
890
|
var MAX_TEXT_NODES = 10000;
|
|
780
891
|
var MAX_TEXT_LENGTH = 500000;
|
|
781
|
-
var
|
|
892
|
+
var pendingNodes = [root];
|
|
782
893
|
var textNodes = [];
|
|
783
894
|
var fullText = '';
|
|
784
|
-
|
|
785
|
-
while (node = walker.nextNode()) {
|
|
895
|
+
while (pendingNodes.length) {
|
|
786
896
|
if (textNodes.length >= MAX_TEXT_NODES || fullText.length >= MAX_TEXT_LENGTH) break;
|
|
787
|
-
|
|
788
|
-
|
|
897
|
+
var node = pendingNodes.pop();
|
|
898
|
+
if (node.nodeType === Node.TEXT_NODE) {
|
|
899
|
+
textNodes.push({node: node, start: fullText.length, length: node.textContent.length});
|
|
900
|
+
fullText += node.textContent;
|
|
901
|
+
} else {
|
|
902
|
+
var children = Array.from(node.childNodes);
|
|
903
|
+
for (var ci = children.length - 1; ci >= 0; --ci) pendingNodes.push(children[ci]);
|
|
904
|
+
if (node.shadowRoot) pendingNodes.push(node.shadowRoot);
|
|
905
|
+
}
|
|
789
906
|
}
|
|
790
907
|
|
|
791
908
|
// 查找第 nth 个匹配
|
|
@@ -848,7 +965,7 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
848
965
|
endY: er.y + er.height / 2,
|
|
849
966
|
formatted: formatted || undefined
|
|
850
967
|
};
|
|
851
|
-
}`, undefined, remainingInputTimeout(deadline), [findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath]);
|
|
968
|
+
}`, undefined, remainingInputTimeout(deadline), [findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath, scopeSemantic], { userAction: 'input' });
|
|
852
969
|
if (privacy &&
|
|
853
970
|
result &&
|
|
854
971
|
result.type !== 'noscope' &&
|
|
@@ -856,7 +973,7 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
856
973
|
privacy.sensitive = sensitiveBefore || ('inputType' in result && result.inputType === 'password');
|
|
857
974
|
}
|
|
858
975
|
if (!result || result.type === 'noscope') {
|
|
859
|
-
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeNth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
976
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeNth, scopeSelector, scopeText, scopeXpath, scopeSemantic, timeout);
|
|
860
977
|
throw new StructuredToolError('TEXT_SCOPE_NOT_FOUND', `未找到目标元素: ${JSON.stringify(scopeTarget)}`, '请检查 target 是否能定位到包含目标文本的元素,或先用 extract(state) 查看可交互元素', diagnostics);
|
|
861
978
|
}
|
|
862
979
|
if (result.type === 'unsupported') {
|
|
@@ -869,7 +986,7 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
869
986
|
});
|
|
870
987
|
}
|
|
871
988
|
if (result.type === 'notfound') {
|
|
872
|
-
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeNth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
989
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeNth, scopeSelector, scopeText, scopeXpath, scopeSemantic, timeout);
|
|
873
990
|
throw new StructuredToolError('TEXT_NOT_FOUND', scopeTarget
|
|
874
991
|
? `目标元素内未找到文本 "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`
|
|
875
992
|
: `未找到文本: "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`, '请检查 find 文本、nth 序号和 target 范围;context.candidates 提供了当前范围内的候选文本和 selector', diagnostics);
|
|
@@ -883,14 +1000,6 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
883
1000
|
}
|
|
884
1001
|
// DOM 文本节点:鼠标选择
|
|
885
1002
|
const coords = result;
|
|
886
|
-
// iframe 坐标修正(precise 模式需要视口绝对坐标)
|
|
887
|
-
const frameOffset = unifiedSession.getFrameOffset();
|
|
888
|
-
if (frameOffset && unifiedSession.getInputMode() !== 'stealth') {
|
|
889
|
-
coords.startX += frameOffset.x;
|
|
890
|
-
coords.startY += frameOffset.y;
|
|
891
|
-
coords.endX += frameOffset.x;
|
|
892
|
-
coords.endY += frameOffset.y;
|
|
893
|
-
}
|
|
894
1003
|
// Step 2: 模拟鼠标选择
|
|
895
1004
|
await unifiedSession.mouseMove(coords.startX, coords.startY, remainingInputTimeout(deadline));
|
|
896
1005
|
remainingInputTimeout(deadline);
|
|
@@ -949,6 +1058,14 @@ function validateEvent(event) {
|
|
|
949
1058
|
requiredEventParam(event, 'text');
|
|
950
1059
|
}
|
|
951
1060
|
break;
|
|
1061
|
+
case 'upload':
|
|
1062
|
+
if (!event.target ||
|
|
1063
|
+
'x' in event.target ||
|
|
1064
|
+
!Array.isArray(event.files) ||
|
|
1065
|
+
event.files.some((path) => typeof path !== 'string' || !path)) {
|
|
1066
|
+
throw new StructuredToolError('INVALID_ARGUMENT', 'upload 需要元素 target 和 files 路径数组,空数组用于清空选择', '请指定 input[type=file] 的 CSS、label、XPath 或其他元素定位', { actionExecuted: false, actionStatus: 'not_started' });
|
|
1067
|
+
}
|
|
1068
|
+
break;
|
|
952
1069
|
case 'select':
|
|
953
1070
|
if (!event.find) {
|
|
954
1071
|
requiredEventParam(event, 'find');
|
|
@@ -1007,6 +1124,7 @@ const unifiedInputHandlers = {
|
|
|
1007
1124
|
touchmove: handleUnifiedTouchMove,
|
|
1008
1125
|
touchend: handleUnifiedTouchEnd,
|
|
1009
1126
|
type: handleUnifiedType,
|
|
1127
|
+
upload: handleUnifiedUpload,
|
|
1010
1128
|
wait: handleUnifiedWait,
|
|
1011
1129
|
select: handleUnifiedSelect,
|
|
1012
1130
|
replace: handleUnifiedReplace,
|
|
@@ -1027,12 +1145,15 @@ async function handleUnifiedKeyDown({ unifiedSession, timeout, frame }, event) {
|
|
|
1027
1145
|
await focusTargetForCommands(unifiedSession, event.target, remainingInputTimeout(deadline), frame);
|
|
1028
1146
|
}
|
|
1029
1147
|
remainingInputTimeout(deadline);
|
|
1030
|
-
await unifiedSession.
|
|
1031
|
-
|
|
1148
|
+
const warning = await unifiedSession.prepareKeyboardFocus(remainingInputTimeout(deadline));
|
|
1149
|
+
await unifiedSession.keyDown(event.key, event.commands, remainingInputTimeout(deadline));
|
|
1150
|
+
return warning;
|
|
1032
1151
|
}
|
|
1033
|
-
async function handleUnifiedKeyUp({ unifiedSession }, event) {
|
|
1034
|
-
|
|
1035
|
-
|
|
1152
|
+
async function handleUnifiedKeyUp({ unifiedSession, timeout }, event) {
|
|
1153
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1154
|
+
const warning = await unifiedSession.prepareKeyboardFocus(remainingInputTimeout(deadline));
|
|
1155
|
+
await unifiedSession.keyUp(event.key, remainingInputTimeout(deadline));
|
|
1156
|
+
return warning;
|
|
1036
1157
|
}
|
|
1037
1158
|
async function handleUnifiedClick({ unifiedSession, humanize, timeout, frame }, event) {
|
|
1038
1159
|
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
@@ -1050,16 +1171,17 @@ async function handleUnifiedClick({ unifiedSession, humanize, timeout, frame },
|
|
|
1050
1171
|
await randomDelay(Math.min(getMouseMoveDelay(), remaining), Math.min(getMouseMoveDelay() * 2, remaining));
|
|
1051
1172
|
}
|
|
1052
1173
|
}
|
|
1053
|
-
else {
|
|
1174
|
+
else if (!point.refId || button !== 'left' || clickCount !== 1) {
|
|
1054
1175
|
await unifiedSession.mouseMove(point.x, point.y, remainingInputTimeout(deadline));
|
|
1055
1176
|
}
|
|
1056
1177
|
remainingInputTimeout(deadline);
|
|
1057
1178
|
if (point.refId) {
|
|
1058
|
-
const result = await unifiedSession.actionableClick(point.refId, event.force === true, button, clickCount);
|
|
1179
|
+
const result = await unifiedSession.actionableClick(point.refId, event.force === true, button, clickCount, point);
|
|
1059
1180
|
if (!result.success) {
|
|
1060
1181
|
throw new StructuredToolError('ACTIONABILITY_FAILED', result.error || 'Click failed', result.suggestions?.[0] ??
|
|
1061
1182
|
'请根据 context.rect、context.clickPoint 和 context.coveringElement 调整 target', result);
|
|
1062
1183
|
}
|
|
1184
|
+
return result.warning;
|
|
1063
1185
|
}
|
|
1064
1186
|
else {
|
|
1065
1187
|
await unifiedSession.mouseClick(button, clickCount);
|
|
@@ -1099,29 +1221,31 @@ async function handleUnifiedMouseMove({ unifiedSession, humanize, timeout, frame
|
|
|
1099
1221
|
}
|
|
1100
1222
|
async function handleUnifiedWheel({ unifiedSession, timeout, frame }, event) {
|
|
1101
1223
|
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1224
|
+
let point;
|
|
1225
|
+
if (event.target && 'x' in event.target && 'y' in event.target) {
|
|
1226
|
+
point = { x: event.target.x, y: event.target.y };
|
|
1227
|
+
}
|
|
1228
|
+
else if (event.target) {
|
|
1229
|
+
const { selector, text, xpath, nth: nthParam, semantic, } = targetToFindParams(event.target);
|
|
1105
1230
|
const nth = nthParam ?? 0;
|
|
1106
|
-
|
|
1231
|
+
const elements = await unifiedSession.find(selector, text, xpath, remainingInputTimeout(deadline), semantic, nth);
|
|
1232
|
+
if (elements.length > 0) {
|
|
1107
1233
|
remainingInputTimeout(deadline);
|
|
1108
|
-
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[
|
|
1234
|
+
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[0].refId);
|
|
1109
1235
|
return undefined;
|
|
1110
1236
|
}
|
|
1111
|
-
|
|
1112
|
-
await unifiedSession.mouseMove(point.x, point.y, remainingInputTimeout(deadline));
|
|
1237
|
+
point = await getTargetPointExtension(unifiedSession, event.target, remainingInputTimeout(deadline), frame, true);
|
|
1113
1238
|
}
|
|
1114
|
-
remainingInputTimeout(deadline);
|
|
1115
|
-
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
1239
|
+
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0, remainingInputTimeout(deadline), point);
|
|
1116
1240
|
return undefined;
|
|
1117
1241
|
}
|
|
1118
1242
|
async function handleUnifiedTouchStart({ unifiedSession, timeout, frame }, event) {
|
|
1119
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
1243
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame, true);
|
|
1120
1244
|
await unifiedSession.touchStart(point.x, point.y);
|
|
1121
1245
|
return undefined;
|
|
1122
1246
|
}
|
|
1123
1247
|
async function handleUnifiedTouchMove({ unifiedSession, timeout, frame }, event) {
|
|
1124
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
1248
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame, true);
|
|
1125
1249
|
await unifiedSession.touchMove(point.x, point.y);
|
|
1126
1250
|
return undefined;
|
|
1127
1251
|
}
|
|
@@ -1129,6 +1253,117 @@ async function handleUnifiedTouchEnd({ unifiedSession }) {
|
|
|
1129
1253
|
await unifiedSession.touchEnd();
|
|
1130
1254
|
return undefined;
|
|
1131
1255
|
}
|
|
1256
|
+
async function handleUnifiedUpload({ unifiedSession, session, mode, timeout, signal, privacy }, event) {
|
|
1257
|
+
if (privacy) {
|
|
1258
|
+
privacy.sensitive = true;
|
|
1259
|
+
}
|
|
1260
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1261
|
+
const key = `__mcpUpload_${randomUUID()}`;
|
|
1262
|
+
let objectId;
|
|
1263
|
+
let preparationStarted = false;
|
|
1264
|
+
let dispatched = false;
|
|
1265
|
+
const active = () => {
|
|
1266
|
+
signal?.throwIfAborted();
|
|
1267
|
+
unifiedSession.assertRequestActive();
|
|
1268
|
+
return remainingInputTimeout(deadline);
|
|
1269
|
+
};
|
|
1270
|
+
const invoke = async (operation, refId = null, payloads = []) => {
|
|
1271
|
+
const args = [operation, key, refId, deadline, event.files.length, payloads];
|
|
1272
|
+
const budget = active();
|
|
1273
|
+
if (operation === 'commit') {
|
|
1274
|
+
dispatched = true;
|
|
1275
|
+
}
|
|
1276
|
+
if (mode !== 'cdp') {
|
|
1277
|
+
return unifiedSession.evaluate(uploadInPage.toString(), undefined, budget, args, {
|
|
1278
|
+
userAction: operation === 'commit' ? 'input' : undefined,
|
|
1279
|
+
});
|
|
1280
|
+
}
|
|
1281
|
+
const response = await session.send('Runtime.callFunctionOn', {
|
|
1282
|
+
objectId,
|
|
1283
|
+
functionDeclaration: uploadInPage.toString(),
|
|
1284
|
+
arguments: args.map((value) => ({ value })),
|
|
1285
|
+
returnByValue: true,
|
|
1286
|
+
awaitPromise: true,
|
|
1287
|
+
}, budget, operation === 'commit' ? 'input' : undefined);
|
|
1288
|
+
if (response.exceptionDetails || !response.result?.value) {
|
|
1289
|
+
throw new Error('上传脚本没有返回执行状态');
|
|
1290
|
+
}
|
|
1291
|
+
return response.result.value;
|
|
1292
|
+
};
|
|
1293
|
+
const checked = (result) => {
|
|
1294
|
+
if (!result.success) {
|
|
1295
|
+
throw new StructuredToolError('UPLOAD_FAILED', result.error ?? '上传失败', '请核对目标控件和页面状态后重新操作', { actionExecuted: result.actionExecuted, actionStatus: result.actionStatus, retryable: false });
|
|
1296
|
+
}
|
|
1297
|
+
return result;
|
|
1298
|
+
};
|
|
1299
|
+
try {
|
|
1300
|
+
active();
|
|
1301
|
+
let refId = null;
|
|
1302
|
+
if (mode === 'cdp') {
|
|
1303
|
+
const nodeId = await session
|
|
1304
|
+
.createLocator(event.target, { timeout: targetLookupTimeout(active()) })
|
|
1305
|
+
.find();
|
|
1306
|
+
const resolved = await session.send('DOM.resolveNode', { nodeId }, active());
|
|
1307
|
+
objectId = resolved.object.objectId;
|
|
1308
|
+
if (!objectId) {
|
|
1309
|
+
// noinspection ExceptionCaughtLocallyJS
|
|
1310
|
+
throw new Error('上传目标引用已失效');
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
else {
|
|
1314
|
+
const params = targetToFindParams(event.target);
|
|
1315
|
+
const elements = await unifiedSession.find(params.selector, params.text, params.xpath, active(), params.semantic, params.nth ?? 0);
|
|
1316
|
+
if (!elements.length) {
|
|
1317
|
+
// noinspection ExceptionCaughtLocallyJS
|
|
1318
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', '未找到上传控件', '请检查 target、frame 和 target.nth', {});
|
|
1319
|
+
}
|
|
1320
|
+
refId = elements[0].refId;
|
|
1321
|
+
}
|
|
1322
|
+
// evaluate 要求受控页面,读取本地文件前确认写权限并固定目标
|
|
1323
|
+
preparationStarted = true;
|
|
1324
|
+
checked(await invoke('prepare', refId));
|
|
1325
|
+
const payloads = await prepareUploadFiles(event.files, deadline, signal);
|
|
1326
|
+
active();
|
|
1327
|
+
const result = checked(await invoke('commit', null, payloads));
|
|
1328
|
+
return { type: 'upload', files: result.files };
|
|
1329
|
+
}
|
|
1330
|
+
catch (error) {
|
|
1331
|
+
const explicit = error instanceof StructuredToolError || error instanceof BrowserError ? error.context : undefined;
|
|
1332
|
+
const actionExecuted = dispatched && explicit?.actionExecuted !== false;
|
|
1333
|
+
throw new StructuredToolError(error instanceof StructuredToolError
|
|
1334
|
+
? error.code
|
|
1335
|
+
: actionExecuted
|
|
1336
|
+
? 'UPLOAD_RESULT_UNKNOWN'
|
|
1337
|
+
: 'UPLOAD_PREPARATION_FAILED', actionExecuted && explicit?.actionStatus === undefined
|
|
1338
|
+
? '上传提交回执未能取得,请核对页面或站点收件结果'
|
|
1339
|
+
: error instanceof Error
|
|
1340
|
+
? error.message
|
|
1341
|
+
: '上传准备失败', actionExecuted ? '请检查页面实际状态,勿自动重发上传' : '请检查目标、文件和连接后重试', {
|
|
1342
|
+
...explicit,
|
|
1343
|
+
actionExecuted,
|
|
1344
|
+
actionStatus: actionExecuted ? (explicit?.actionStatus ?? 'unknown') : 'not_started',
|
|
1345
|
+
retryable: false,
|
|
1346
|
+
});
|
|
1347
|
+
}
|
|
1348
|
+
finally {
|
|
1349
|
+
try {
|
|
1350
|
+
if (preparationStarted && Date.now() < deadline && !signal?.aborted) {
|
|
1351
|
+
await invoke('cleanup');
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
catch {
|
|
1355
|
+
console.warn('[MCP] Upload target cleanup deferred to page deadline');
|
|
1356
|
+
}
|
|
1357
|
+
if (objectId) {
|
|
1358
|
+
try {
|
|
1359
|
+
await session.send('Runtime.releaseObject', { objectId }, 1000);
|
|
1360
|
+
}
|
|
1361
|
+
catch {
|
|
1362
|
+
console.warn('[MCP] Upload remote object cleanup failed after context loss');
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1132
1367
|
async function handleUnifiedType(context, event) {
|
|
1133
1368
|
const { unifiedSession, humanize, timeout, frame } = context;
|
|
1134
1369
|
if (event.mode === 'controlled' || event.dispatch) {
|
|
@@ -1143,9 +1378,11 @@ async function handleUnifiedType(context, event) {
|
|
|
1143
1378
|
}
|
|
1144
1379
|
}
|
|
1145
1380
|
else {
|
|
1146
|
-
const hasActiveFocus = await unifiedSession.evaluate(
|
|
1147
|
-
|
|
1148
|
-
|
|
1381
|
+
const hasActiveFocus = await unifiedSession.evaluate(`(() => {
|
|
1382
|
+
(${installDomHelpers.toString()})();
|
|
1383
|
+
const active = globalThis.__mcpDomHelpers.activeElement();
|
|
1384
|
+
return !!active && active !== document.body && active !== document.documentElement;
|
|
1385
|
+
})()`, undefined, remainingInputTimeout(deadline));
|
|
1149
1386
|
if (!hasActiveFocus) {
|
|
1150
1387
|
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
1151
1388
|
}
|
|
@@ -1171,7 +1408,7 @@ function ensureWaitFitsTimeout(ms, timeout) {
|
|
|
1171
1408
|
}
|
|
1172
1409
|
async function handleUnifiedWait({ timeout }, event) {
|
|
1173
1410
|
ensureWaitFitsTimeout(event.ms, timeout);
|
|
1174
|
-
await
|
|
1411
|
+
await getUnifiedSession().wait(event.ms);
|
|
1175
1412
|
return undefined;
|
|
1176
1413
|
}
|
|
1177
1414
|
async function handleUnifiedSelect({ unifiedSession, session, mode, timeout, frame, privacy }, event) {
|
|
@@ -1182,7 +1419,8 @@ async function handleUnifiedSelect({ unifiedSession, session, mode, timeout, fra
|
|
|
1182
1419
|
}
|
|
1183
1420
|
await focusTargetIfNeeded(unifiedSession, session, mode, event.target, remainingInputTimeout(deadline), frame);
|
|
1184
1421
|
const selectionTarget = await unifiedSession.evaluate(`(() => {
|
|
1185
|
-
|
|
1422
|
+
(${installDomHelpers.toString()})();
|
|
1423
|
+
const el = globalThis.__mcpDomHelpers.activeElement();
|
|
1186
1424
|
if (!(el instanceof HTMLInputElement)) return null;
|
|
1187
1425
|
return {
|
|
1188
1426
|
tag: 'input',
|
|
@@ -1213,10 +1451,13 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1213
1451
|
}
|
|
1214
1452
|
await focusTargetIfNeeded(unifiedSession, session, mode, event.target, remainingInputTimeout(deadline), frame);
|
|
1215
1453
|
const directResult = await unifiedSession.evaluate(`function(findText, replacementText, nth) {
|
|
1216
|
-
|
|
1454
|
+
(${installDomHelpers.toString()})();
|
|
1455
|
+
var el = globalThis.__mcpDomHelpers.activeElement();
|
|
1217
1456
|
if (!(el instanceof HTMLInputElement)) return {handled: false};
|
|
1218
1457
|
var type = (el.type || 'text').toLowerCase();
|
|
1219
|
-
if (['text', 'search', 'tel', 'url', 'password'].indexOf(type) !== -1)
|
|
1458
|
+
if (['text', 'search', 'tel', 'url', 'password'].indexOf(type) !== -1) {
|
|
1459
|
+
return {handled: false, inputType: type};
|
|
1460
|
+
}
|
|
1220
1461
|
var current = String(el.value || '');
|
|
1221
1462
|
var pos = -1;
|
|
1222
1463
|
for (var i = 0; i <= nth; i++) {
|
|
@@ -1256,7 +1497,7 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1256
1497
|
};
|
|
1257
1498
|
}
|
|
1258
1499
|
setter.call(el, requested);
|
|
1259
|
-
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
1500
|
+
el.dispatchEvent(new Event('input', {bubbles: true, composed: true}));
|
|
1260
1501
|
el.dispatchEvent(new Event('change', {bubbles: true}));
|
|
1261
1502
|
var actual = String(el.value || '');
|
|
1262
1503
|
return {
|
|
@@ -1269,7 +1510,7 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1269
1510
|
actualValue: actual.slice(0, 160),
|
|
1270
1511
|
reason: actual === requested ? undefined : 'value_not_applied'
|
|
1271
1512
|
};
|
|
1272
|
-
}`, undefined, remainingInputTimeout(deadline), [event.find, event.text, event.nth ?? 0]);
|
|
1513
|
+
}`, undefined, remainingInputTimeout(deadline), [event.find, event.text, event.nth ?? 0], { userAction: 'input' });
|
|
1273
1514
|
if (privacy) {
|
|
1274
1515
|
privacy.sensitive = sensitiveBefore || directResult.inputType === 'password';
|
|
1275
1516
|
}
|
|
@@ -1309,7 +1550,8 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1309
1550
|
let selectionConfirmed = false;
|
|
1310
1551
|
for (let i = 0; i < 25; i++) {
|
|
1311
1552
|
const hasSelection = await unifiedSession.evaluate(`(function() {
|
|
1312
|
-
|
|
1553
|
+
(${installDomHelpers.toString()})();
|
|
1554
|
+
var el = globalThis.__mcpDomHelpers.activeElement();
|
|
1313
1555
|
if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA')) {
|
|
1314
1556
|
return el.selectionStart !== el.selectionEnd;
|
|
1315
1557
|
}
|
|
@@ -1326,11 +1568,12 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1326
1568
|
throw new Error(`选区同步失败:文本 "${event.find}" 已定位但未能建立选区,无法执行替换`);
|
|
1327
1569
|
}
|
|
1328
1570
|
const replaceResult = await unifiedSession.evaluate(`function(replacementText) {
|
|
1329
|
-
|
|
1571
|
+
(${installDomHelpers.toString()})();
|
|
1572
|
+
var el = globalThis.__mcpDomHelpers.activeElement();
|
|
1330
1573
|
if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') && el.setRangeText) {
|
|
1331
1574
|
if (el.readOnly || el.disabled) return 'readonly';
|
|
1332
1575
|
el.setRangeText(replacementText, el.selectionStart, el.selectionEnd, 'end');
|
|
1333
|
-
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
1576
|
+
el.dispatchEvent(new Event('input', {bubbles: true, composed: true}));
|
|
1334
1577
|
return 'ok';
|
|
1335
1578
|
}
|
|
1336
1579
|
var sel = window.getSelection();
|
|
@@ -1343,7 +1586,7 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1343
1586
|
if (!editable || !editable.isContentEditable) return 'readonly';
|
|
1344
1587
|
if (document.execCommand('insertText', false, replacementText)) return 'ok';
|
|
1345
1588
|
return 'fallback';
|
|
1346
|
-
}`, undefined, remainingInputTimeout(deadline), [event.text]);
|
|
1589
|
+
}`, undefined, remainingInputTimeout(deadline), [event.text], { userAction: 'input' });
|
|
1347
1590
|
if (replaceResult === 'readonly') {
|
|
1348
1591
|
throw new Error(`目标元素不可编辑,已选中文本 "${event.find}" 但无法替换`);
|
|
1349
1592
|
}
|
|
@@ -1384,15 +1627,15 @@ async function handleUnifiedDrag({ unifiedSession, timeout, frame }, event) {
|
|
|
1384
1627
|
const srcNth = srcParams.nth ?? 0;
|
|
1385
1628
|
const dstNth = dstParams.nth ?? 0;
|
|
1386
1629
|
const attemptDrag = async () => {
|
|
1387
|
-
const srcEls = await unifiedSession.find(srcParams.selector, srcParams.text, srcParams.xpath, timeout);
|
|
1388
|
-
const dstEls = await unifiedSession.find(dstParams.selector, dstParams.text, dstParams.xpath, timeout);
|
|
1389
|
-
if (srcEls.length
|
|
1390
|
-
throw new StructuredToolError(
|
|
1630
|
+
const srcEls = await unifiedSession.find(srcParams.selector, srcParams.text, srcParams.xpath, timeout, srcParams.semantic, srcNth);
|
|
1631
|
+
const dstEls = await unifiedSession.find(dstParams.selector, dstParams.text, dstParams.xpath, timeout, dstParams.semantic, dstNth);
|
|
1632
|
+
if (srcEls.length === 0) {
|
|
1633
|
+
throw new StructuredToolError(srcNth === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', `drag 源元素未找到: ${JSON.stringify(event.target)}`, '请检查 drag 源 target、frame 和 nth', await buildTargetDiagnosticContext(unifiedSession, event.target, null, srcNth, timeout, 'drag 源元素未找到', frame));
|
|
1391
1634
|
}
|
|
1392
|
-
if (dstEls.length
|
|
1393
|
-
throw new StructuredToolError(
|
|
1635
|
+
if (dstEls.length === 0) {
|
|
1636
|
+
throw new StructuredToolError(dstNth === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', `drag 目标元素未找到: ${JSON.stringify(event.to)}`, '请检查 drag 目标 to、frame 和 nth', await buildTargetDiagnosticContext(unifiedSession, event.to, null, dstNth, timeout, 'drag 目标元素未找到', frame));
|
|
1394
1637
|
}
|
|
1395
|
-
return unifiedSession.dragAndDrop(srcEls[
|
|
1638
|
+
return unifiedSession.dragAndDrop(srcEls[0].refId, dstEls[0].refId);
|
|
1396
1639
|
};
|
|
1397
1640
|
let dragResult = await attemptDrag();
|
|
1398
1641
|
let retried = false;
|
|
@@ -1424,15 +1667,15 @@ async function inputControlled(unifiedSession, event, timeout, frame) {
|
|
|
1424
1667
|
if ('x' in event.target && 'y' in event.target) {
|
|
1425
1668
|
throw new Error('controlled 输入不支持坐标型 target,请使用 CSS 选择器、role 或文本定位');
|
|
1426
1669
|
}
|
|
1427
|
-
const { selector, text: searchText, xpath, nth: nthParam, } = targetToFindParams(event.target);
|
|
1428
|
-
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
1670
|
+
const { selector, text: searchText, xpath, nth: nthParam, semantic, } = targetToFindParams(event.target);
|
|
1429
1671
|
const nth = nthParam ?? 0;
|
|
1430
|
-
|
|
1431
|
-
|
|
1672
|
+
const elements = await unifiedSession.find(selector, searchText, xpath, timeout, semantic, nth);
|
|
1673
|
+
if (elements.length === 0) {
|
|
1674
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', `controlled 输入目标未找到: ${JSON.stringify(event.target)}`, '请检查 target 是否能定位到 input、textarea、select 或 contenteditable 元素;context.candidates 提供当前页面候选元素', await buildTargetDiagnosticContext(unifiedSession, event.target, null, nth, timeout, 'controlled 输入目标未找到', frame));
|
|
1432
1675
|
}
|
|
1433
|
-
const result = await unifiedSession.dispatchInput(elements[
|
|
1676
|
+
const result = await unifiedSession.dispatchInput(elements[0].refId, event.text ?? '');
|
|
1434
1677
|
if (!result.success) {
|
|
1435
|
-
throw new StructuredToolError('CONTROLLED_INPUT_FAILED', result.error || 'controlled 输入失败', '请检查目标元素是否可编辑、未 disabled,并确认当前 frame 与 target 匹配', await buildTargetDiagnosticContext(unifiedSession, event.target,
|
|
1678
|
+
throw new StructuredToolError('CONTROLLED_INPUT_FAILED', result.error || 'controlled 输入失败', '请检查目标元素是否可编辑、未 disabled,并确认当前 frame 与 target 匹配', await buildTargetDiagnosticContext(unifiedSession, event.target, null, nth, timeout, result.error || 'controlled 输入失败', frame));
|
|
1436
1679
|
}
|
|
1437
1680
|
}
|
|
1438
1681
|
async function buildTargetDiagnosticContext(unifiedSession, target, matchCount, nth, timeout, reason = '目标元素未找到', frame) {
|
|
@@ -1445,11 +1688,13 @@ async function buildTargetDiagnosticContext(unifiedSession, target, matchCount,
|
|
|
1445
1688
|
});
|
|
1446
1689
|
}
|
|
1447
1690
|
async function editorAction(unifiedSession, action, event, timeout, frame) {
|
|
1691
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1448
1692
|
if (event.target) {
|
|
1449
|
-
await focusTargetForCommands(unifiedSession, event.target,
|
|
1693
|
+
await focusTargetForCommands(unifiedSession, event.target, remainingInputTimeout(deadline), frame);
|
|
1450
1694
|
}
|
|
1451
1695
|
return unifiedSession.evaluate(`function(action, text, command) {
|
|
1452
|
-
|
|
1696
|
+
(${installDomHelpers.toString()})();
|
|
1697
|
+
var active = globalThis.__mcpDomHelpers.activeElement();
|
|
1453
1698
|
var sel = window.getSelection();
|
|
1454
1699
|
var editable = active;
|
|
1455
1700
|
if (!editable || editable === document.body || editable === document.documentElement) {
|
|
@@ -1495,7 +1740,7 @@ async function editorAction(unifiedSession, action, event, timeout, frame) {
|
|
|
1495
1740
|
if (action === 'insert') {
|
|
1496
1741
|
if (isInput) {
|
|
1497
1742
|
active.setRangeText(text, active.selectionStart || 0, active.selectionEnd || 0, 'end');
|
|
1498
|
-
active.dispatchEvent(new Event('input', {bubbles: true}));
|
|
1743
|
+
active.dispatchEvent(new Event('input', {bubbles: true, composed: true}));
|
|
1499
1744
|
active.dispatchEvent(new Event('change', {bubbles: true}));
|
|
1500
1745
|
return {success: true};
|
|
1501
1746
|
}
|
|
@@ -1509,7 +1754,7 @@ async function editorAction(unifiedSession, action, event, timeout, frame) {
|
|
|
1509
1754
|
throw new Error('editorCommand 执行失败: ' + command);
|
|
1510
1755
|
}
|
|
1511
1756
|
return {success: true, command: command};
|
|
1512
|
-
}`, undefined,
|
|
1757
|
+
}`, undefined, remainingInputTimeout(deadline), [action, event.text ?? '', event.command ?? ''], { userAction: 'input' });
|
|
1513
1758
|
}
|
|
1514
1759
|
async function getTargetPointExtension(unifiedSession, target, timeout, frame, scroll = true) {
|
|
1515
1760
|
const budget = timeout ?? DEFAULT_TIMEOUT;
|
|
@@ -1521,66 +1766,162 @@ async function getTargetPointExtension(unifiedSession, target, timeout, frame, s
|
|
|
1521
1766
|
}
|
|
1522
1767
|
return value;
|
|
1523
1768
|
};
|
|
1524
|
-
const frameOffset = unifiedSession.getFrameOffset();
|
|
1525
|
-
const isStealth = unifiedSession.getInputMode() === 'stealth';
|
|
1526
|
-
// 原始坐标:用户意图为 iframe 相对坐标
|
|
1527
1769
|
if ('x' in target && 'y' in target) {
|
|
1528
|
-
if (frameOffset && !isStealth) {
|
|
1529
|
-
// precise: 转为视口绝对坐标
|
|
1530
|
-
return { x: target.x + frameOffset.x, y: target.y + frameOffset.y };
|
|
1531
|
-
}
|
|
1532
1770
|
return { x: target.x, y: target.y };
|
|
1533
1771
|
}
|
|
1534
|
-
const { selector, text, xpath, nth: nthParam } = targetToFindParams(target);
|
|
1535
|
-
const elements = await unifiedSession.find(selector, text, xpath, remaining());
|
|
1772
|
+
const { selector, text, xpath, nth: nthParam, semantic } = targetToFindParams(target);
|
|
1536
1773
|
const nth = nthParam ?? 0;
|
|
1774
|
+
const elements = await unifiedSession.find(selector, text, xpath, remaining(), semantic, nth);
|
|
1537
1775
|
if (elements.length === 0) {
|
|
1538
|
-
throw new StructuredToolError('TARGET_NOT_FOUND', `未找到目标元素: ${JSON.stringify(target)}`, '请检查 target 是否正确,或先用 extract type=state 查看 interactiveElements 候选', await buildTargetDiagnosticContext(unifiedSession, target,
|
|
1539
|
-
}
|
|
1540
|
-
if (nth >= elements.length) {
|
|
1541
|
-
throw new StructuredToolError('TARGET_INDEX_OUT_OF_RANGE', `第 ${nth} 个匹配元素不存在(共 ${elements.length} 个)`, '请降低 nth,或先用 extract type=state 查看当前匹配到的候选元素', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout, '目标元素序号越界', frame));
|
|
1776
|
+
throw new StructuredToolError(nth === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', `未找到目标元素: ${JSON.stringify(target)}`, '请检查 target 是否正确,或先用 extract type=state 查看 interactiveElements 候选', await buildTargetDiagnosticContext(unifiedSession, target, null, nth, timeout, '目标元素未找到', frame));
|
|
1542
1777
|
}
|
|
1543
1778
|
// 视口外时滚动后重新取 rect:与 actionableClick (left+single) 行为对齐,
|
|
1544
1779
|
// 否则非左键 / 多击的坐标路径在视口外坐标 dispatch,浏览器找不到元素,事件丢失
|
|
1545
|
-
const refId = elements[
|
|
1546
|
-
|
|
1780
|
+
const refId = elements[0].refId;
|
|
1781
|
+
let rect = await unifiedSession.evaluate(`(async () => {
|
|
1547
1782
|
const ref = window.__mcpElementMap?.[${JSON.stringify(refId)}];
|
|
1548
1783
|
const el = ref?.deref();
|
|
1549
|
-
if (!el) return null;
|
|
1784
|
+
if (!el || !el.isConnected || el.ownerDocument !== document) return null;
|
|
1550
1785
|
const r = el.getBoundingClientRect();
|
|
1551
|
-
if (${scroll} && (r.top < 0 || r.bottom > window.innerHeight ||
|
|
1786
|
+
if (${scroll} && (r.top < 0 || r.bottom > window.innerHeight ||
|
|
1787
|
+
r.left < 0 || r.right > window.innerWidth)) {
|
|
1788
|
+
const positions = [];
|
|
1789
|
+
for (let node = el; node; node = node.assignedSlot || node.parentElement || node.getRootNode?.().host) {
|
|
1790
|
+
if (node !== document.scrollingElement) {
|
|
1791
|
+
positions.push({ target: node, x: node.scrollLeft, y: node.scrollTop });
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
positions.push({ target: document, x: window.scrollX, y: window.scrollY });
|
|
1552
1795
|
el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
|
|
1796
|
+
const changed = positions.filter(({ target, x, y }) => target === document
|
|
1797
|
+
? x !== window.scrollX || y !== window.scrollY
|
|
1798
|
+
: x !== target.scrollLeft || y !== target.scrollTop);
|
|
1799
|
+
if (changed.length) {
|
|
1800
|
+
const channel = new MessageChannel();
|
|
1801
|
+
const scrollTargets = new Set([window, document, ...positions.flatMap(({ target }) =>
|
|
1802
|
+
[target, target.getRootNode?.() || document]),
|
|
1803
|
+
...globalThis.__mcpDomHelpers.elements().flatMap(node => node.shadowRoot ? [node.shadowRoot] : [])]);
|
|
1804
|
+
let timer, frame, visibilityChanged, scrollChanged, scrollEnded;
|
|
1805
|
+
try {
|
|
1806
|
+
await new Promise((resolve, reject) => {
|
|
1807
|
+
let turn = 0, previous;
|
|
1808
|
+
const scrollPosition = target => target === document
|
|
1809
|
+
? [window.scrollX, window.scrollY] : [target.scrollLeft, target.scrollTop];
|
|
1810
|
+
const observedScroll = new Map();
|
|
1811
|
+
scrollChanged = event => {
|
|
1812
|
+
const { target } = event, last = observedScroll.get(target);
|
|
1813
|
+
if (last?.event === event && last.listenerTarget !== event.currentTarget) return;
|
|
1814
|
+
if (!last || positions.some(position => position.target === target)) previous = undefined;
|
|
1815
|
+
const offset = scrollPosition(target);
|
|
1816
|
+
const offsets = last?.offsets || new Set();
|
|
1817
|
+
offsets.add(offset.join(','));
|
|
1818
|
+
observedScroll.set(target, { event, listenerTarget: event.currentTarget, offset, offsets });
|
|
1819
|
+
};
|
|
1820
|
+
scrollEnded = ({ target }) => {
|
|
1821
|
+
const last = observedScroll.get(target), offset = scrollPosition(target);
|
|
1822
|
+
if (last && offset.some((value, index) => value !== last.offset[index]) &&
|
|
1823
|
+
(positions.some(position => position.target === target) || !last.offsets.has(offset.join(',')))) {
|
|
1824
|
+
previous = undefined;
|
|
1825
|
+
}
|
|
1826
|
+
};
|
|
1827
|
+
const observeScroll = target => {
|
|
1828
|
+
target.addEventListener('scroll', scrollChanged, true);
|
|
1829
|
+
target.addEventListener('scrollend', scrollEnded, true);
|
|
1830
|
+
};
|
|
1831
|
+
for (const target of scrollTargets) observeScroll(target);
|
|
1832
|
+
const schedule = () => {
|
|
1833
|
+
const currentTurn = ++turn;
|
|
1834
|
+
if (frame !== undefined) cancelAnimationFrame(frame);
|
|
1835
|
+
frame = undefined;
|
|
1836
|
+
if (document.visibilityState === 'visible') {
|
|
1837
|
+
frame = requestAnimationFrame(() => {
|
|
1838
|
+
channel.port2.postMessage({ turn: currentTurn, rendered: true });
|
|
1839
|
+
});
|
|
1840
|
+
} else {
|
|
1841
|
+
channel.port2.postMessage({ turn: currentTurn, rendered: false });
|
|
1842
|
+
}
|
|
1843
|
+
};
|
|
1844
|
+
channel.port1.onmessage = ({ data }) => {
|
|
1845
|
+
if (data.turn !== turn) return;
|
|
1846
|
+
for (const node of globalThis.__mcpDomHelpers.elements()) {
|
|
1847
|
+
if (node.shadowRoot && !scrollTargets.has(node.shadowRoot)) {
|
|
1848
|
+
scrollTargets.add(node.shadowRoot);
|
|
1849
|
+
observeScroll(node.shadowRoot);
|
|
1850
|
+
previous = undefined;
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
if (document.visibilityState === 'visible') {
|
|
1854
|
+
if (!data.rendered) {
|
|
1855
|
+
schedule();
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
const bounds = el.getBoundingClientRect();
|
|
1859
|
+
const current = [...positions.flatMap(({ target }) => scrollPosition(target)),
|
|
1860
|
+
bounds.x, bounds.y, bounds.width, bounds.height];
|
|
1861
|
+
if (!previous || !current.every((value, index) => value === previous[index])) {
|
|
1862
|
+
previous = current;
|
|
1863
|
+
schedule();
|
|
1864
|
+
return;
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
resolve();
|
|
1868
|
+
};
|
|
1869
|
+
visibilityChanged = schedule;
|
|
1870
|
+
document.addEventListener('visibilitychange', visibilityChanged);
|
|
1871
|
+
schedule();
|
|
1872
|
+
timer = setTimeout(() => reject(new Error(
|
|
1873
|
+
'SCROLL_SETTLEMENT_TIMEOUT: rendering did not finish before the request deadline'
|
|
1874
|
+
)), Math.max(1, ${deadline} - Date.now()));
|
|
1875
|
+
});
|
|
1876
|
+
} finally {
|
|
1877
|
+
clearTimeout(timer);
|
|
1878
|
+
if (frame !== undefined) cancelAnimationFrame(frame);
|
|
1879
|
+
if (visibilityChanged) document.removeEventListener('visibilitychange', visibilityChanged);
|
|
1880
|
+
if (scrollChanged) {
|
|
1881
|
+
for (const target of scrollTargets) {
|
|
1882
|
+
target.removeEventListener('scroll', scrollChanged, true);
|
|
1883
|
+
target.removeEventListener('scrollend', scrollEnded, true);
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
channel.port1.close();
|
|
1887
|
+
channel.port2.close();
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1553
1890
|
}
|
|
1554
|
-
|
|
1555
|
-
|
|
1891
|
+
if (!el.isConnected || el.ownerDocument !== document) return null;
|
|
1892
|
+
const current = el.getBoundingClientRect();
|
|
1893
|
+
return { x: current.x, y: current.y, width: current.width, height: current.height };
|
|
1556
1894
|
})()`, undefined, remaining());
|
|
1557
1895
|
remaining();
|
|
1558
|
-
if (!
|
|
1896
|
+
if (!rect) {
|
|
1559
1897
|
throw new StructuredToolError('TARGET_NOT_FOUND', '目标元素引用已失效,未使用旧坐标派发事件', '请重新读取页面并确认 target', { target, refId, actionExecuted: false });
|
|
1560
1898
|
}
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
}
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1899
|
+
if (frame !== undefined && scroll) {
|
|
1900
|
+
const refreshed = await unifiedSession.find(selector, text, xpath, remaining(), semantic, nth);
|
|
1901
|
+
if (!refreshed[0] || refreshed[0].refId !== refId) {
|
|
1902
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', '滚动后目标元素已失效', '重新读取目标元素', {
|
|
1903
|
+
actionExecuted: false,
|
|
1904
|
+
});
|
|
1905
|
+
}
|
|
1906
|
+
rect = await unifiedSession.evaluate(`(() => {
|
|
1907
|
+
const el = window.__mcpElementMap?.[${JSON.stringify(refId)}]?.deref();
|
|
1908
|
+
if (!el || !el.isConnected || el.ownerDocument !== document) return null;
|
|
1909
|
+
const r = el.getBoundingClientRect();
|
|
1910
|
+
return { x: r.x, y: r.y, width: r.width, height: r.height };
|
|
1911
|
+
})()`, undefined, remaining());
|
|
1912
|
+
if (!rect) {
|
|
1913
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', '最终定位时目标元素已失效', '重新读取目标元素', {
|
|
1914
|
+
target,
|
|
1915
|
+
refId,
|
|
1916
|
+
actionExecuted: false,
|
|
1917
|
+
});
|
|
1918
|
+
}
|
|
1576
1919
|
}
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
const rect = refreshed[nth]?.rect ?? elements[nth].rect;
|
|
1583
|
-
return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2, refId };
|
|
1920
|
+
return {
|
|
1921
|
+
x: rect.x + rect.width / 2,
|
|
1922
|
+
y: rect.y + rect.height / 2,
|
|
1923
|
+
refId,
|
|
1924
|
+
};
|
|
1584
1925
|
}
|
|
1585
1926
|
async function executeCdpEvent(session, event, humanize, timeout) {
|
|
1586
1927
|
validateEvent(event);
|
|
@@ -1605,7 +1946,9 @@ const cdpInputHandlers = {
|
|
|
1605
1946
|
};
|
|
1606
1947
|
async function handleCdpClick({ session, humanize, timeout }, event) {
|
|
1607
1948
|
if (event.force) {
|
|
1608
|
-
const locator = session.createLocator(event.target, {
|
|
1949
|
+
const locator = session.createLocator(event.target, {
|
|
1950
|
+
timeout: targetLookupTimeout(timeout ?? DEFAULT_TIMEOUT),
|
|
1951
|
+
});
|
|
1609
1952
|
await locator.evaluateOn(`function() {
|
|
1610
1953
|
if (!this.isConnected) throw new Error('目标元素已移除,未执行点击');
|
|
1611
1954
|
const button = ${JSON.stringify(event.button ?? 'left')};
|
|
@@ -1614,7 +1957,9 @@ async function handleCdpClick({ session, humanize, timeout }, event) {
|
|
|
1614
1957
|
const masks = {left: 1, middle: 4, right: 2, back: 8, forward: 16};
|
|
1615
1958
|
const rect = this.getBoundingClientRect();
|
|
1616
1959
|
const base = {bubbles: true, cancelable: true, composed: true, view: window,
|
|
1617
|
-
button: codes[button], clientX: rect.x + rect.width / 2, clientY: rect.y + rect.height / 2
|
|
1960
|
+
button: codes[button], clientX: rect.x + rect.width / 2, clientY: rect.y + rect.height / 2,
|
|
1961
|
+
altKey: ${Boolean(session.getModifiers() & 1)}, ctrlKey: ${Boolean(session.getModifiers() & 2)},
|
|
1962
|
+
metaKey: ${Boolean(session.getModifiers() & 4)}, shiftKey: ${Boolean(session.getModifiers() & 8)}};
|
|
1618
1963
|
for (let detail = 1; detail <= count; ++detail) {
|
|
1619
1964
|
const down = this.dispatchEvent(new MouseEvent('mousedown', {...base, detail, buttons: masks[button]}));
|
|
1620
1965
|
if (down && button === 'left' && typeof this.focus === 'function') this.focus();
|
|
@@ -1696,9 +2041,11 @@ async function handleCdpType({ session, humanize, timeout }, event) {
|
|
|
1696
2041
|
}
|
|
1697
2042
|
}
|
|
1698
2043
|
else {
|
|
1699
|
-
const hasActiveFocus = await session.evaluate(
|
|
1700
|
-
|
|
1701
|
-
|
|
2044
|
+
const hasActiveFocus = await session.evaluate(`(() => {
|
|
2045
|
+
(${installDomHelpers.toString()})();
|
|
2046
|
+
const active = globalThis.__mcpDomHelpers.activeElement();
|
|
2047
|
+
return !!active && active !== document.body && active !== document.documentElement;
|
|
2048
|
+
})()`, undefined, remainingInputTimeout(deadline));
|
|
1702
2049
|
if (!hasActiveFocus) {
|
|
1703
2050
|
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
1704
2051
|
}
|
|
@@ -1718,7 +2065,7 @@ async function handleCdpType({ session, humanize, timeout }, event) {
|
|
|
1718
2065
|
}
|
|
1719
2066
|
async function handleCdpWait({ timeout }, event) {
|
|
1720
2067
|
ensureWaitFitsTimeout(event.ms, timeout);
|
|
1721
|
-
await
|
|
2068
|
+
await getUnifiedSession().wait(event.ms);
|
|
1722
2069
|
}
|
|
1723
2070
|
async function handleCdpDrag() {
|
|
1724
2071
|
throw new Error('drag 事件仅在 Extension 模式下可用,当前为 CDP 模式');
|
|
@@ -1759,9 +2106,10 @@ async function moveToTarget(session, target, humanize, timeout, steps, waitType
|
|
|
1759
2106
|
* 获取目标点坐标(带自动等待)
|
|
1760
2107
|
*/
|
|
1761
2108
|
async function getTargetPoint(session, target, timeout, waitType = 'click') {
|
|
1762
|
-
// 如果是坐标,直接返回
|
|
1763
2109
|
if ('x' in target && 'y' in target) {
|
|
1764
|
-
return
|
|
2110
|
+
return session
|
|
2111
|
+
.createLocator(target, { timeout: targetLookupTimeout(timeout ?? DEFAULT_TIMEOUT) })
|
|
2112
|
+
.getClickablePoint();
|
|
1765
2113
|
}
|
|
1766
2114
|
const budget = timeout ?? DEFAULT_TIMEOUT;
|
|
1767
2115
|
const deadline = Date.now() + budget;
|
|
@@ -1773,7 +2121,7 @@ async function getTargetPoint(session, target, timeout, waitType = 'click') {
|
|
|
1773
2121
|
return value;
|
|
1774
2122
|
};
|
|
1775
2123
|
// 使用 Locator 定位元素
|
|
1776
|
-
const locator = session.createLocator(target, { timeout: remaining() });
|
|
2124
|
+
const locator = session.createLocator(target, { timeout: targetLookupTimeout(remaining()) });
|
|
1777
2125
|
const nodeId = await locator.find();
|
|
1778
2126
|
// 根据操作类型执行自动等待
|
|
1779
2127
|
if (waitType !== 'none') {
|
|
@@ -1796,7 +2144,9 @@ async function getTargetPoint(session, target, timeout, waitType = 'click') {
|
|
|
1796
2144
|
export function registerInputTool(server) {
|
|
1797
2145
|
server.registerTool('input', {
|
|
1798
2146
|
description: [
|
|
1799
|
-
'
|
|
2147
|
+
'键鼠输入及文件上传:键盘、鼠标及任意组合。网页上传优先使用本工具的 upload 事件,',
|
|
2148
|
+
'files 支持绝对、相对及 ~/ 路径,沿用当前页面与 Extension/CDP 连接,总大小严格小于 50 MiB;',
|
|
2149
|
+
'默认优先使用已连接的 Chrome Extension;',
|
|
1800
2150
|
'只有 Extension 不可用或 browse(action="connect", port=…) 已显式选择 CDP 时才使用 CDP。',
|
|
1801
2151
|
'访问 Cloudflare 时默认不传 diagnostics,Extension 先被动等待真实 tab 标题恢复,',
|
|
1802
2152
|
'持续存在可点击 Turnstile 时才点击。同页测试按钮走页面点击;',
|
|
@@ -1830,11 +2180,13 @@ export function registerInputTool(server) {
|
|
|
1830
2180
|
type: {events:[{type:"type",target:{css:"input[name=q]"},text:"hello"}]}
|
|
1831
2181
|
controlled type: {events:[{type:"type",mode:"controlled",target:{css:"input[name=q]"},text:"hello"}]}
|
|
1832
2182
|
replace: {events:[{type:"replace",target:{css:"textarea"},find:"old",text:"new"}]}
|
|
2183
|
+
upload: {events:[{type:"upload",target:{css:"input[type=file]"},files:["./attachment.zip"]}]}
|
|
2184
|
+
文件路径在 Server 机器上解析;files:[] 清空选择,总大小须小于 50 MiB;合成 input/change 事件不保证站点已收件,使用 postCondition 验证
|
|
1833
2185
|
注意:纯键盘事件(不带 commands)仅保证 JS keyboard event 可被监听,不保证触发浏览器原生编辑行为;
|
|
1834
2186
|
全选/复制/粘贴等语义用 commands;"全选并替换文本"用 select/replace 事件更简洁;
|
|
1835
2187
|
commands 成功表示浏览器已接受命令分发,页面结果需用 postCondition 或 extract 验证;
|
|
1836
2188
|
commands 仅支持 inputMode=precise,stealth 模式下会报错`,
|
|
1837
2189
|
inputSchema: inputSchema,
|
|
1838
|
-
}, (args) => handleInput(args));
|
|
2190
|
+
}, (args, extra) => handleInput(args, extra?.signal));
|
|
1839
2191
|
}
|
|
1840
2192
|
//# sourceMappingURL=input.js.map
|