@pyrokine/mcp-chrome 2.4.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +546 -192
- package/README_zh.md +490 -192
- package/dist/anti-detection/behavior.js +1 -1
- package/dist/anti-detection/behavior.js.map +1 -1
- package/dist/anti-detection/injection.d.ts.map +1 -1
- package/dist/anti-detection/injection.js +5 -4
- package/dist/anti-detection/injection.js.map +1 -1
- package/dist/cdp/client.d.ts +2 -2
- package/dist/cdp/client.d.ts.map +1 -1
- package/dist/cdp/client.js +24 -10
- package/dist/cdp/client.js.map +1 -1
- package/dist/cdp/launcher.d.ts +2 -3
- package/dist/cdp/launcher.d.ts.map +1 -1
- package/dist/cdp/launcher.js +84 -24
- package/dist/cdp/launcher.js.map +1 -1
- package/dist/core/auto-wait.d.ts +3 -3
- package/dist/core/auto-wait.d.ts.map +1 -1
- package/dist/core/auto-wait.js +45 -20
- package/dist/core/auto-wait.js.map +1 -1
- package/dist/core/browser-driver.d.ts +104 -14
- package/dist/core/browser-driver.d.ts.map +1 -1
- package/dist/core/browser-driver.js +1 -0
- package/dist/core/browser-driver.js.map +1 -1
- package/dist/core/errors.d.ts +25 -0
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +93 -1
- package/dist/core/errors.js.map +1 -1
- package/dist/core/locator.d.ts.map +1 -1
- package/dist/core/locator.js +6 -4
- package/dist/core/locator.js.map +1 -1
- package/dist/core/os-mouse.d.ts +78 -0
- package/dist/core/os-mouse.d.ts.map +1 -0
- package/dist/core/os-mouse.js +378 -0
- package/dist/core/os-mouse.js.map +1 -0
- package/dist/core/session.d.ts +46 -21
- package/dist/core/session.d.ts.map +1 -1
- package/dist/core/session.js +724 -175
- package/dist/core/session.js.map +1 -1
- package/dist/core/types.d.ts +12 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +2 -0
- package/dist/core/types.js.map +1 -1
- package/dist/core/unified-session.d.ts +117 -21
- package/dist/core/unified-session.d.ts.map +1 -1
- package/dist/core/unified-session.js +619 -129
- package/dist/core/unified-session.js.map +1 -1
- package/dist/core/utils.d.ts +5 -0
- package/dist/core/utils.d.ts.map +1 -1
- package/dist/core/utils.js +475 -54
- package/dist/core/utils.js.map +1 -1
- package/dist/extension/bridge.d.ts +61 -35
- package/dist/extension/bridge.d.ts.map +1 -1
- package/dist/extension/bridge.js +290 -106
- package/dist/extension/bridge.js.map +1 -1
- package/dist/extension/http-server.d.ts +3 -0
- package/dist/extension/http-server.d.ts.map +1 -1
- package/dist/extension/http-server.js +25 -3
- package/dist/extension/http-server.js.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/tools/browse.d.ts +1 -1
- package/dist/tools/browse.d.ts.map +1 -1
- package/dist/tools/browse.js +291 -175
- package/dist/tools/browse.js.map +1 -1
- package/dist/tools/challenge.d.ts +153 -0
- package/dist/tools/challenge.d.ts.map +1 -0
- package/dist/tools/challenge.js +611 -0
- package/dist/tools/challenge.js.map +1 -0
- package/dist/tools/cookies.d.ts.map +1 -1
- package/dist/tools/cookies.js +13 -7
- package/dist/tools/cookies.js.map +1 -1
- package/dist/tools/diagnostics.d.ts +5 -5
- package/dist/tools/diagnostics.d.ts.map +1 -1
- package/dist/tools/diagnostics.js +39 -20
- package/dist/tools/diagnostics.js.map +1 -1
- package/dist/tools/evaluate.d.ts.map +1 -1
- package/dist/tools/evaluate.js +106 -23
- package/dist/tools/evaluate.js.map +1 -1
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +199 -208
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/input.d.ts +1 -1
- package/dist/tools/input.d.ts.map +1 -1
- package/dist/tools/input.js +523 -175
- package/dist/tools/input.js.map +1 -1
- package/dist/tools/logs.d.ts.map +1 -1
- package/dist/tools/logs.js +7 -7
- package/dist/tools/logs.js.map +1 -1
- package/dist/tools/manage.d.ts.map +1 -1
- package/dist/tools/manage.js +400 -64
- package/dist/tools/manage.js.map +1 -1
- package/dist/tools/network-sanitizer.d.ts.map +1 -1
- package/dist/tools/network-sanitizer.js +1 -0
- package/dist/tools/network-sanitizer.js.map +1 -1
- package/dist/tools/page-context.d.ts +17 -0
- package/dist/tools/page-context.d.ts.map +1 -0
- package/dist/tools/page-context.js +103 -0
- package/dist/tools/page-context.js.map +1 -0
- package/dist/tools/png.d.ts +5 -0
- package/dist/tools/png.d.ts.map +1 -1
- package/dist/tools/png.js +89 -22
- package/dist/tools/png.js.map +1 -1
- package/dist/tools/post-condition.d.ts.map +1 -1
- package/dist/tools/post-condition.js +17 -5
- package/dist/tools/post-condition.js.map +1 -1
- package/dist/tools/schema.d.ts +9 -0
- package/dist/tools/schema.d.ts.map +1 -1
- package/dist/tools/schema.js +15 -0
- package/dist/tools/schema.js.map +1 -1
- package/dist/tools/target-diagnostics.d.ts +7 -0
- package/dist/tools/target-diagnostics.d.ts.map +1 -1
- package/dist/tools/target-diagnostics.js +47 -17
- package/dist/tools/target-diagnostics.js.map +1 -1
- package/dist/tools/wait.d.ts.map +1 -1
- package/dist/tools/wait.js +52 -56
- package/dist/tools/wait.js.map +1 -1
- package/extension/dist/assets/content.ts-C4imZ6A9.js +13 -0
- package/extension/dist/assets/content.ts-loader-CnJuCGhl.js +13 -0
- package/extension/dist/assets/{index.ts-GG6dPUPX.js → index.ts-GeHNZBml.js} +2156 -1950
- package/extension/dist/assets/page-scripts-CMIY1zhd.js +1379 -0
- package/extension/dist/assets/{popup.html-B81xccfq.js → popup.html-CgtWL6Py.js} +48 -1
- package/extension/dist/manifest.json +4 -3
- package/extension/dist/service-worker-loader.js +1 -1
- package/extension/dist/src/popup/popup.html +54 -44
- package/package.json +11 -8
- package/scripts/start-chrome.ps1 +58 -0
- package/dist/extension/native-host-installer.d.ts +0 -21
- package/dist/extension/native-host-installer.d.ts.map +0 -1
- package/dist/extension/native-host-installer.js +0 -147
- package/dist/extension/native-host-installer.js.map +0 -1
- package/dist/extension/socket-server.d.ts +0 -32
- package/dist/extension/socket-server.d.ts.map +0 -1
- package/dist/extension/socket-server.js +0 -177
- package/dist/extension/socket-server.js.map +0 -1
- package/dist/extension/ws-server.d.ts +0 -40
- package/dist/extension/ws-server.d.ts.map +0 -1
- package/dist/extension/ws-server.js +0 -246
- package/dist/extension/ws-server.js.map +0 -1
- package/dist/native-host/index.js +0 -280
- package/dist/native-host/mcp-chrome-host +0 -2
- package/extension/dist/assets/content.ts-Cj_iiU2s.js +0 -25
package/dist/tools/input.js
CHANGED
|
@@ -11,22 +11,38 @@
|
|
|
11
11
|
import { z } from 'zod';
|
|
12
12
|
import { generateBezierPath, getMouseMoveDelay, getTypingDelay, randomDelay } from '../anti-detection/index.js';
|
|
13
13
|
import { formatErrorResponse, formatResponse, getSession, getUnifiedSession } from '../core/index.js';
|
|
14
|
+
import { StructuredToolError, TimeoutError } from '../core/errors.js';
|
|
15
|
+
import { DEFAULT_TIMEOUT } from '../core/types.js';
|
|
16
|
+
import { ChallengeDeniedError, ChallengeTimeoutError, resolveChallenge } from './challenge.js';
|
|
14
17
|
import { appendDiagnostics, finishDiagnostics, startDiagnostics } from './diagnostics.js';
|
|
18
|
+
import { withPageContext } from './page-context.js';
|
|
15
19
|
import { postConditionSchema, waitForPostCondition } from './post-condition.js';
|
|
16
|
-
import { targetToFindParams, targetZodSchema } from './schema.js';
|
|
20
|
+
import { pageConditionSchema, pageTargetIdSchema, resolvePageTargetId, targetToFindParams, targetZodSchema, } from './schema.js';
|
|
17
21
|
import { buildTargetDiagnostics } from './target-diagnostics.js';
|
|
18
22
|
const INPUT_WAIT_MAX_MS = 60_000;
|
|
19
23
|
const TEXT_SELECTION_INPUT_TYPES = new Set(['text', 'search', 'tel', 'url', 'password']);
|
|
24
|
+
function throwInputTimeoutError(timeout) {
|
|
25
|
+
throw new Error(`input timeout before event dispatch (${timeout}ms)`);
|
|
26
|
+
}
|
|
27
|
+
function remainingInputTimeout(deadline) {
|
|
28
|
+
const remaining = deadline - Date.now();
|
|
29
|
+
if (remaining <= 0) {
|
|
30
|
+
throw new TimeoutError('input timeout during event execution');
|
|
31
|
+
}
|
|
32
|
+
return remaining;
|
|
33
|
+
}
|
|
20
34
|
export function supportsTextSelection(tag, inputType) {
|
|
21
35
|
return (tag.toLowerCase() === 'textarea' ||
|
|
22
36
|
(tag.toLowerCase() === 'input' && TEXT_SELECTION_INPUT_TYPES.has((inputType ?? 'text').toLowerCase())));
|
|
23
37
|
}
|
|
38
|
+
// noinspection JSUnusedGlobalSymbols — 回归测试从构建产物导入该 helper
|
|
24
39
|
export function replaceNthOccurrence(value, find, replacement, nth = 0) {
|
|
25
40
|
let index = -1;
|
|
26
41
|
for (let occurrence = 0; occurrence <= nth; occurrence++) {
|
|
27
42
|
index = value.indexOf(find, index + (occurrence > 0 ? 1 : 0));
|
|
28
|
-
if (index === -1)
|
|
43
|
+
if (index === -1) {
|
|
29
44
|
return null;
|
|
45
|
+
}
|
|
30
46
|
}
|
|
31
47
|
return value.slice(0, index) + replacement + value.slice(index + find.length);
|
|
32
48
|
}
|
|
@@ -73,7 +89,8 @@ const inputEventSchema = z.object({
|
|
|
73
89
|
commands: z
|
|
74
90
|
.array(z.string())
|
|
75
91
|
.optional()
|
|
76
|
-
.describe('浏览器编辑命令(keydown 专用),如
|
|
92
|
+
.describe('浏览器编辑命令(keydown 专用),如 selectAll、copy、paste、cut、undo、redo;' +
|
|
93
|
+
'成功表示命令已分发,页面结果需用 postCondition 或 extract 验证')
|
|
77
94
|
.describe('用于跨平台快捷键场景,需要 inputMode=precise'),
|
|
78
95
|
button: z.enum(['left', 'middle', 'right', 'back', 'forward']).optional().describe('鼠标按钮'),
|
|
79
96
|
clickCount: z
|
|
@@ -119,54 +136,81 @@ const inputEventSchema = z.object({
|
|
|
119
136
|
* input 参数 schema
|
|
120
137
|
*/
|
|
121
138
|
const inputSchema = z.object({
|
|
139
|
+
...pageConditionSchema,
|
|
122
140
|
events: z.array(inputEventSchema).describe('事件序列'),
|
|
123
141
|
humanize: z.boolean().optional().describe('启用人类行为模拟(贝塞尔曲线移动、随机延迟)'),
|
|
124
|
-
diagnostics: z
|
|
142
|
+
diagnostics: z
|
|
143
|
+
.boolean()
|
|
144
|
+
.optional()
|
|
145
|
+
.describe('执行后返回新增 console error/warning 和失败网络请求摘要。访问 Cloudflare 页面时默认不传;开启后会启用 debugger 日志采集'),
|
|
125
146
|
postCondition: postConditionSchema
|
|
126
147
|
.optional()
|
|
127
148
|
.describe('动作执行后要验证的页面状态;不传时 success 只表示事件已发出,不表示业务结果已达成'),
|
|
128
|
-
|
|
129
|
-
|
|
149
|
+
targetId: pageTargetIdSchema.describe('目标页面 ID(Extension 或 CDP),优先于默认操作目标'),
|
|
150
|
+
tabId: pageTargetIdSchema.describe('旧版目标 Tab ID,与 targetId 相同语义,保留兼容'),
|
|
151
|
+
timeout: z
|
|
152
|
+
.number()
|
|
130
153
|
.optional()
|
|
131
|
-
.describe('
|
|
132
|
-
timeout: z.number().optional().describe('超时毫秒'),
|
|
154
|
+
.describe('超时毫秒,含 Cloudflare Challenge 等待、目标准备、鼠标移动、字符输入和延迟;到期后停止后续动作,不自动重放'),
|
|
133
155
|
frame: z
|
|
134
156
|
.union([z.string(), z.number()])
|
|
135
157
|
.optional()
|
|
136
158
|
.describe('iframe 定位(可选,仅 Extension 模式),CSS 选择器(如 "iframe#main")或索引(如 0),不指定则在主框架操作'),
|
|
137
159
|
});
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
context;
|
|
142
|
-
constructor(code, message, suggestion, context) {
|
|
143
|
-
super(message);
|
|
144
|
-
this.code = code;
|
|
145
|
-
this.suggestion = suggestion;
|
|
146
|
-
this.context = context;
|
|
147
|
-
this.name = 'StructuredToolError';
|
|
148
|
-
}
|
|
149
|
-
toJSON() {
|
|
150
|
-
return {
|
|
151
|
-
error: {
|
|
152
|
-
code: this.code,
|
|
153
|
-
message: this.message,
|
|
154
|
-
suggestion: this.suggestion,
|
|
155
|
-
context: this.context,
|
|
156
|
-
},
|
|
157
|
-
};
|
|
160
|
+
function sanitizeInputResponse(response, events, sensitiveInput) {
|
|
161
|
+
if (!response.isError && !sensitiveInput) {
|
|
162
|
+
return response;
|
|
158
163
|
}
|
|
159
|
-
}
|
|
160
|
-
function sanitizeInputFailureResponse(response, events) {
|
|
161
164
|
const sensitiveValues = events
|
|
162
165
|
.flatMap((event) => [event.find, event.text])
|
|
163
166
|
.filter((value) => !!value);
|
|
164
|
-
if (sensitiveValues.length === 0) {
|
|
165
|
-
return response;
|
|
166
|
-
}
|
|
167
167
|
for (const item of response.content) {
|
|
168
168
|
try {
|
|
169
|
-
|
|
169
|
+
const payload = JSON.parse(item.text);
|
|
170
|
+
for (const key of ['warnings', 'eventResults', 'diagnostics', 'diagnosticsError']) {
|
|
171
|
+
if (key in payload) {
|
|
172
|
+
payload[key] = redactSensitiveInputData(payload[key], sensitiveValues);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const pageContext = payload.pageContext;
|
|
176
|
+
const postCondition = payload.postCondition;
|
|
177
|
+
const details = [
|
|
178
|
+
payload.error,
|
|
179
|
+
pageContext,
|
|
180
|
+
pageContext?.before,
|
|
181
|
+
pageContext?.after,
|
|
182
|
+
postCondition,
|
|
183
|
+
...(Array.isArray(postCondition?.checks) ? postCondition.checks : []),
|
|
184
|
+
];
|
|
185
|
+
for (const detail of details) {
|
|
186
|
+
if (!detail || typeof detail !== 'object') {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
const record = detail;
|
|
190
|
+
for (const key of [
|
|
191
|
+
'message',
|
|
192
|
+
'suggestion',
|
|
193
|
+
'context',
|
|
194
|
+
'contextError',
|
|
195
|
+
'url',
|
|
196
|
+
'title',
|
|
197
|
+
'actual',
|
|
198
|
+
'error',
|
|
199
|
+
]) {
|
|
200
|
+
if (key in record) {
|
|
201
|
+
record[key] = redactSensitiveInputData(record[key], sensitiveValues);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
if (sensitiveInput) {
|
|
206
|
+
if (payload.diagnostics) {
|
|
207
|
+
payload.diagnostics = { redacted: true, reason: 'sensitive_input_or_unresolved_target' };
|
|
208
|
+
}
|
|
209
|
+
if (payload.diagnosticsError) {
|
|
210
|
+
payload.diagnosticsError = '敏感输入的诊断详情已隐藏';
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
item.text = JSON.stringify(payload, null, 2);
|
|
170
214
|
}
|
|
171
215
|
catch {
|
|
172
216
|
item.text = redactSensitiveInputData(item.text, sensitiveValues);
|
|
@@ -178,27 +222,63 @@ function sanitizeInputFailureResponse(response, events) {
|
|
|
178
222
|
* input 工具处理器
|
|
179
223
|
*/
|
|
180
224
|
async function handleInput(args) {
|
|
225
|
+
let eventsExecuted = 0;
|
|
226
|
+
let actionMayHaveExecuted = false;
|
|
227
|
+
const privacy = { sensitive: false };
|
|
228
|
+
const progress = {
|
|
229
|
+
actionStatus: 'not_started',
|
|
230
|
+
};
|
|
231
|
+
let failureStage = 'action';
|
|
181
232
|
try {
|
|
233
|
+
if (args.events.length === 0) {
|
|
234
|
+
return {
|
|
235
|
+
content: [
|
|
236
|
+
{
|
|
237
|
+
type: 'text',
|
|
238
|
+
text: JSON.stringify({
|
|
239
|
+
error: {
|
|
240
|
+
code: 'INVALID_ARGUMENT',
|
|
241
|
+
message: 'events 至少需要一个事件,未执行输入操作',
|
|
242
|
+
},
|
|
243
|
+
actionExecuted: false,
|
|
244
|
+
actionStatus: 'failed',
|
|
245
|
+
}),
|
|
246
|
+
},
|
|
247
|
+
],
|
|
248
|
+
isError: true,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
182
251
|
const unifiedSession = getUnifiedSession();
|
|
183
252
|
const humanize = args.humanize ?? false;
|
|
184
|
-
|
|
185
|
-
return
|
|
253
|
+
const response = await withPageContext(unifiedSession, resolvePageTargetId(args), async () => {
|
|
254
|
+
return unifiedSession.withPageCondition(args, () => unifiedSession.withFrame(args.frame, async () => {
|
|
186
255
|
const mode = unifiedSession.getMode();
|
|
256
|
+
const inputStartedAt = Date.now();
|
|
257
|
+
if (args.expectedUrl === undefined && args.expectedRevision === undefined) {
|
|
258
|
+
await resolveChallenge(unifiedSession, 'input', 'not_started', args.timeout);
|
|
259
|
+
}
|
|
187
260
|
const diagnostics = await startDiagnostics(unifiedSession, args.diagnostics);
|
|
188
261
|
const warnings = [];
|
|
189
262
|
const eventResults = [];
|
|
190
263
|
const session = mode === 'cdp' ? getSession() : undefined;
|
|
191
|
-
const inputStartedAt = Date.now();
|
|
192
|
-
let eventsExecuted = 0;
|
|
193
264
|
try {
|
|
194
265
|
for (const event of args.events) {
|
|
195
266
|
const eventTimeout = args.timeout === undefined
|
|
196
267
|
? undefined
|
|
197
268
|
: Math.max(0, args.timeout - (Date.now() - inputStartedAt));
|
|
198
269
|
if (eventTimeout !== undefined && eventTimeout <= 0) {
|
|
199
|
-
|
|
270
|
+
throwInputTimeoutError(args.timeout);
|
|
200
271
|
}
|
|
201
|
-
|
|
272
|
+
progress.actionStatus = 'failed';
|
|
273
|
+
const result = await executeInputEvent({
|
|
274
|
+
unifiedSession,
|
|
275
|
+
session,
|
|
276
|
+
mode,
|
|
277
|
+
humanize,
|
|
278
|
+
timeout: eventTimeout,
|
|
279
|
+
frame: args.frame,
|
|
280
|
+
privacy,
|
|
281
|
+
}, event);
|
|
202
282
|
++eventsExecuted;
|
|
203
283
|
if (typeof result === 'string') {
|
|
204
284
|
warnings.push(result);
|
|
@@ -211,20 +291,50 @@ async function handleInput(args) {
|
|
|
211
291
|
catch (error) {
|
|
212
292
|
const response = formatErrorResponse(error);
|
|
213
293
|
const text = response.content[0]?.text;
|
|
294
|
+
if ((error instanceof ChallengeTimeoutError || error instanceof ChallengeDeniedError) && text) {
|
|
295
|
+
try {
|
|
296
|
+
const payload = JSON.parse(text);
|
|
297
|
+
Object.assign(payload, {
|
|
298
|
+
eventsExecuted,
|
|
299
|
+
remainingEventsNotExecuted: args.events.length - eventsExecuted,
|
|
300
|
+
verificationRequested: Boolean(args.postCondition),
|
|
301
|
+
});
|
|
302
|
+
appendDiagnostics(payload, await finishDiagnostics(unifiedSession, diagnostics));
|
|
303
|
+
response.content[0].text = JSON.stringify(payload, null, 2);
|
|
304
|
+
}
|
|
305
|
+
catch {
|
|
306
|
+
// 保留原始 Challenge 错误响应
|
|
307
|
+
}
|
|
308
|
+
return response;
|
|
309
|
+
}
|
|
214
310
|
if (text) {
|
|
215
311
|
try {
|
|
216
312
|
const payload = JSON.parse(text);
|
|
217
313
|
const message = error instanceof Error ? error.message : String(error);
|
|
218
314
|
const timedOut = /timeout|超时/i.test(message);
|
|
219
315
|
const preActionTimeout = /timeout before event dispatch|超过 input 剩余 timeout/i.test(message);
|
|
220
|
-
|
|
316
|
+
actionMayHaveExecuted = timedOut && !preActionTimeout;
|
|
317
|
+
const preconditionFailed = error !== null &&
|
|
318
|
+
typeof error === 'object' &&
|
|
319
|
+
'code' in error &&
|
|
320
|
+
error.code === 'ACTION_PRECONDITION_FAILED';
|
|
321
|
+
actionMayHaveExecuted = actionMayHaveExecuted && !preconditionFailed;
|
|
322
|
+
progress.actionStatus =
|
|
323
|
+
preconditionFailed && eventsExecuted === 0
|
|
324
|
+
? 'not_started'
|
|
325
|
+
: actionMayHaveExecuted
|
|
326
|
+
? 'unknown'
|
|
327
|
+
: 'failed';
|
|
221
328
|
payload.eventsExecuted = eventsExecuted;
|
|
222
329
|
payload.actionExecuted = eventsExecuted > 0 || actionMayHaveExecuted;
|
|
223
|
-
payload.actionStatus =
|
|
330
|
+
payload.actionStatus = progress.actionStatus;
|
|
224
331
|
payload.verificationRequested = Boolean(args.postCondition);
|
|
225
332
|
payload.verificationStatus = 'unavailable';
|
|
226
333
|
payload.failureStage = 'action';
|
|
227
|
-
payload.retryable =
|
|
334
|
+
payload.retryable =
|
|
335
|
+
eventsExecuted === 0 &&
|
|
336
|
+
!actionMayHaveExecuted &&
|
|
337
|
+
/timeout|超时|disconnect|未连接|context|debugger/i.test(message);
|
|
228
338
|
appendDiagnostics(payload, await finishDiagnostics(unifiedSession, diagnostics));
|
|
229
339
|
response.content[0].text = JSON.stringify(payload, null, 2);
|
|
230
340
|
}
|
|
@@ -232,8 +342,10 @@ async function handleInput(args) {
|
|
|
232
342
|
// 保留原始错误响应
|
|
233
343
|
}
|
|
234
344
|
}
|
|
235
|
-
return
|
|
345
|
+
return response;
|
|
236
346
|
}
|
|
347
|
+
progress.actionStatus = 'completed';
|
|
348
|
+
failureStage = 'context';
|
|
237
349
|
const result = {
|
|
238
350
|
success: true,
|
|
239
351
|
eventsExecuted,
|
|
@@ -251,7 +363,9 @@ async function handleInput(args) {
|
|
|
251
363
|
result.eventResults = eventResults;
|
|
252
364
|
}
|
|
253
365
|
if (args.postCondition) {
|
|
366
|
+
failureStage = 'verification';
|
|
254
367
|
const postCondition = await waitForPostCondition(unifiedSession, args.postCondition, 'input');
|
|
368
|
+
failureStage = 'context';
|
|
255
369
|
result.postCondition = postCondition;
|
|
256
370
|
result.verificationStatus = postCondition.verificationStatus;
|
|
257
371
|
result.retryable = postCondition.retryable;
|
|
@@ -262,8 +376,9 @@ async function handleInput(args) {
|
|
|
262
376
|
}
|
|
263
377
|
appendDiagnostics(result, await finishDiagnostics(unifiedSession, diagnostics));
|
|
264
378
|
return formatResponse(result);
|
|
265
|
-
}); // withFrame
|
|
379
|
+
})); // withPageCondition, withFrame
|
|
266
380
|
}); // withTabId
|
|
381
|
+
return sanitizeInputResponse(response, args.events, privacy.sensitive);
|
|
267
382
|
}
|
|
268
383
|
catch (error) {
|
|
269
384
|
const response = formatErrorResponse(error);
|
|
@@ -272,17 +387,42 @@ async function handleInput(args) {
|
|
|
272
387
|
try {
|
|
273
388
|
const payload = JSON.parse(text);
|
|
274
389
|
const message = error instanceof Error ? error.message : String(error);
|
|
390
|
+
const unavailableCondition = eventsExecuted === 0 &&
|
|
391
|
+
!actionMayHaveExecuted &&
|
|
392
|
+
progress.actionStatus === 'not_started' &&
|
|
393
|
+
error instanceof StructuredToolError &&
|
|
394
|
+
error.code === 'DRIVER_CAPABILITY_ERROR' &&
|
|
395
|
+
error.context.actionExecuted === false &&
|
|
396
|
+
error.context.actionStatus === 'not_started' &&
|
|
397
|
+
error.context.failureStage === 'precondition';
|
|
398
|
+
const preconditionFailed = unavailableCondition ||
|
|
399
|
+
(error !== null &&
|
|
400
|
+
typeof error === 'object' &&
|
|
401
|
+
'code' in error &&
|
|
402
|
+
error.code === 'ACTION_PRECONDITION_FAILED');
|
|
275
403
|
Object.assign(payload, {
|
|
276
|
-
|
|
277
|
-
|
|
404
|
+
eventsExecuted,
|
|
405
|
+
actionExecuted: eventsExecuted > 0 || actionMayHaveExecuted,
|
|
406
|
+
actionStatus: preconditionFailed && eventsExecuted === 0
|
|
407
|
+
? 'not_started'
|
|
408
|
+
: progress.actionStatus === 'not_started'
|
|
409
|
+
? 'failed'
|
|
410
|
+
: progress.actionStatus,
|
|
278
411
|
verificationRequested: Boolean(args.postCondition),
|
|
279
412
|
verificationStatus: 'unavailable',
|
|
280
|
-
failureStage: '
|
|
281
|
-
retryable:
|
|
413
|
+
failureStage: unavailableCondition ? 'precondition' : failureStage,
|
|
414
|
+
retryable: progress.actionStatus !== 'completed' &&
|
|
415
|
+
eventsExecuted === 0 &&
|
|
416
|
+
!actionMayHaveExecuted &&
|
|
417
|
+
!preconditionFailed &&
|
|
418
|
+
/timeout|超时|disconnect|未连接|context|debugger|attach/i.test(message),
|
|
282
419
|
diagnosticsStatus: args.diagnostics ? 'unavailable' : 'disabled',
|
|
283
420
|
});
|
|
284
421
|
if (args.diagnostics) {
|
|
285
|
-
payload.diagnosticsError =
|
|
422
|
+
payload.diagnosticsError =
|
|
423
|
+
eventsExecuted > 0 || actionMayHaveExecuted
|
|
424
|
+
? 'input 执行后未能取得 diagnostics'
|
|
425
|
+
: 'diagnostics 未能在 tab/frame 初始化失败前启动';
|
|
286
426
|
}
|
|
287
427
|
response.content[0].text = JSON.stringify(payload, null, 2);
|
|
288
428
|
}
|
|
@@ -290,10 +430,12 @@ async function handleInput(args) {
|
|
|
290
430
|
// 保留原始错误响应
|
|
291
431
|
}
|
|
292
432
|
}
|
|
293
|
-
return
|
|
433
|
+
return sanitizeInputResponse(response, args.events, privacy.sensitive);
|
|
294
434
|
}
|
|
295
435
|
}
|
|
296
436
|
const unifiedOnlyEventTypes = new Set([
|
|
437
|
+
'keydown',
|
|
438
|
+
'keyup',
|
|
297
439
|
'select',
|
|
298
440
|
'replace',
|
|
299
441
|
'editorContext',
|
|
@@ -309,6 +451,7 @@ async function executeInputEvent(context, event) {
|
|
|
309
451
|
humanize: context.humanize,
|
|
310
452
|
timeout: context.timeout,
|
|
311
453
|
frame: context.frame,
|
|
454
|
+
privacy: context.privacy,
|
|
312
455
|
}, event);
|
|
313
456
|
}
|
|
314
457
|
if (!context.session) {
|
|
@@ -331,14 +474,23 @@ async function executeInputEvent(context, event) {
|
|
|
331
474
|
* select/replace 事件用,保证选区建立前 activeElement 就是目标
|
|
332
475
|
*/
|
|
333
476
|
async function focusTargetForCommands(unifiedSession, target, timeout, frame) {
|
|
477
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
334
478
|
if ('x' in target || 'y' in target) {
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
479
|
+
await handleUnifiedClick({
|
|
480
|
+
unifiedSession,
|
|
481
|
+
mode: unifiedSession.getMode(),
|
|
482
|
+
humanize: false,
|
|
483
|
+
timeout: remainingInputTimeout(deadline),
|
|
484
|
+
frame,
|
|
485
|
+
}, { type: 'click', target });
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
if (unifiedSession.getMode() === 'cdp') {
|
|
489
|
+
await focusCdpTarget(getSession(), target, remainingInputTimeout(deadline));
|
|
338
490
|
return;
|
|
339
491
|
}
|
|
340
492
|
const params = targetToFindParams(target);
|
|
341
|
-
const els = await unifiedSession.find(params.selector, params.text, params.xpath,
|
|
493
|
+
const els = await unifiedSession.find(params.selector, params.text, params.xpath, remainingInputTimeout(deadline));
|
|
342
494
|
const nth0 = params.nth ?? 0;
|
|
343
495
|
if (els.length <= nth0) {
|
|
344
496
|
throw new StructuredToolError(els.length === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', els.length === 0
|
|
@@ -353,7 +505,7 @@ async function focusTargetForCommands(unifiedSession, target, timeout, frame) {
|
|
|
353
505
|
}
|
|
354
506
|
el.focus()
|
|
355
507
|
return document.activeElement === el
|
|
356
|
-
})()`, undefined,
|
|
508
|
+
})()`, undefined, remainingInputTimeout(deadline));
|
|
357
509
|
if (focused !== true) {
|
|
358
510
|
throw new StructuredToolError('TARGET_FOCUS_FAILED', '命令目标未成功聚焦', '请检查目标是否可编辑、未 disabled 并处于正确 frame', await buildTargetDiagnosticContext(unifiedSession, target, els.length, nth0, timeout, '命令目标聚焦失败', frame));
|
|
359
511
|
}
|
|
@@ -373,27 +525,33 @@ async function focusTargetIfNeeded(unifiedSession, session, mode, target, timeou
|
|
|
373
525
|
if (!target) {
|
|
374
526
|
return;
|
|
375
527
|
}
|
|
528
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
376
529
|
if ('x' in target || 'y' in target) {
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
530
|
+
await handleUnifiedClick({
|
|
531
|
+
unifiedSession,
|
|
532
|
+
mode: unifiedSession.getMode(),
|
|
533
|
+
humanize: false,
|
|
534
|
+
timeout: remainingInputTimeout(deadline),
|
|
535
|
+
frame,
|
|
536
|
+
}, { type: 'click', target });
|
|
380
537
|
return;
|
|
381
538
|
}
|
|
382
539
|
if (mode === 'cdp') {
|
|
383
540
|
if (!session) {
|
|
384
541
|
throw new Error('CDP locator 聚焦缺少 session');
|
|
385
542
|
}
|
|
386
|
-
await focusCdpTarget(session, target,
|
|
543
|
+
await focusCdpTarget(session, target, remainingInputTimeout(deadline));
|
|
387
544
|
return;
|
|
388
545
|
}
|
|
389
546
|
const params = targetToFindParams(target);
|
|
390
|
-
const els = await unifiedSession.find(params.selector, params.text, params.xpath,
|
|
547
|
+
const els = await unifiedSession.find(params.selector, params.text, params.xpath, remainingInputTimeout(deadline));
|
|
391
548
|
const nth0 = params.nth ?? 0;
|
|
392
549
|
if (els.length <= nth0) {
|
|
393
550
|
throw new StructuredToolError(els.length === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', els.length === 0
|
|
394
551
|
? `未找到目标元素: ${JSON.stringify(target)}`
|
|
395
552
|
: `第 ${nth0} 个匹配元素不存在(共 ${els.length} 个)`, '请检查 target、frame 和 target.nth,或先用 extract type=state 查看当前候选元素', await buildTargetDiagnosticContext(unifiedSession, target, els.length, nth0, timeout, '聚焦目标未找到', frame));
|
|
396
553
|
}
|
|
554
|
+
remainingInputTimeout(deadline);
|
|
397
555
|
const result = await unifiedSession.actionableClick(els[nth0].refId);
|
|
398
556
|
if (!result.success) {
|
|
399
557
|
throw new StructuredToolError('ACTIONABILITY_FAILED', result.error || '目标元素无法聚焦', result.suggestions?.[0] ?? '请检查元素是否可见、未被遮挡并处于正确 frame', {
|
|
@@ -412,11 +570,16 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
412
570
|
while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 4) {
|
|
413
571
|
var part = current.tagName.toLowerCase();
|
|
414
572
|
if (current.classList && current.classList.length) {
|
|
415
|
-
part += '.' + Array.from(current.classList)
|
|
573
|
+
part += '.' + Array.from(current.classList)
|
|
574
|
+
.slice(0, 2)
|
|
575
|
+
.map(function(cls) { return CSS.escape(cls); })
|
|
576
|
+
.join('.');
|
|
416
577
|
}
|
|
417
578
|
var parent = current.parentElement;
|
|
418
579
|
if (parent) {
|
|
419
|
-
var siblings = Array.from(parent.children).filter(function(child) {
|
|
580
|
+
var siblings = Array.from(parent.children).filter(function(child) {
|
|
581
|
+
return child.tagName === current.tagName;
|
|
582
|
+
});
|
|
420
583
|
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
|
421
584
|
}
|
|
422
585
|
parts.unshift(part);
|
|
@@ -430,7 +593,9 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
430
593
|
var value = 'value' in el ? String(el.value || '') : '';
|
|
431
594
|
var inputType = el instanceof HTMLInputElement ? (el.type || 'text').toLowerCase() : undefined;
|
|
432
595
|
var sensitive = inputType === 'password';
|
|
433
|
-
var text = sensitive
|
|
596
|
+
var text = sensitive
|
|
597
|
+
? (el.innerText || el.textContent || '')
|
|
598
|
+
: (el.innerText || el.textContent || value || '');
|
|
434
599
|
text = text.replace(new RegExp('\\\\s+', 'g'), ' ').trim();
|
|
435
600
|
return {
|
|
436
601
|
tag: el.tagName.toLowerCase(),
|
|
@@ -470,10 +635,16 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
470
635
|
var candidateElements = [];
|
|
471
636
|
if (root instanceof Element) {
|
|
472
637
|
candidateElements.push(root);
|
|
473
|
-
|
|
638
|
+
var candidateSelector =
|
|
639
|
+
'input, textarea, [contenteditable="true"], [role="textbox"], button, a';
|
|
640
|
+
candidateElements = candidateElements.concat(
|
|
641
|
+
Array.from(root.querySelectorAll(candidateSelector)).slice(0, 12)
|
|
642
|
+
);
|
|
474
643
|
}
|
|
475
644
|
var active = document.activeElement;
|
|
476
|
-
var activeIsPassword =
|
|
645
|
+
var activeIsPassword =
|
|
646
|
+
active instanceof HTMLInputElement &&
|
|
647
|
+
(active.type || 'text').toLowerCase() === 'password';
|
|
477
648
|
var selection = window.getSelection();
|
|
478
649
|
return {
|
|
479
650
|
scope: {
|
|
@@ -506,7 +677,12 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
506
677
|
};
|
|
507
678
|
}`, undefined, timeout, [findText, occurrenceNth, scopeNth, scopeTarget ?? null, scopeSelector, scopeText, scopeXpath]);
|
|
508
679
|
}
|
|
509
|
-
async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeout) {
|
|
680
|
+
async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeout, privacy) {
|
|
681
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
682
|
+
const sensitiveBefore = privacy?.sensitive ?? false;
|
|
683
|
+
if (privacy) {
|
|
684
|
+
privacy.sensitive = true;
|
|
685
|
+
}
|
|
510
686
|
// 将 target 转为查询参数,传入注入脚本进行 DOM 查询
|
|
511
687
|
let scopeSelector = null;
|
|
512
688
|
let scopeText = null;
|
|
@@ -520,6 +696,7 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
520
696
|
scopeNth = params.nth ?? 0;
|
|
521
697
|
}
|
|
522
698
|
// Step 1: 注入脚本定位文本
|
|
699
|
+
//noinspection SpellCheckingInspection -- 模板中的坐标字段名
|
|
523
700
|
const result = await unifiedSession.evaluate(`function(findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath) {
|
|
524
701
|
// 确定搜索根节点
|
|
525
702
|
var root = document.body;
|
|
@@ -547,19 +724,26 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
547
724
|
if (tag === 'INPUT' || tag === 'TEXTAREA') {
|
|
548
725
|
var val = root.value || '';
|
|
549
726
|
var inputType = tag === 'INPUT' ? (root.type || 'text').toLowerCase() : 'textarea';
|
|
550
|
-
var selectionCapable =
|
|
727
|
+
var selectionCapable =
|
|
728
|
+
tag === 'TEXTAREA' ||
|
|
729
|
+
['text', 'search', 'tel', 'url', 'password'].indexOf(inputType) !== -1;
|
|
551
730
|
if (!selectionCapable) {
|
|
552
|
-
return {
|
|
731
|
+
return {
|
|
732
|
+
type: 'unsupported',
|
|
733
|
+
tag: tag.toLowerCase(),
|
|
734
|
+
inputType: inputType,
|
|
735
|
+
currentValue: String(val).slice(0, 160)
|
|
736
|
+
};
|
|
553
737
|
}
|
|
554
738
|
var pos = -1;
|
|
555
739
|
for (var n = 0; n <= nth; n++) {
|
|
556
740
|
pos = val.indexOf(findText, pos + (n > 0 ? 1 : 0));
|
|
557
|
-
if (pos === -1) return {type: 'notfound'};
|
|
741
|
+
if (pos === -1) return {type: 'notfound', inputType: inputType};
|
|
558
742
|
}
|
|
559
743
|
// 原子化:定位到 input 同时完成 focus + setSelectionRange,避免外层 mouseClick 聚焦不可靠
|
|
560
744
|
root.focus();
|
|
561
745
|
root.setSelectionRange(pos, pos + findText.length);
|
|
562
|
-
return {type: 'input', selectionStart: pos, selectionEnd: pos + findText.length};
|
|
746
|
+
return {type: 'input', inputType: inputType, selectionStart: pos, selectionEnd: pos + findText.length};
|
|
563
747
|
}
|
|
564
748
|
|
|
565
749
|
// 在子树中查找 input/textarea
|
|
@@ -568,19 +752,26 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
568
752
|
var inp = inputs[k];
|
|
569
753
|
var v = inp.value || '';
|
|
570
754
|
var childType = inp.tagName === 'INPUT' ? (inp.type || 'text').toLowerCase() : 'textarea';
|
|
571
|
-
var childSelectionCapable =
|
|
755
|
+
var childSelectionCapable =
|
|
756
|
+
inp.tagName === 'TEXTAREA' ||
|
|
757
|
+
['text', 'search', 'tel', 'url', 'password'].indexOf(childType) !== -1;
|
|
572
758
|
var ip = -1;
|
|
573
759
|
for (var n2 = 0; n2 <= nth; n2++) {
|
|
574
760
|
ip = v.indexOf(findText, ip + (n2 > 0 ? 1 : 0));
|
|
575
761
|
if (ip === -1) break;
|
|
576
762
|
}
|
|
577
763
|
if (ip !== -1 && !childSelectionCapable) {
|
|
578
|
-
return {
|
|
764
|
+
return {
|
|
765
|
+
type: 'unsupported',
|
|
766
|
+
tag: inp.tagName.toLowerCase(),
|
|
767
|
+
inputType: childType,
|
|
768
|
+
currentValue: String(v).slice(0, 160)
|
|
769
|
+
};
|
|
579
770
|
}
|
|
580
771
|
if (ip !== -1) {
|
|
581
772
|
inp.focus();
|
|
582
773
|
inp.setSelectionRange(ip, ip + findText.length);
|
|
583
|
-
return {type: 'input', selectionStart: ip, selectionEnd: ip + findText.length};
|
|
774
|
+
return {type: 'input', inputType: childType, selectionStart: ip, selectionEnd: ip + findText.length};
|
|
584
775
|
}
|
|
585
776
|
}
|
|
586
777
|
|
|
@@ -657,7 +848,13 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
657
848
|
endY: er.y + er.height / 2,
|
|
658
849
|
formatted: formatted || undefined
|
|
659
850
|
};
|
|
660
|
-
}`, undefined,
|
|
851
|
+
}`, undefined, remainingInputTimeout(deadline), [findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath]);
|
|
852
|
+
if (privacy &&
|
|
853
|
+
result &&
|
|
854
|
+
result.type !== 'noscope' &&
|
|
855
|
+
(result.type !== 'notfound' || result.inputType !== undefined)) {
|
|
856
|
+
privacy.sensitive = sensitiveBefore || ('inputType' in result && result.inputType === 'password');
|
|
857
|
+
}
|
|
661
858
|
if (!result || result.type === 'noscope') {
|
|
662
859
|
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeNth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
663
860
|
throw new StructuredToolError('TEXT_SCOPE_NOT_FOUND', `未找到目标元素: ${JSON.stringify(scopeTarget)}`, '请检查 target 是否能定位到包含目标文本的元素,或先用 extract(state) 查看可交互元素', diagnostics);
|
|
@@ -695,11 +892,25 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
695
892
|
coords.endY += frameOffset.y;
|
|
696
893
|
}
|
|
697
894
|
// Step 2: 模拟鼠标选择
|
|
698
|
-
await unifiedSession.mouseMove(coords.startX, coords.startY);
|
|
699
|
-
|
|
700
|
-
await unifiedSession.keyDown('Shift');
|
|
701
|
-
await unifiedSession.mouseMove(coords.endX, coords.endY);
|
|
895
|
+
await unifiedSession.mouseMove(coords.startX, coords.startY, remainingInputTimeout(deadline));
|
|
896
|
+
remainingInputTimeout(deadline);
|
|
702
897
|
await unifiedSession.mouseClick('left');
|
|
898
|
+
remainingInputTimeout(deadline);
|
|
899
|
+
try {
|
|
900
|
+
await unifiedSession.keyDown('Shift');
|
|
901
|
+
await unifiedSession.mouseMove(coords.endX, coords.endY, remainingInputTimeout(deadline));
|
|
902
|
+
remainingInputTimeout(deadline);
|
|
903
|
+
await unifiedSession.mouseClick('left');
|
|
904
|
+
}
|
|
905
|
+
catch (error) {
|
|
906
|
+
try {
|
|
907
|
+
await unifiedSession.keyUp('Shift');
|
|
908
|
+
}
|
|
909
|
+
catch (releaseError) {
|
|
910
|
+
console.error('[MCP] Shift release failed after selection error:', releaseError);
|
|
911
|
+
}
|
|
912
|
+
throw error;
|
|
913
|
+
}
|
|
703
914
|
await unifiedSession.keyUp('Shift');
|
|
704
915
|
return coords.formatted;
|
|
705
916
|
}
|
|
@@ -710,6 +921,9 @@ function requiredEventParam(event, name) {
|
|
|
710
921
|
throw new Error(`events[].${String(name)} 是 ${event.type} 事件的必填参数`);
|
|
711
922
|
}
|
|
712
923
|
function validateEvent(event) {
|
|
924
|
+
if (event.force && (event.type !== 'click' || !event.target || 'x' in event.target)) {
|
|
925
|
+
throw new StructuredToolError('INVALID_ARGUMENT', 'force 仅支持带元素 target 的 click 事件,不能使用坐标或省略 target', '请指定 CSS、role、文本或其他元素定位方式', { actionExecuted: false });
|
|
926
|
+
}
|
|
713
927
|
if (event.commands && event.commands.length > 0 && event.type !== 'keydown') {
|
|
714
928
|
throw new Error(`events[].commands 只能用于 keydown 事件,当前事件类型为 ${event.type},如需触发编辑命令,请把 commands 放在 keydown 事件上`);
|
|
715
929
|
}
|
|
@@ -802,12 +1016,17 @@ const unifiedInputHandlers = {
|
|
|
802
1016
|
drag: handleUnifiedDrag,
|
|
803
1017
|
};
|
|
804
1018
|
async function handleUnifiedKeyDown({ unifiedSession, timeout, frame }, event) {
|
|
805
|
-
|
|
1019
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1020
|
+
if (unifiedSession.isExtensionConnected() &&
|
|
1021
|
+
unifiedSession.getInputMode() === 'stealth' &&
|
|
1022
|
+
event.commands &&
|
|
1023
|
+
event.commands.length > 0) {
|
|
806
1024
|
throw new Error('commands 参数不支持 stealth 输入模式,请先调用 manage action=inputMode inputMode=precise 切换后重试');
|
|
807
1025
|
}
|
|
808
1026
|
if (event.commands && event.commands.length > 0 && event.target) {
|
|
809
|
-
await focusTargetForCommands(unifiedSession, event.target,
|
|
1027
|
+
await focusTargetForCommands(unifiedSession, event.target, remainingInputTimeout(deadline), frame);
|
|
810
1028
|
}
|
|
1029
|
+
remainingInputTimeout(deadline);
|
|
811
1030
|
await unifiedSession.keyDown(event.key, event.commands);
|
|
812
1031
|
return undefined;
|
|
813
1032
|
}
|
|
@@ -815,42 +1034,45 @@ async function handleUnifiedKeyUp({ unifiedSession }, event) {
|
|
|
815
1034
|
await unifiedSession.keyUp(event.key);
|
|
816
1035
|
return undefined;
|
|
817
1036
|
}
|
|
818
|
-
async function handleUnifiedClick({ unifiedSession, timeout, frame }, event) {
|
|
1037
|
+
async function handleUnifiedClick({ unifiedSession, humanize, timeout, frame }, event) {
|
|
1038
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
819
1039
|
const button = event.button ?? 'left';
|
|
820
1040
|
const clickCount = event.clickCount ?? 1;
|
|
821
1041
|
if (!event.target) {
|
|
822
1042
|
await unifiedSession.mouseClick(button, clickCount);
|
|
823
1043
|
return undefined;
|
|
824
1044
|
}
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
1045
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, remainingInputTimeout(deadline), frame, !event.force);
|
|
1046
|
+
if (humanize) {
|
|
1047
|
+
for (const p of generateBezierPath(unifiedSession.getMousePosition(), point, event.steps)) {
|
|
1048
|
+
await unifiedSession.mouseMove(p.x, p.y, remainingInputTimeout(deadline));
|
|
1049
|
+
const remaining = remainingInputTimeout(deadline);
|
|
1050
|
+
await randomDelay(Math.min(getMouseMoveDelay(), remaining), Math.min(getMouseMoveDelay() * 2, remaining));
|
|
1051
|
+
}
|
|
830
1052
|
}
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
}
|
|
841
|
-
return undefined;
|
|
1053
|
+
else {
|
|
1054
|
+
await unifiedSession.mouseMove(point.x, point.y, remainingInputTimeout(deadline));
|
|
1055
|
+
}
|
|
1056
|
+
remainingInputTimeout(deadline);
|
|
1057
|
+
if (point.refId) {
|
|
1058
|
+
const result = await unifiedSession.actionableClick(point.refId, event.force === true, button, clickCount);
|
|
1059
|
+
if (!result.success) {
|
|
1060
|
+
throw new StructuredToolError('ACTIONABILITY_FAILED', result.error || 'Click failed', result.suggestions?.[0] ??
|
|
1061
|
+
'请根据 context.rect、context.clickPoint 和 context.coveringElement 调整 target', result);
|
|
842
1062
|
}
|
|
843
1063
|
}
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1064
|
+
else {
|
|
1065
|
+
await unifiedSession.mouseClick(button, clickCount);
|
|
1066
|
+
}
|
|
847
1067
|
return undefined;
|
|
848
1068
|
}
|
|
849
1069
|
async function handleUnifiedMouseDown({ unifiedSession, timeout, frame }, event) {
|
|
1070
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
850
1071
|
if (event.target) {
|
|
851
|
-
const point = await getTargetPointExtension(unifiedSession, event.target,
|
|
852
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
1072
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, remainingInputTimeout(deadline), frame);
|
|
1073
|
+
await unifiedSession.mouseMove(point.x, point.y, remainingInputTimeout(deadline));
|
|
853
1074
|
}
|
|
1075
|
+
remainingInputTimeout(deadline);
|
|
854
1076
|
await unifiedSession.mouseDown(event.button ?? 'left');
|
|
855
1077
|
return undefined;
|
|
856
1078
|
}
|
|
@@ -859,31 +1081,37 @@ async function handleUnifiedMouseUp({ unifiedSession }, event) {
|
|
|
859
1081
|
return undefined;
|
|
860
1082
|
}
|
|
861
1083
|
async function handleUnifiedMouseMove({ unifiedSession, humanize, timeout, frame }, event) {
|
|
862
|
-
const
|
|
1084
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1085
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, remainingInputTimeout(deadline), frame);
|
|
863
1086
|
if (humanize && event.steps && event.steps > 1) {
|
|
864
1087
|
const path = generateBezierPath(unifiedSession.getMousePosition(), point, event.steps);
|
|
865
1088
|
for (const p of path) {
|
|
866
|
-
await unifiedSession.mouseMove(p.x, p.y);
|
|
867
|
-
|
|
1089
|
+
await unifiedSession.mouseMove(p.x, p.y, remainingInputTimeout(deadline));
|
|
1090
|
+
const remaining = remainingInputTimeout(deadline);
|
|
1091
|
+
await randomDelay(Math.min(getMouseMoveDelay(), remaining), Math.min(getMouseMoveDelay() * 2, remaining));
|
|
868
1092
|
}
|
|
869
1093
|
}
|
|
870
1094
|
else {
|
|
871
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
1095
|
+
await unifiedSession.mouseMove(point.x, point.y, remainingInputTimeout(deadline));
|
|
872
1096
|
}
|
|
1097
|
+
remainingInputTimeout(deadline);
|
|
873
1098
|
return undefined;
|
|
874
1099
|
}
|
|
875
1100
|
async function handleUnifiedWheel({ unifiedSession, timeout, frame }, event) {
|
|
1101
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
876
1102
|
if (event.target) {
|
|
877
1103
|
const { selector, text, xpath, nth: nthParam } = targetToFindParams(event.target);
|
|
878
|
-
const elements = await unifiedSession.find(selector, text, xpath,
|
|
1104
|
+
const elements = await unifiedSession.find(selector, text, xpath, remainingInputTimeout(deadline));
|
|
879
1105
|
const nth = nthParam ?? 0;
|
|
880
1106
|
if (elements.length > nth) {
|
|
1107
|
+
remainingInputTimeout(deadline);
|
|
881
1108
|
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[nth].refId);
|
|
882
1109
|
return undefined;
|
|
883
1110
|
}
|
|
884
|
-
const point = await getTargetPointExtension(unifiedSession, event.target,
|
|
885
|
-
await unifiedSession.mouseMove(point.x, point.y);
|
|
1111
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, remainingInputTimeout(deadline), frame);
|
|
1112
|
+
await unifiedSession.mouseMove(point.x, point.y, remainingInputTimeout(deadline));
|
|
886
1113
|
}
|
|
1114
|
+
remainingInputTimeout(deadline);
|
|
887
1115
|
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
888
1116
|
return undefined;
|
|
889
1117
|
}
|
|
@@ -901,33 +1129,38 @@ async function handleUnifiedTouchEnd({ unifiedSession }) {
|
|
|
901
1129
|
await unifiedSession.touchEnd();
|
|
902
1130
|
return undefined;
|
|
903
1131
|
}
|
|
904
|
-
async function handleUnifiedType(
|
|
1132
|
+
async function handleUnifiedType(context, event) {
|
|
1133
|
+
const { unifiedSession, humanize, timeout, frame } = context;
|
|
905
1134
|
if (event.mode === 'controlled' || event.dispatch) {
|
|
906
1135
|
await inputControlled(unifiedSession, event, timeout, frame);
|
|
907
1136
|
return undefined;
|
|
908
1137
|
}
|
|
1138
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
909
1139
|
if (event.target) {
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1140
|
+
await handleUnifiedClick({ ...context, timeout: remainingInputTimeout(deadline) }, { type: 'click', target: event.target });
|
|
1141
|
+
if (!('x' in event.target)) {
|
|
1142
|
+
await focusTargetForCommands(unifiedSession, event.target, remainingInputTimeout(deadline), frame);
|
|
1143
|
+
}
|
|
913
1144
|
}
|
|
914
1145
|
else {
|
|
915
1146
|
const hasActiveFocus = await unifiedSession.evaluate('!!document.activeElement && ' +
|
|
916
1147
|
'document.activeElement !== document.body && ' +
|
|
917
|
-
'document.activeElement !== document.documentElement');
|
|
1148
|
+
'document.activeElement !== document.documentElement', undefined, remainingInputTimeout(deadline));
|
|
918
1149
|
if (!hasActiveFocus) {
|
|
919
1150
|
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
920
1151
|
}
|
|
921
1152
|
}
|
|
922
1153
|
const delay = event.delay ?? 0;
|
|
923
1154
|
if (humanize) {
|
|
924
|
-
for (const char of event.text) {
|
|
925
|
-
await unifiedSession.typeText(char);
|
|
926
|
-
|
|
1155
|
+
for (const char of event.text.replace(/\r\n?/g, '\n')) {
|
|
1156
|
+
await unifiedSession.typeText(char, 0, remainingInputTimeout(deadline));
|
|
1157
|
+
const remaining = remainingInputTimeout(deadline);
|
|
1158
|
+
await randomDelay(Math.min(getTypingDelay(delay), remaining), Math.min(getTypingDelay(delay) * 1.5, remaining));
|
|
927
1159
|
}
|
|
1160
|
+
remainingInputTimeout(deadline);
|
|
928
1161
|
}
|
|
929
1162
|
else {
|
|
930
|
-
await unifiedSession.typeText(event.text, delay);
|
|
1163
|
+
await unifiedSession.typeText(event.text, delay, remainingInputTimeout(deadline));
|
|
931
1164
|
}
|
|
932
1165
|
return undefined;
|
|
933
1166
|
}
|
|
@@ -941,13 +1174,25 @@ async function handleUnifiedWait({ timeout }, event) {
|
|
|
941
1174
|
await new Promise((resolve) => setTimeout(resolve, event.ms));
|
|
942
1175
|
return undefined;
|
|
943
1176
|
}
|
|
944
|
-
async function handleUnifiedSelect({ unifiedSession, session, mode, timeout, frame }, event) {
|
|
945
|
-
|
|
1177
|
+
async function handleUnifiedSelect({ unifiedSession, session, mode, timeout, frame, privacy }, event) {
|
|
1178
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1179
|
+
const sensitiveBefore = privacy?.sensitive ?? false;
|
|
1180
|
+
if (privacy) {
|
|
1181
|
+
privacy.sensitive = true;
|
|
1182
|
+
}
|
|
1183
|
+
await focusTargetIfNeeded(unifiedSession, session, mode, event.target, remainingInputTimeout(deadline), frame);
|
|
946
1184
|
const selectionTarget = await unifiedSession.evaluate(`(() => {
|
|
947
1185
|
const el = document.activeElement;
|
|
948
1186
|
if (!(el instanceof HTMLInputElement)) return null;
|
|
949
|
-
return {
|
|
950
|
-
|
|
1187
|
+
return {
|
|
1188
|
+
tag: 'input',
|
|
1189
|
+
inputType: (el.type || 'text').toLowerCase(),
|
|
1190
|
+
value: el.type === 'password' ? undefined : String(el.value || '').slice(0, 160)
|
|
1191
|
+
};
|
|
1192
|
+
})()`, undefined, remainingInputTimeout(deadline));
|
|
1193
|
+
if (privacy) {
|
|
1194
|
+
privacy.sensitive = sensitiveBefore || selectionTarget?.inputType === 'password';
|
|
1195
|
+
}
|
|
951
1196
|
if (selectionTarget && !supportsTextSelection(selectionTarget.tag, selectionTarget.inputType)) {
|
|
952
1197
|
throw new StructuredToolError('UNSUPPORTED_SELECTION', `input[type=${selectionTarget.inputType}] 不支持文本 selection`, '请改用 type mode="controlled" 设置完整 value,replace 事件会对这类 input 使用完整 value 更新路径', {
|
|
953
1198
|
elementTag: selectionTarget.tag,
|
|
@@ -957,32 +1202,77 @@ async function handleUnifiedSelect({ unifiedSession, session, mode, timeout, fra
|
|
|
957
1202
|
recommendedMode: 'controlled',
|
|
958
1203
|
});
|
|
959
1204
|
}
|
|
960
|
-
await selectText(unifiedSession, event.find, event.target, event.nth,
|
|
1205
|
+
await selectText(unifiedSession, event.find, event.target, event.nth, remainingInputTimeout(deadline), privacy);
|
|
961
1206
|
return undefined;
|
|
962
1207
|
}
|
|
963
|
-
async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, frame }, event) {
|
|
964
|
-
|
|
1208
|
+
async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, frame, privacy }, event) {
|
|
1209
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1210
|
+
const sensitiveBefore = privacy?.sensitive ?? false;
|
|
1211
|
+
if (privacy) {
|
|
1212
|
+
privacy.sensitive = true;
|
|
1213
|
+
}
|
|
1214
|
+
await focusTargetIfNeeded(unifiedSession, session, mode, event.target, remainingInputTimeout(deadline), frame);
|
|
965
1215
|
const directResult = await unifiedSession.evaluate(`function(findText, replacementText, nth) {
|
|
966
1216
|
var el = document.activeElement;
|
|
967
1217
|
if (!(el instanceof HTMLInputElement)) return {handled: false};
|
|
968
1218
|
var type = (el.type || 'text').toLowerCase();
|
|
969
|
-
if (['text', 'search', 'tel', 'url', 'password'].indexOf(type) !== -1) return {handled: false};
|
|
1219
|
+
if (['text', 'search', 'tel', 'url', 'password'].indexOf(type) !== -1) return {handled: false, inputType: type};
|
|
970
1220
|
var current = String(el.value || '');
|
|
971
1221
|
var pos = -1;
|
|
972
1222
|
for (var i = 0; i <= nth; i++) {
|
|
973
1223
|
pos = current.indexOf(findText, pos + (i > 0 ? 1 : 0));
|
|
974
|
-
if (pos === -1)
|
|
1224
|
+
if (pos === -1) {
|
|
1225
|
+
return {
|
|
1226
|
+
handled: true,
|
|
1227
|
+
success: false,
|
|
1228
|
+
tag: 'input',
|
|
1229
|
+
inputType: type,
|
|
1230
|
+
currentValue: current.slice(0, 160),
|
|
1231
|
+
reason: 'text_not_found'
|
|
1232
|
+
};
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
if (el.disabled || el.readOnly) {
|
|
1236
|
+
return {
|
|
1237
|
+
handled: true,
|
|
1238
|
+
success: false,
|
|
1239
|
+
tag: 'input',
|
|
1240
|
+
inputType: type,
|
|
1241
|
+
currentValue: current.slice(0, 160),
|
|
1242
|
+
reason: 'not_editable'
|
|
1243
|
+
};
|
|
975
1244
|
}
|
|
976
|
-
if (el.disabled || el.readOnly) return {handled: true, success: false, tag: 'input', inputType: type, currentValue: current.slice(0, 160), reason: 'not_editable'};
|
|
977
1245
|
var requested = current.slice(0, pos) + replacementText + current.slice(pos + findText.length);
|
|
978
1246
|
var setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
|
|
979
|
-
if (!setter)
|
|
1247
|
+
if (!setter) {
|
|
1248
|
+
return {
|
|
1249
|
+
handled: true,
|
|
1250
|
+
success: false,
|
|
1251
|
+
tag: 'input',
|
|
1252
|
+
inputType: type,
|
|
1253
|
+
currentValue: current.slice(0, 160),
|
|
1254
|
+
requestedValue: requested.slice(0, 160),
|
|
1255
|
+
reason: 'no_native_setter'
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
980
1258
|
setter.call(el, requested);
|
|
981
1259
|
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
982
1260
|
el.dispatchEvent(new Event('change', {bubbles: true}));
|
|
983
1261
|
var actual = String(el.value || '');
|
|
984
|
-
return {
|
|
985
|
-
|
|
1262
|
+
return {
|
|
1263
|
+
handled: true,
|
|
1264
|
+
success: actual === requested,
|
|
1265
|
+
tag: 'input',
|
|
1266
|
+
inputType: type,
|
|
1267
|
+
currentValue: current.slice(0, 160),
|
|
1268
|
+
requestedValue: requested.slice(0, 160),
|
|
1269
|
+
actualValue: actual.slice(0, 160),
|
|
1270
|
+
reason: actual === requested ? undefined : 'value_not_applied'
|
|
1271
|
+
};
|
|
1272
|
+
}`, undefined, remainingInputTimeout(deadline), [event.find, event.text, event.nth ?? 0]);
|
|
1273
|
+
if (privacy) {
|
|
1274
|
+
privacy.sensitive = sensitiveBefore || directResult.inputType === 'password';
|
|
1275
|
+
}
|
|
986
1276
|
if (directResult.handled) {
|
|
987
1277
|
if (!directResult.success) {
|
|
988
1278
|
const valueNotApplied = directResult.reason === 'value_not_applied';
|
|
@@ -1015,7 +1305,7 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1015
1305
|
normalized: directResult.actualValue !== directResult.requestedValue,
|
|
1016
1306
|
};
|
|
1017
1307
|
}
|
|
1018
|
-
const formatted = await selectText(unifiedSession, event.find, event.target, event.nth,
|
|
1308
|
+
const formatted = await selectText(unifiedSession, event.find, event.target, event.nth, remainingInputTimeout(deadline), privacy);
|
|
1019
1309
|
let selectionConfirmed = false;
|
|
1020
1310
|
for (let i = 0; i < 25; i++) {
|
|
1021
1311
|
const hasSelection = await unifiedSession.evaluate(`(function() {
|
|
@@ -1025,12 +1315,12 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1025
1315
|
}
|
|
1026
1316
|
var sel = window.getSelection();
|
|
1027
1317
|
return sel && !sel.isCollapsed;
|
|
1028
|
-
})()
|
|
1318
|
+
})()`, undefined, remainingInputTimeout(deadline));
|
|
1029
1319
|
if (hasSelection) {
|
|
1030
1320
|
selectionConfirmed = true;
|
|
1031
1321
|
break;
|
|
1032
1322
|
}
|
|
1033
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1323
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(20, remainingInputTimeout(deadline))));
|
|
1034
1324
|
}
|
|
1035
1325
|
if (!selectionConfirmed) {
|
|
1036
1326
|
throw new Error(`选区同步失败:文本 "${event.find}" 已定位但未能建立选区,无法执行替换`);
|
|
@@ -1053,12 +1343,12 @@ async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, fr
|
|
|
1053
1343
|
if (!editable || !editable.isContentEditable) return 'readonly';
|
|
1054
1344
|
if (document.execCommand('insertText', false, replacementText)) return 'ok';
|
|
1055
1345
|
return 'fallback';
|
|
1056
|
-
}`, undefined,
|
|
1346
|
+
}`, undefined, remainingInputTimeout(deadline), [event.text]);
|
|
1057
1347
|
if (replaceResult === 'readonly') {
|
|
1058
1348
|
throw new Error(`目标元素不可编辑,已选中文本 "${event.find}" 但无法替换`);
|
|
1059
1349
|
}
|
|
1060
1350
|
if (replaceResult === 'fallback') {
|
|
1061
|
-
await unifiedSession.typeText(event.text);
|
|
1351
|
+
await unifiedSession.typeText(event.text, 0, remainingInputTimeout(deadline));
|
|
1062
1352
|
}
|
|
1063
1353
|
if (formatted) {
|
|
1064
1354
|
return `替换的文本原在 <${formatted}> 标签内,替换后格式可能丢失`;
|
|
@@ -1196,7 +1486,9 @@ async function editorAction(unifiedSession, action, event, timeout, frame) {
|
|
|
1196
1486
|
isContentEditable: !!editable.isContentEditable
|
|
1197
1487
|
} : null,
|
|
1198
1488
|
selectedText: selectedText,
|
|
1199
|
-
selectionCollapsed: isInput
|
|
1489
|
+
selectionCollapsed: isInput
|
|
1490
|
+
? active.selectionStart === active.selectionEnd
|
|
1491
|
+
: !sel || sel.isCollapsed
|
|
1200
1492
|
};
|
|
1201
1493
|
}
|
|
1202
1494
|
|
|
@@ -1219,7 +1511,16 @@ async function editorAction(unifiedSession, action, event, timeout, frame) {
|
|
|
1219
1511
|
return {success: true, command: command};
|
|
1220
1512
|
}`, undefined, timeout, [action, event.text ?? '', event.command ?? '']);
|
|
1221
1513
|
}
|
|
1222
|
-
async function getTargetPointExtension(unifiedSession, target, timeout, frame) {
|
|
1514
|
+
async function getTargetPointExtension(unifiedSession, target, timeout, frame, scroll = true) {
|
|
1515
|
+
const budget = timeout ?? DEFAULT_TIMEOUT;
|
|
1516
|
+
const deadline = Date.now() + budget;
|
|
1517
|
+
const remaining = () => {
|
|
1518
|
+
const value = deadline - Date.now();
|
|
1519
|
+
if (value <= 0) {
|
|
1520
|
+
throwInputTimeoutError(budget);
|
|
1521
|
+
}
|
|
1522
|
+
return value;
|
|
1523
|
+
};
|
|
1223
1524
|
const frameOffset = unifiedSession.getFrameOffset();
|
|
1224
1525
|
const isStealth = unifiedSession.getInputMode() === 'stealth';
|
|
1225
1526
|
// 原始坐标:用户意图为 iframe 相对坐标
|
|
@@ -1231,7 +1532,7 @@ async function getTargetPointExtension(unifiedSession, target, timeout, frame) {
|
|
|
1231
1532
|
return { x: target.x, y: target.y };
|
|
1232
1533
|
}
|
|
1233
1534
|
const { selector, text, xpath, nth: nthParam } = targetToFindParams(target);
|
|
1234
|
-
const elements = await unifiedSession.find(selector, text, xpath,
|
|
1535
|
+
const elements = await unifiedSession.find(selector, text, xpath, remaining());
|
|
1235
1536
|
const nth = nthParam ?? 0;
|
|
1236
1537
|
if (elements.length === 0) {
|
|
1237
1538
|
throw new StructuredToolError('TARGET_NOT_FOUND', `未找到目标元素: ${JSON.stringify(target)}`, '请检查 target 是否正确,或先用 extract type=state 查看 interactiveElements 候选', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout, '目标元素未找到', frame));
|
|
@@ -1247,20 +1548,15 @@ async function getTargetPointExtension(unifiedSession, target, timeout, frame) {
|
|
|
1247
1548
|
const el = ref?.deref();
|
|
1248
1549
|
if (!el) return null;
|
|
1249
1550
|
const r = el.getBoundingClientRect();
|
|
1250
|
-
if (r.top < 0 || r.bottom > window.innerHeight || r.left < 0 || r.right > window.innerWidth) {
|
|
1551
|
+
if (${scroll} && (r.top < 0 || r.bottom > window.innerHeight || r.left < 0 || r.right > window.innerWidth)) {
|
|
1251
1552
|
el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
|
|
1252
1553
|
}
|
|
1253
1554
|
const r2 = el.getBoundingClientRect();
|
|
1254
1555
|
return { x: r2.x, y: r2.y, width: r2.width, height: r2.height };
|
|
1255
|
-
})()
|
|
1256
|
-
|
|
1556
|
+
})()`, undefined, remaining());
|
|
1557
|
+
remaining();
|
|
1257
1558
|
if (!iframeRelRect) {
|
|
1258
|
-
|
|
1259
|
-
const point = { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 };
|
|
1260
|
-
if (frameOffset && isStealth) {
|
|
1261
|
-
return { x: point.x - frameOffset.x, y: point.y - frameOffset.y, refId };
|
|
1262
|
-
}
|
|
1263
|
-
return { ...point, refId };
|
|
1559
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', '目标元素引用已失效,未使用旧坐标派发事件', '请重新读取页面并确认 target', { target, refId, actionExecuted: false });
|
|
1264
1560
|
}
|
|
1265
1561
|
// 主 frame:iframeRelRect 就是父视口绝对
|
|
1266
1562
|
if (!frameOffset) {
|
|
@@ -1281,7 +1577,8 @@ async function getTargetPointExtension(unifiedSession, target, timeout, frame) {
|
|
|
1281
1577
|
// iframe + precise:消费者(chrome.debugger)需要父视口绝对,
|
|
1282
1578
|
// scrollIntoView({block:'center'}) 会 cascade 到父框架,导致 frameOffset 与父绝对 rect 都过期,
|
|
1283
1579
|
// refetch find() 让 content-handler 重新计算 frameOffset 并返回最新父绝对 rect
|
|
1284
|
-
const refreshed = await unifiedSession.find(selector, text, xpath,
|
|
1580
|
+
const refreshed = await unifiedSession.find(selector, text, xpath, remaining());
|
|
1581
|
+
remaining();
|
|
1285
1582
|
const rect = refreshed[nth]?.rect ?? elements[nth].rect;
|
|
1286
1583
|
return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2, refId };
|
|
1287
1584
|
}
|
|
@@ -1294,8 +1591,6 @@ async function executeCdpEvent(session, event, humanize, timeout) {
|
|
|
1294
1591
|
await handler({ session, humanize, timeout }, event);
|
|
1295
1592
|
}
|
|
1296
1593
|
const cdpInputHandlers = {
|
|
1297
|
-
keydown: handleCdpKeyDown,
|
|
1298
|
-
keyup: handleCdpKeyUp,
|
|
1299
1594
|
click: handleCdpClick,
|
|
1300
1595
|
mousedown: handleCdpMouseDown,
|
|
1301
1596
|
mouseup: handleCdpMouseUp,
|
|
@@ -1308,13 +1603,31 @@ const cdpInputHandlers = {
|
|
|
1308
1603
|
wait: handleCdpWait,
|
|
1309
1604
|
drag: handleCdpDrag,
|
|
1310
1605
|
};
|
|
1311
|
-
async function handleCdpKeyDown({ session }, event) {
|
|
1312
|
-
await session.keyDown(event.key, event.commands);
|
|
1313
|
-
}
|
|
1314
|
-
async function handleCdpKeyUp({ session }, event) {
|
|
1315
|
-
await session.keyUp(event.key);
|
|
1316
|
-
}
|
|
1317
1606
|
async function handleCdpClick({ session, humanize, timeout }, event) {
|
|
1607
|
+
if (event.force) {
|
|
1608
|
+
const locator = session.createLocator(event.target, { timeout });
|
|
1609
|
+
await locator.evaluateOn(`function() {
|
|
1610
|
+
if (!this.isConnected) throw new Error('目标元素已移除,未执行点击');
|
|
1611
|
+
const button = ${JSON.stringify(event.button ?? 'left')};
|
|
1612
|
+
const count = ${event.clickCount ?? 1};
|
|
1613
|
+
const codes = {left: 0, middle: 1, right: 2, back: 3, forward: 4};
|
|
1614
|
+
const masks = {left: 1, middle: 4, right: 2, back: 8, forward: 16};
|
|
1615
|
+
const rect = this.getBoundingClientRect();
|
|
1616
|
+
const base = {bubbles: true, cancelable: true, composed: true, view: window,
|
|
1617
|
+
button: codes[button], clientX: rect.x + rect.width / 2, clientY: rect.y + rect.height / 2};
|
|
1618
|
+
for (let detail = 1; detail <= count; ++detail) {
|
|
1619
|
+
const down = this.dispatchEvent(new MouseEvent('mousedown', {...base, detail, buttons: masks[button]}));
|
|
1620
|
+
if (down && button === 'left' && typeof this.focus === 'function') this.focus();
|
|
1621
|
+
this.dispatchEvent(new MouseEvent('mouseup', {...base, detail, buttons: 0}));
|
|
1622
|
+
const type = button === 'left' ? 'click' : button === 'right' ? 'contextmenu' : 'auxclick';
|
|
1623
|
+
this.dispatchEvent(new MouseEvent(type, {...base, detail, buttons: 0}));
|
|
1624
|
+
if (button === 'left' && detail === 2) {
|
|
1625
|
+
this.dispatchEvent(new MouseEvent('dblclick', {...base, detail, buttons: 0}));
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
}`);
|
|
1629
|
+
return;
|
|
1630
|
+
}
|
|
1318
1631
|
if (event.target) {
|
|
1319
1632
|
await moveToTarget(session, event.target, humanize, timeout);
|
|
1320
1633
|
}
|
|
@@ -1348,18 +1661,23 @@ async function handleCdpTouchStart({ session, timeout }, event) {
|
|
|
1348
1661
|
await session.touchStart(point.x, point.y);
|
|
1349
1662
|
}
|
|
1350
1663
|
async function handleCdpTouchMove({ session, humanize, timeout }, event) {
|
|
1351
|
-
const
|
|
1664
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1665
|
+
const point = await getTargetPoint(session, event.target, remainingInputTimeout(deadline));
|
|
1352
1666
|
if (humanize && event.steps && event.steps > 1) {
|
|
1353
1667
|
const current = session.getBehaviorSimulator().getCurrentPosition();
|
|
1354
1668
|
const path = generateBezierPath(current, point, event.steps);
|
|
1355
1669
|
for (const p of path) {
|
|
1670
|
+
remainingInputTimeout(deadline);
|
|
1356
1671
|
await session.touchMove(p.x, p.y);
|
|
1357
|
-
|
|
1672
|
+
const remaining = remainingInputTimeout(deadline);
|
|
1673
|
+
await randomDelay(Math.min(5, remaining), Math.min(15, remaining));
|
|
1358
1674
|
}
|
|
1359
1675
|
}
|
|
1360
1676
|
else {
|
|
1677
|
+
remainingInputTimeout(deadline);
|
|
1361
1678
|
await session.touchMove(point.x, point.y);
|
|
1362
1679
|
}
|
|
1680
|
+
remainingInputTimeout(deadline);
|
|
1363
1681
|
}
|
|
1364
1682
|
async function handleCdpTouchEnd({ session }) {
|
|
1365
1683
|
await session.touchEnd();
|
|
@@ -1368,28 +1686,34 @@ async function handleCdpType({ session, humanize, timeout }, event) {
|
|
|
1368
1686
|
if (event.dispatch || event.mode === 'controlled') {
|
|
1369
1687
|
throw new Error('controlled/dispatch 输入需要 Extension 连接,当前为 CDP 模式');
|
|
1370
1688
|
}
|
|
1689
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1371
1690
|
if (event.target) {
|
|
1372
|
-
await moveToTarget(session, event.target, humanize,
|
|
1691
|
+
await moveToTarget(session, event.target, humanize, remainingInputTimeout(deadline), undefined, 'input');
|
|
1373
1692
|
await session.mouseDown('left');
|
|
1374
1693
|
await session.mouseUp('left');
|
|
1694
|
+
if (!('x' in event.target)) {
|
|
1695
|
+
await focusCdpTarget(session, event.target, remainingInputTimeout(deadline));
|
|
1696
|
+
}
|
|
1375
1697
|
}
|
|
1376
1698
|
else {
|
|
1377
1699
|
const hasActiveFocus = await session.evaluate('!!document.activeElement && ' +
|
|
1378
1700
|
'document.activeElement !== document.body && ' +
|
|
1379
|
-
'document.activeElement !== document.documentElement');
|
|
1701
|
+
'document.activeElement !== document.documentElement', undefined, remainingInputTimeout(deadline));
|
|
1380
1702
|
if (!hasActiveFocus) {
|
|
1381
1703
|
throw new Error('type 事件在无 target 时需要页面已有焦点元素,请提供 target 或先 click 目标元素');
|
|
1382
1704
|
}
|
|
1383
1705
|
}
|
|
1384
1706
|
const delay = event.delay ?? 0;
|
|
1385
1707
|
if (humanize) {
|
|
1386
|
-
for (const char of event.text) {
|
|
1387
|
-
await session.type(char);
|
|
1388
|
-
|
|
1708
|
+
for (const char of event.text.replace(/\r\n?/g, '\n')) {
|
|
1709
|
+
await session.type(char, 0, remainingInputTimeout(deadline));
|
|
1710
|
+
const remaining = remainingInputTimeout(deadline);
|
|
1711
|
+
await randomDelay(Math.min(getTypingDelay(delay), remaining), Math.min(getTypingDelay(delay) * 1.5, remaining));
|
|
1389
1712
|
}
|
|
1713
|
+
remainingInputTimeout(deadline);
|
|
1390
1714
|
}
|
|
1391
1715
|
else {
|
|
1392
|
-
await session.type(event.text, delay);
|
|
1716
|
+
await session.type(event.text, delay, remainingInputTimeout(deadline));
|
|
1393
1717
|
}
|
|
1394
1718
|
}
|
|
1395
1719
|
async function handleCdpWait({ timeout }, event) {
|
|
@@ -1403,14 +1727,16 @@ async function handleCdpDrag() {
|
|
|
1403
1727
|
* 移动到目标位置
|
|
1404
1728
|
*/
|
|
1405
1729
|
async function moveToTarget(session, target, humanize, timeout, steps, waitType = 'click') {
|
|
1406
|
-
const
|
|
1730
|
+
const deadline = Date.now() + (timeout ?? DEFAULT_TIMEOUT);
|
|
1731
|
+
const point = await getTargetPoint(session, target, remainingInputTimeout(deadline), waitType);
|
|
1407
1732
|
const simulator = session.getBehaviorSimulator();
|
|
1408
1733
|
if (humanize) {
|
|
1409
1734
|
// 人类化鼠标移动(贝塞尔曲线)
|
|
1410
1735
|
const path = generateBezierPath(simulator.getCurrentPosition(), point, steps);
|
|
1411
1736
|
for (const p of path) {
|
|
1412
|
-
await session.mouseMove(p.x, p.y);
|
|
1413
|
-
|
|
1737
|
+
await session.mouseMove(p.x, p.y, remainingInputTimeout(deadline));
|
|
1738
|
+
const remaining = remainingInputTimeout(deadline);
|
|
1739
|
+
await randomDelay(Math.min(getMouseMoveDelay(), remaining), Math.min(getMouseMoveDelay() * 2, remaining));
|
|
1414
1740
|
}
|
|
1415
1741
|
}
|
|
1416
1742
|
else if (steps && steps > 1) {
|
|
@@ -1420,13 +1746,14 @@ async function moveToTarget(session, target, humanize, timeout, steps, waitType
|
|
|
1420
1746
|
const t = i / steps;
|
|
1421
1747
|
const x = current.x + (point.x - current.x) * t;
|
|
1422
1748
|
const y = current.y + (point.y - current.y) * t;
|
|
1423
|
-
await session.mouseMove(x, y);
|
|
1749
|
+
await session.mouseMove(x, y, remainingInputTimeout(deadline));
|
|
1424
1750
|
}
|
|
1425
1751
|
}
|
|
1426
1752
|
else {
|
|
1427
1753
|
// 直接移动
|
|
1428
|
-
await session.mouseMove(point.x, point.y);
|
|
1754
|
+
await session.mouseMove(point.x, point.y, remainingInputTimeout(deadline));
|
|
1429
1755
|
}
|
|
1756
|
+
remainingInputTimeout(deadline);
|
|
1430
1757
|
}
|
|
1431
1758
|
/**
|
|
1432
1759
|
* 获取目标点坐标(带自动等待)
|
|
@@ -1436,12 +1763,21 @@ async function getTargetPoint(session, target, timeout, waitType = 'click') {
|
|
|
1436
1763
|
if ('x' in target && 'y' in target) {
|
|
1437
1764
|
return { x: target.x, y: target.y };
|
|
1438
1765
|
}
|
|
1766
|
+
const budget = timeout ?? DEFAULT_TIMEOUT;
|
|
1767
|
+
const deadline = Date.now() + budget;
|
|
1768
|
+
const remaining = () => {
|
|
1769
|
+
const value = deadline - Date.now();
|
|
1770
|
+
if (value <= 0) {
|
|
1771
|
+
throwInputTimeoutError(budget);
|
|
1772
|
+
}
|
|
1773
|
+
return value;
|
|
1774
|
+
};
|
|
1439
1775
|
// 使用 Locator 定位元素
|
|
1440
|
-
const locator = session.createLocator(target, { timeout });
|
|
1776
|
+
const locator = session.createLocator(target, { timeout: remaining() });
|
|
1441
1777
|
const nodeId = await locator.find();
|
|
1442
1778
|
// 根据操作类型执行自动等待
|
|
1443
1779
|
if (waitType !== 'none') {
|
|
1444
|
-
const autoWait = session.createAutoWait({ timeout });
|
|
1780
|
+
const autoWait = session.createAutoWait({ timeout: remaining() });
|
|
1445
1781
|
if (waitType === 'click') {
|
|
1446
1782
|
await autoWait.waitForClickable(nodeId);
|
|
1447
1783
|
}
|
|
@@ -1449,14 +1785,26 @@ async function getTargetPoint(session, target, timeout, waitType = 'click') {
|
|
|
1449
1785
|
await autoWait.waitForInputReady(nodeId);
|
|
1450
1786
|
}
|
|
1451
1787
|
}
|
|
1452
|
-
|
|
1788
|
+
remaining();
|
|
1789
|
+
const point = await locator.getClickablePoint();
|
|
1790
|
+
remaining();
|
|
1791
|
+
return point;
|
|
1453
1792
|
}
|
|
1454
1793
|
/**
|
|
1455
1794
|
* 注册 input 工具
|
|
1456
1795
|
*/
|
|
1457
1796
|
export function registerInputTool(server) {
|
|
1458
1797
|
server.registerTool('input', {
|
|
1459
|
-
description:
|
|
1798
|
+
description: [
|
|
1799
|
+
'键鼠输入:键盘、鼠标及任意组合。默认优先使用已连接的 Chrome Extension;',
|
|
1800
|
+
'只有 Extension 不可用或 browse(action="connect", port=…) 已显式选择 CDP 时才使用 CDP。',
|
|
1801
|
+
'访问 Cloudflare 时默认不传 diagnostics,Extension 先被动等待真实 tab 标题恢复,',
|
|
1802
|
+
'持续存在可点击 Turnstile 时才点击。同页测试按钮走页面点击;',
|
|
1803
|
+
'Extension inspect 走 isolated world,不先挂 debugger。',
|
|
1804
|
+
'Cloudflare iframe 或宿主页验证按钮只在已聚焦窗口的活动受控 tab 上用系统鼠标,',
|
|
1805
|
+
'Linux 用窗口原点加边距,沿路径移动、悬停后再按下抬起,混合窗口不会调用 focusWindow',
|
|
1806
|
+
].join('') +
|
|
1807
|
+
`
|
|
1460
1808
|
|
|
1461
1809
|
推荐操作顺序:
|
|
1462
1810
|
1. 先用 extract type=state 或 type=html 了解页面结构
|