@pyrokine/mcp-chrome 2.3.2 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -13
- package/README_zh.md +30 -11
- package/dist/anti-detection/behavior.d.ts.map +1 -1
- package/dist/anti-detection/behavior.js.map +1 -1
- package/dist/anti-detection/injection.js.map +1 -1
- package/dist/cdp/client.d.ts.map +1 -1
- package/dist/cdp/client.js.map +1 -1
- package/dist/cdp/launcher.d.ts.map +1 -1
- package/dist/cdp/launcher.js.map +1 -1
- package/dist/core/auto-wait.d.ts.map +1 -1
- package/dist/core/auto-wait.js.map +1 -1
- package/dist/core/browser-driver.d.ts +6 -1
- package/dist/core/browser-driver.d.ts.map +1 -1
- package/dist/core/browser-driver.js.map +1 -1
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js.map +1 -1
- package/dist/core/extension-errors.d.ts.map +1 -1
- package/dist/core/locator.d.ts.map +1 -1
- package/dist/core/locator.js.map +1 -1
- package/dist/core/retry.js.map +1 -1
- package/dist/core/session.d.ts +2 -2
- package/dist/core/session.d.ts.map +1 -1
- package/dist/core/session.js +1 -1
- package/dist/core/session.js.map +1 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/core/unified-session.d.ts +17 -2
- package/dist/core/unified-session.d.ts.map +1 -1
- package/dist/core/unified-session.js +44 -6
- package/dist/core/unified-session.js.map +1 -1
- package/dist/extension/bridge.d.ts +8 -2
- package/dist/extension/bridge.d.ts.map +1 -1
- package/dist/extension/bridge.js +5 -1
- package/dist/extension/bridge.js.map +1 -1
- package/dist/extension/http-server.d.ts +17 -0
- package/dist/extension/http-server.d.ts.map +1 -1
- package/dist/extension/http-server.js +63 -8
- package/dist/extension/http-server.js.map +1 -1
- package/dist/extension/index.d.ts +1 -0
- package/dist/extension/index.d.ts.map +1 -1
- package/dist/extension/index.js.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/tools/browse.d.ts.map +1 -1
- package/dist/tools/browse.js +13 -37
- package/dist/tools/browse.js.map +1 -1
- package/dist/tools/cookies.js.map +1 -1
- package/dist/tools/diagnostics.d.ts +31 -0
- package/dist/tools/diagnostics.d.ts.map +1 -0
- package/dist/tools/diagnostics.js +93 -0
- package/dist/tools/diagnostics.js.map +1 -0
- package/dist/tools/evaluate.d.ts +15 -0
- package/dist/tools/evaluate.d.ts.map +1 -1
- package/dist/tools/evaluate.js +136 -84
- package/dist/tools/evaluate.js.map +1 -1
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +36 -17
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/input.d.ts +6 -0
- package/dist/tools/input.d.ts.map +1 -1
- package/dist/tools/input.js +394 -248
- package/dist/tools/input.js.map +1 -1
- package/dist/tools/logs.d.ts +17 -0
- package/dist/tools/logs.d.ts.map +1 -1
- package/dist/tools/logs.js +41 -10
- package/dist/tools/logs.js.map +1 -1
- package/dist/tools/network-sanitizer.d.ts +15 -0
- package/dist/tools/network-sanitizer.d.ts.map +1 -0
- package/dist/tools/network-sanitizer.js +61 -0
- package/dist/tools/network-sanitizer.js.map +1 -0
- package/dist/tools/post-condition.d.ts +18 -1
- package/dist/tools/post-condition.d.ts.map +1 -1
- package/dist/tools/post-condition.js +52 -94
- package/dist/tools/post-condition.js.map +1 -1
- package/dist/tools/target-diagnostics.d.ts +31 -0
- package/dist/tools/target-diagnostics.d.ts.map +1 -0
- package/dist/tools/target-diagnostics.js +91 -0
- package/dist/tools/target-diagnostics.js.map +1 -0
- package/dist/tools/wait.d.ts.map +1 -1
- package/dist/tools/wait.js +183 -209
- package/dist/tools/wait.js.map +1 -1
- package/extension/dist/assets/content.ts-Cj_iiU2s.js +25 -0
- package/extension/dist/assets/{index.ts-CJXGbOcp.js → index.ts-GG6dPUPX.js} +569 -347
- package/extension/dist/manifest.json +3 -3
- package/extension/dist/service-worker-loader.js +1 -1
- package/package.json +14 -10
- package/extension/dist/assets/content.ts-pEASbjEL.js +0 -6
package/dist/tools/input.js
CHANGED
|
@@ -11,9 +11,38 @@
|
|
|
11
11
|
import { z } from 'zod';
|
|
12
12
|
import { generateBezierPath, getMouseMoveDelay, getTypingDelay, randomDelay } from '../anti-detection/index.js';
|
|
13
13
|
import { formatErrorResponse, formatResponse, getSession, getUnifiedSession } from '../core/index.js';
|
|
14
|
+
import { appendDiagnostics, finishDiagnostics, startDiagnostics } from './diagnostics.js';
|
|
14
15
|
import { postConditionSchema, waitForPostCondition } from './post-condition.js';
|
|
15
16
|
import { targetToFindParams, targetZodSchema } from './schema.js';
|
|
17
|
+
import { buildTargetDiagnostics } from './target-diagnostics.js';
|
|
16
18
|
const INPUT_WAIT_MAX_MS = 60_000;
|
|
19
|
+
const TEXT_SELECTION_INPUT_TYPES = new Set(['text', 'search', 'tel', 'url', 'password']);
|
|
20
|
+
export function supportsTextSelection(tag, inputType) {
|
|
21
|
+
return (tag.toLowerCase() === 'textarea' ||
|
|
22
|
+
(tag.toLowerCase() === 'input' && TEXT_SELECTION_INPUT_TYPES.has((inputType ?? 'text').toLowerCase())));
|
|
23
|
+
}
|
|
24
|
+
export function replaceNthOccurrence(value, find, replacement, nth = 0) {
|
|
25
|
+
let index = -1;
|
|
26
|
+
for (let occurrence = 0; occurrence <= nth; occurrence++) {
|
|
27
|
+
index = value.indexOf(find, index + (occurrence > 0 ? 1 : 0));
|
|
28
|
+
if (index === -1)
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
return value.slice(0, index) + replacement + value.slice(index + find.length);
|
|
32
|
+
}
|
|
33
|
+
export function redactSensitiveInputData(value, sensitiveValues) {
|
|
34
|
+
const secrets = [...new Set(sensitiveValues.filter((item) => item.length > 0))].sort((left, right) => right.length - left.length);
|
|
35
|
+
if (typeof value === 'string') {
|
|
36
|
+
return secrets.reduce((text, secret) => text.replaceAll(secret, '[REDACTED]'), value);
|
|
37
|
+
}
|
|
38
|
+
if (Array.isArray(value)) {
|
|
39
|
+
return value.map((item) => redactSensitiveInputData(item, secrets));
|
|
40
|
+
}
|
|
41
|
+
if (value && typeof value === 'object') {
|
|
42
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, redactSensitiveInputData(item, secrets)]));
|
|
43
|
+
}
|
|
44
|
+
return value;
|
|
45
|
+
}
|
|
17
46
|
/**
|
|
18
47
|
* InputEvent schema
|
|
19
48
|
*/
|
|
@@ -44,7 +73,7 @@ const inputEventSchema = z.object({
|
|
|
44
73
|
commands: z
|
|
45
74
|
.array(z.string())
|
|
46
75
|
.optional()
|
|
47
|
-
.describe('浏览器编辑命令(keydown 专用),如 ["selectAll"]、["copy"]、["paste"]、["cut"]、["undo"]、["redo"]
|
|
76
|
+
.describe('浏览器编辑命令(keydown 专用),如 ["selectAll"]、["copy"]、["paste"]、["cut"]、["undo"]、["redo"];成功表示命令已分发,页面结果需用 postCondition 或 extract 验证')
|
|
48
77
|
.describe('用于跨平台快捷键场景,需要 inputMode=precise'),
|
|
49
78
|
button: z.enum(['left', 'middle', 'right', 'back', 'forward']).optional().describe('鼠标按钮'),
|
|
50
79
|
clickCount: z
|
|
@@ -128,40 +157,92 @@ class StructuredToolError extends Error {
|
|
|
128
157
|
};
|
|
129
158
|
}
|
|
130
159
|
}
|
|
160
|
+
function sanitizeInputFailureResponse(response, events) {
|
|
161
|
+
const sensitiveValues = events
|
|
162
|
+
.flatMap((event) => [event.find, event.text])
|
|
163
|
+
.filter((value) => !!value);
|
|
164
|
+
if (sensitiveValues.length === 0) {
|
|
165
|
+
return response;
|
|
166
|
+
}
|
|
167
|
+
for (const item of response.content) {
|
|
168
|
+
try {
|
|
169
|
+
item.text = JSON.stringify(redactSensitiveInputData(JSON.parse(item.text), sensitiveValues), null, 2);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
item.text = redactSensitiveInputData(item.text, sensitiveValues);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return response;
|
|
176
|
+
}
|
|
131
177
|
/**
|
|
132
178
|
* input 工具处理器
|
|
133
179
|
*/
|
|
134
180
|
async function handleInput(args) {
|
|
135
181
|
try {
|
|
136
182
|
const unifiedSession = getUnifiedSession();
|
|
137
|
-
const mode = unifiedSession.getMode();
|
|
138
183
|
const humanize = args.humanize ?? false;
|
|
139
184
|
return await unifiedSession.withTabId(args.tabId, async () => {
|
|
140
185
|
return await unifiedSession.withFrame(args.frame, async () => {
|
|
141
|
-
const
|
|
186
|
+
const mode = unifiedSession.getMode();
|
|
187
|
+
const diagnostics = await startDiagnostics(unifiedSession, args.diagnostics);
|
|
142
188
|
const warnings = [];
|
|
143
189
|
const eventResults = [];
|
|
144
|
-
const session = mode === '
|
|
190
|
+
const session = mode === 'cdp' ? getSession() : undefined;
|
|
145
191
|
const inputStartedAt = Date.now();
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
192
|
+
let eventsExecuted = 0;
|
|
193
|
+
try {
|
|
194
|
+
for (const event of args.events) {
|
|
195
|
+
const eventTimeout = args.timeout === undefined
|
|
196
|
+
? undefined
|
|
197
|
+
: Math.max(0, args.timeout - (Date.now() - inputStartedAt));
|
|
198
|
+
if (eventTimeout !== undefined && eventTimeout <= 0) {
|
|
199
|
+
throw new Error(`input timeout before event dispatch (${args.timeout}ms)`);
|
|
200
|
+
}
|
|
201
|
+
const result = await executeInputEvent({ unifiedSession, session, mode, humanize, timeout: eventTimeout, frame: args.frame }, event);
|
|
202
|
+
++eventsExecuted;
|
|
203
|
+
if (typeof result === 'string') {
|
|
204
|
+
warnings.push(result);
|
|
205
|
+
}
|
|
206
|
+
else if (result) {
|
|
207
|
+
eventResults.push(result);
|
|
208
|
+
}
|
|
156
209
|
}
|
|
157
|
-
|
|
158
|
-
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
const response = formatErrorResponse(error);
|
|
213
|
+
const text = response.content[0]?.text;
|
|
214
|
+
if (text) {
|
|
215
|
+
try {
|
|
216
|
+
const payload = JSON.parse(text);
|
|
217
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
218
|
+
const timedOut = /timeout|超时/i.test(message);
|
|
219
|
+
const preActionTimeout = /timeout before event dispatch|超过 input 剩余 timeout/i.test(message);
|
|
220
|
+
const actionMayHaveExecuted = timedOut && !preActionTimeout;
|
|
221
|
+
payload.eventsExecuted = eventsExecuted;
|
|
222
|
+
payload.actionExecuted = eventsExecuted > 0 || actionMayHaveExecuted;
|
|
223
|
+
payload.actionStatus = actionMayHaveExecuted ? 'unknown' : 'failed';
|
|
224
|
+
payload.verificationRequested = Boolean(args.postCondition);
|
|
225
|
+
payload.verificationStatus = 'unavailable';
|
|
226
|
+
payload.failureStage = 'action';
|
|
227
|
+
payload.retryable = /timeout|超时|disconnect|未连接|context|debugger/i.test(message);
|
|
228
|
+
appendDiagnostics(payload, await finishDiagnostics(unifiedSession, diagnostics));
|
|
229
|
+
response.content[0].text = JSON.stringify(payload, null, 2);
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
// 保留原始错误响应
|
|
233
|
+
}
|
|
159
234
|
}
|
|
235
|
+
return sanitizeInputFailureResponse(response, args.events);
|
|
160
236
|
}
|
|
161
237
|
const result = {
|
|
162
238
|
success: true,
|
|
163
|
-
eventsExecuted
|
|
239
|
+
eventsExecuted,
|
|
164
240
|
mode,
|
|
241
|
+
actionExecuted: true,
|
|
242
|
+
actionStatus: 'completed',
|
|
243
|
+
verificationRequested: Boolean(args.postCondition),
|
|
244
|
+
verificationStatus: 'unavailable',
|
|
245
|
+
retryable: false,
|
|
165
246
|
};
|
|
166
247
|
if (warnings.length > 0) {
|
|
167
248
|
result.warnings = warnings;
|
|
@@ -170,17 +251,46 @@ async function handleInput(args) {
|
|
|
170
251
|
result.eventResults = eventResults;
|
|
171
252
|
}
|
|
172
253
|
if (args.postCondition) {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
result.
|
|
254
|
+
const postCondition = await waitForPostCondition(unifiedSession, args.postCondition, 'input');
|
|
255
|
+
result.postCondition = postCondition;
|
|
256
|
+
result.verificationStatus = postCondition.verificationStatus;
|
|
257
|
+
result.retryable = postCondition.retryable;
|
|
258
|
+
if (postCondition.verificationStatus !== 'matched') {
|
|
259
|
+
result.success = false;
|
|
260
|
+
result.failureStage = 'verification';
|
|
261
|
+
}
|
|
177
262
|
}
|
|
263
|
+
appendDiagnostics(result, await finishDiagnostics(unifiedSession, diagnostics));
|
|
178
264
|
return formatResponse(result);
|
|
179
265
|
}); // withFrame
|
|
180
266
|
}); // withTabId
|
|
181
267
|
}
|
|
182
268
|
catch (error) {
|
|
183
|
-
|
|
269
|
+
const response = formatErrorResponse(error);
|
|
270
|
+
const text = response.content[0]?.text;
|
|
271
|
+
if (text) {
|
|
272
|
+
try {
|
|
273
|
+
const payload = JSON.parse(text);
|
|
274
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
275
|
+
Object.assign(payload, {
|
|
276
|
+
actionExecuted: false,
|
|
277
|
+
actionStatus: 'failed',
|
|
278
|
+
verificationRequested: Boolean(args.postCondition),
|
|
279
|
+
verificationStatus: 'unavailable',
|
|
280
|
+
failureStage: 'action',
|
|
281
|
+
retryable: /timeout|超时|disconnect|未连接|context|debugger|attach/i.test(message),
|
|
282
|
+
diagnosticsStatus: args.diagnostics ? 'unavailable' : 'disabled',
|
|
283
|
+
});
|
|
284
|
+
if (args.diagnostics) {
|
|
285
|
+
payload.diagnosticsError = 'diagnostics 未能在 tab/frame 初始化失败前启动';
|
|
286
|
+
}
|
|
287
|
+
response.content[0].text = JSON.stringify(payload, null, 2);
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
// 保留原始错误响应
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return sanitizeInputFailureResponse(response, args.events);
|
|
184
294
|
}
|
|
185
295
|
}
|
|
186
296
|
const unifiedOnlyEventTypes = new Set([
|
|
@@ -192,7 +302,14 @@ const unifiedOnlyEventTypes = new Set([
|
|
|
192
302
|
]);
|
|
193
303
|
async function executeInputEvent(context, event) {
|
|
194
304
|
if (context.mode === 'extension' || unifiedOnlyEventTypes.has(event.type)) {
|
|
195
|
-
return executeUnifiedEvent(
|
|
305
|
+
return executeUnifiedEvent({
|
|
306
|
+
unifiedSession: context.unifiedSession,
|
|
307
|
+
session: context.session,
|
|
308
|
+
mode: context.mode,
|
|
309
|
+
humanize: context.humanize,
|
|
310
|
+
timeout: context.timeout,
|
|
311
|
+
frame: context.frame,
|
|
312
|
+
}, event);
|
|
196
313
|
}
|
|
197
314
|
if (!context.session) {
|
|
198
315
|
throw new Error('CDP 输入事件缺少 session');
|
|
@@ -200,27 +317,6 @@ async function executeInputEvent(context, event) {
|
|
|
200
317
|
await executeCdpEvent(context.session, event, context.humanize, context.timeout);
|
|
201
318
|
return undefined;
|
|
202
319
|
}
|
|
203
|
-
async function captureDiagnosticsStart(unifiedSession) {
|
|
204
|
-
await unifiedSession.enableConsole();
|
|
205
|
-
await unifiedSession.enableNetwork();
|
|
206
|
-
const consoleLogs = await unifiedSession.getConsoleLogs();
|
|
207
|
-
const network = await unifiedSession.getNetworkRequests();
|
|
208
|
-
return { consoleCount: consoleLogs.length, networkCount: network.length };
|
|
209
|
-
}
|
|
210
|
-
async function captureDiagnosticsDelta(unifiedSession, start) {
|
|
211
|
-
const consoleLogs = await unifiedSession.getConsoleLogs();
|
|
212
|
-
const network = await unifiedSession.getNetworkRequests();
|
|
213
|
-
return {
|
|
214
|
-
console: consoleLogs
|
|
215
|
-
.slice(start.consoleCount)
|
|
216
|
-
.filter((item) => ['error', 'warning', 'warn'].includes(item.level))
|
|
217
|
-
.slice(-20),
|
|
218
|
-
failedRequests: network
|
|
219
|
-
.slice(start.networkCount)
|
|
220
|
-
.filter((item) => item.errorText || (item.status !== undefined && item.status >= 400))
|
|
221
|
-
.slice(-20),
|
|
222
|
-
};
|
|
223
|
-
}
|
|
224
320
|
/**
|
|
225
321
|
* 通过真实鼠标事件选中页面文本
|
|
226
322
|
*
|
|
@@ -234,15 +330,20 @@ async function captureDiagnosticsDelta(unifiedSession, start) {
|
|
|
234
330
|
* 如果 target 是选择器类型,先通过 actionableClick 聚焦
|
|
235
331
|
* select/replace 事件用,保证选区建立前 activeElement 就是目标
|
|
236
332
|
*/
|
|
237
|
-
async function focusTargetForCommands(unifiedSession, target, timeout) {
|
|
333
|
+
async function focusTargetForCommands(unifiedSession, target, timeout, frame) {
|
|
238
334
|
if ('x' in target || 'y' in target) {
|
|
239
|
-
|
|
335
|
+
const point = await getTargetPointExtension(unifiedSession, target, timeout, frame);
|
|
336
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
337
|
+
await unifiedSession.mouseClick('left');
|
|
338
|
+
return;
|
|
240
339
|
}
|
|
241
340
|
const params = targetToFindParams(target);
|
|
242
341
|
const els = await unifiedSession.find(params.selector, params.text, params.xpath, timeout);
|
|
243
342
|
const nth0 = params.nth ?? 0;
|
|
244
343
|
if (els.length <= nth0) {
|
|
245
|
-
|
|
344
|
+
throw new StructuredToolError(els.length === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', els.length === 0
|
|
345
|
+
? `未找到命令目标: ${JSON.stringify(target)}`
|
|
346
|
+
: `第 ${nth0} 个命令目标不存在(共 ${els.length} 个)`, '请检查 target、frame 和 target.nth,或先用 extract type=state 查看当前候选元素', await buildTargetDiagnosticContext(unifiedSession, target, els.length, nth0, timeout, '命令目标未找到', frame));
|
|
246
347
|
}
|
|
247
348
|
const focused = await unifiedSession.evaluate(`(() => {
|
|
248
349
|
const ref = window.__mcpElementMap?.[${JSON.stringify(els[nth0].refId)}]
|
|
@@ -253,27 +354,56 @@ async function focusTargetForCommands(unifiedSession, target, timeout) {
|
|
|
253
354
|
el.focus()
|
|
254
355
|
return document.activeElement === el
|
|
255
356
|
})()`, undefined, timeout);
|
|
256
|
-
|
|
357
|
+
if (focused !== true) {
|
|
358
|
+
throw new StructuredToolError('TARGET_FOCUS_FAILED', '命令目标未成功聚焦', '请检查目标是否可编辑、未 disabled 并处于正确 frame', await buildTargetDiagnosticContext(unifiedSession, target, els.length, nth0, timeout, '命令目标聚焦失败', frame));
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
export async function focusCdpTarget(session, target, timeout) {
|
|
362
|
+
const locator = session.createLocator(target, timeout !== undefined ? { timeout } : undefined);
|
|
363
|
+
const focused = await locator.evaluateOn(`function() {
|
|
364
|
+
if (!(this instanceof HTMLElement)) return false;
|
|
365
|
+
this.focus({preventScroll: true});
|
|
366
|
+
return document.activeElement === this;
|
|
367
|
+
}`);
|
|
368
|
+
if (!focused) {
|
|
369
|
+
throw new StructuredToolError('TARGET_FOCUS_FAILED', '目标元素未成功聚焦', '请检查目标是否可编辑、未 disabled,并确认 locator 指向 input、textarea 或 contenteditable 元素', { target });
|
|
370
|
+
}
|
|
257
371
|
}
|
|
258
|
-
async function focusTargetIfNeeded(unifiedSession, target,
|
|
259
|
-
if (!target
|
|
372
|
+
async function focusTargetIfNeeded(unifiedSession, session, mode, target, timeout, frame) {
|
|
373
|
+
if (!target) {
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
if ('x' in target || 'y' in target) {
|
|
377
|
+
const point = await getTargetPointExtension(unifiedSession, target, timeout, frame);
|
|
378
|
+
await unifiedSession.mouseMove(point.x, point.y);
|
|
379
|
+
await unifiedSession.mouseClick('left');
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
if (mode === 'cdp') {
|
|
383
|
+
if (!session) {
|
|
384
|
+
throw new Error('CDP locator 聚焦缺少 session');
|
|
385
|
+
}
|
|
386
|
+
await focusCdpTarget(session, target, timeout);
|
|
260
387
|
return;
|
|
261
388
|
}
|
|
262
389
|
const params = targetToFindParams(target);
|
|
263
390
|
const els = await unifiedSession.find(params.selector, params.text, params.xpath, timeout);
|
|
264
|
-
const nth0 = params.nth ??
|
|
265
|
-
if (els.length
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
391
|
+
const nth0 = params.nth ?? 0;
|
|
392
|
+
if (els.length <= nth0) {
|
|
393
|
+
throw new StructuredToolError(els.length === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', els.length === 0
|
|
394
|
+
? `未找到目标元素: ${JSON.stringify(target)}`
|
|
395
|
+
: `第 ${nth0} 个匹配元素不存在(共 ${els.length} 个)`, '请检查 target、frame 和 target.nth,或先用 extract type=state 查看当前候选元素', await buildTargetDiagnosticContext(unifiedSession, target, els.length, nth0, timeout, '聚焦目标未找到', frame));
|
|
396
|
+
}
|
|
397
|
+
const result = await unifiedSession.actionableClick(els[nth0].refId);
|
|
398
|
+
if (!result.success) {
|
|
399
|
+
throw new StructuredToolError('ACTIONABILITY_FAILED', result.error || '目标元素无法聚焦', result.suggestions?.[0] ?? '请检查元素是否可见、未被遮挡并处于正确 frame', {
|
|
400
|
+
...result,
|
|
401
|
+
...(await buildTargetDiagnosticContext(unifiedSession, target, els.length, nth0, timeout, result.error || '目标元素无法聚焦', frame)),
|
|
402
|
+
});
|
|
273
403
|
}
|
|
274
404
|
}
|
|
275
|
-
async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget,
|
|
276
|
-
return await unifiedSession.evaluate(`function(findText,
|
|
405
|
+
async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, occurrenceNth, scopeNth, scopeSelector, scopeText, scopeXpath, timeout) {
|
|
406
|
+
return await unifiedSession.evaluate(`function(findText, occurrenceNth, scopeNth, scopeTarget, scopeSelector, scopeText, scopeXpath) {
|
|
277
407
|
function selectorFor(el) {
|
|
278
408
|
if (!(el instanceof Element)) return null;
|
|
279
409
|
if (el.id) return '#' + CSS.escape(el.id);
|
|
@@ -298,13 +428,20 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
298
428
|
if (!(el instanceof Element)) return null;
|
|
299
429
|
var rect = el.getBoundingClientRect();
|
|
300
430
|
var value = 'value' in el ? String(el.value || '') : '';
|
|
301
|
-
var
|
|
431
|
+
var inputType = el instanceof HTMLInputElement ? (el.type || 'text').toLowerCase() : undefined;
|
|
432
|
+
var sensitive = inputType === 'password';
|
|
433
|
+
var text = sensitive ? (el.innerText || el.textContent || '') : (el.innerText || el.textContent || value || '');
|
|
434
|
+
text = text.replace(new RegExp('\\\\s+', 'g'), ' ').trim();
|
|
302
435
|
return {
|
|
303
436
|
tag: el.tagName.toLowerCase(),
|
|
304
437
|
id: el.id || undefined,
|
|
305
438
|
selector: selectorFor(el),
|
|
306
|
-
|
|
307
|
-
|
|
439
|
+
inputType: inputType,
|
|
440
|
+
text: text ? text.slice(0, 160) : undefined,
|
|
441
|
+
valuePreview: !sensitive && value ? value.slice(0, 160) : undefined,
|
|
442
|
+
valueLength: value.length,
|
|
443
|
+
hasValue: value.length > 0,
|
|
444
|
+
redacted: sensitive || undefined,
|
|
308
445
|
visible: rect.width > 0 && rect.height > 0,
|
|
309
446
|
disabled: Boolean(el.disabled),
|
|
310
447
|
readOnly: Boolean(el.readOnly),
|
|
@@ -314,22 +451,21 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
314
451
|
}
|
|
315
452
|
var root = document.body;
|
|
316
453
|
if (scopeXpath) {
|
|
317
|
-
var xr = document.evaluate(scopeXpath, document, null, XPathResult.
|
|
318
|
-
root = xr.
|
|
454
|
+
var xr = document.evaluate(scopeXpath, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
455
|
+
root = xr.snapshotItem(scopeNth);
|
|
319
456
|
} else if (scopeSelector) {
|
|
320
|
-
var candidates = document.querySelectorAll(scopeSelector);
|
|
457
|
+
var candidates = Array.from(document.querySelectorAll(scopeSelector));
|
|
321
458
|
if (scopeText) {
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
}
|
|
325
|
-
} else {
|
|
326
|
-
root = candidates[0];
|
|
459
|
+
candidates = candidates.filter(function(candidate) {
|
|
460
|
+
return (candidate.textContent || '').includes(scopeText);
|
|
461
|
+
});
|
|
327
462
|
}
|
|
463
|
+
root = candidates[scopeNth];
|
|
328
464
|
} else if (scopeText) {
|
|
329
|
-
var all = document.querySelectorAll('*')
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
465
|
+
var all = Array.from(document.querySelectorAll('*')).filter(function(candidate) {
|
|
466
|
+
return (candidate.textContent || '').includes(scopeText);
|
|
467
|
+
});
|
|
468
|
+
root = all[scopeNth];
|
|
333
469
|
}
|
|
334
470
|
var candidateElements = [];
|
|
335
471
|
if (root instanceof Element) {
|
|
@@ -337,12 +473,14 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
337
473
|
candidateElements = candidateElements.concat(Array.from(root.querySelectorAll('input, textarea, [contenteditable="true"], [role="textbox"], button, a')).slice(0, 12));
|
|
338
474
|
}
|
|
339
475
|
var active = document.activeElement;
|
|
476
|
+
var activeIsPassword = active instanceof HTMLInputElement && (active.type || 'text').toLowerCase() === 'password';
|
|
340
477
|
var selection = window.getSelection();
|
|
341
478
|
return {
|
|
342
479
|
scope: {
|
|
343
480
|
target: scopeTarget,
|
|
344
|
-
|
|
345
|
-
|
|
481
|
+
targetNth: scopeNth,
|
|
482
|
+
occurrenceNth: occurrenceNth,
|
|
483
|
+
findLength: findText.length,
|
|
346
484
|
selector: scopeSelector,
|
|
347
485
|
text: scopeText,
|
|
348
486
|
xpath: scopeXpath,
|
|
@@ -350,52 +488,57 @@ async function collectTextSelectionDiagnostics(unifiedSession, findText, scopeTa
|
|
|
350
488
|
},
|
|
351
489
|
activeElement: summarizeElement(active),
|
|
352
490
|
selection: {
|
|
353
|
-
text: selection ? selection.toString().slice(0, 160) : '',
|
|
491
|
+
text: activeIsPassword ? undefined : (selection ? selection.toString().slice(0, 160) : ''),
|
|
492
|
+
textLength: selection ? selection.toString().length : 0,
|
|
493
|
+
redacted: activeIsPassword || undefined,
|
|
354
494
|
collapsed: selection ? selection.isCollapsed : true,
|
|
355
495
|
anchorNode: selection && selection.anchorNode ? selection.anchorNode.nodeName : null,
|
|
356
496
|
focusNode: selection && selection.focusNode ? selection.focusNode.nodeName : null,
|
|
357
497
|
inputSelection: active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA') ? {
|
|
358
498
|
selectionStart: active.selectionStart,
|
|
359
499
|
selectionEnd: active.selectionEnd,
|
|
360
|
-
valuePreview: String(active.value || '').slice(0, 160)
|
|
500
|
+
valuePreview: activeIsPassword ? undefined : String(active.value || '').slice(0, 160),
|
|
501
|
+
valueLength: String(active.value || '').length,
|
|
502
|
+
redacted: activeIsPassword || undefined
|
|
361
503
|
} : undefined
|
|
362
504
|
},
|
|
363
505
|
candidates: candidateElements.map(summarizeElement).filter(Boolean)
|
|
364
506
|
};
|
|
365
|
-
}`, undefined, timeout, [findText,
|
|
507
|
+
}`, undefined, timeout, [findText, occurrenceNth, scopeNth, scopeTarget ?? null, scopeSelector, scopeText, scopeXpath]);
|
|
366
508
|
}
|
|
367
509
|
async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeout) {
|
|
368
510
|
// 将 target 转为查询参数,传入注入脚本进行 DOM 查询
|
|
369
511
|
let scopeSelector = null;
|
|
370
512
|
let scopeText = null;
|
|
371
513
|
let scopeXpath = null;
|
|
514
|
+
let scopeNth = 0;
|
|
372
515
|
if (scopeTarget && !('x' in scopeTarget) && !('y' in scopeTarget)) {
|
|
373
516
|
const params = targetToFindParams(scopeTarget);
|
|
374
517
|
scopeSelector = params.selector ?? null;
|
|
375
518
|
scopeText = params.text ?? null;
|
|
376
519
|
scopeXpath = params.xpath ?? null;
|
|
520
|
+
scopeNth = params.nth ?? 0;
|
|
377
521
|
}
|
|
378
522
|
// Step 1: 注入脚本定位文本
|
|
379
|
-
const result = await unifiedSession.evaluate(`function(findText, nth, scopeSelector, scopeText, scopeXpath) {
|
|
523
|
+
const result = await unifiedSession.evaluate(`function(findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath) {
|
|
380
524
|
// 确定搜索根节点
|
|
381
525
|
var root = document.body;
|
|
382
526
|
if (scopeXpath) {
|
|
383
|
-
var xr = document.evaluate(scopeXpath, document, null, XPathResult.
|
|
384
|
-
root = xr.
|
|
527
|
+
var xr = document.evaluate(scopeXpath, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
528
|
+
root = xr.snapshotItem(scopeNth);
|
|
385
529
|
} else if (scopeSelector) {
|
|
386
|
-
var candidates = document.querySelectorAll(scopeSelector);
|
|
530
|
+
var candidates = Array.from(document.querySelectorAll(scopeSelector));
|
|
387
531
|
if (scopeText) {
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
}
|
|
391
|
-
} else {
|
|
392
|
-
root = candidates[0];
|
|
532
|
+
candidates = candidates.filter(function(candidate) {
|
|
533
|
+
return (candidate.textContent || '').includes(scopeText);
|
|
534
|
+
});
|
|
393
535
|
}
|
|
536
|
+
root = candidates[scopeNth];
|
|
394
537
|
} else if (scopeText) {
|
|
395
|
-
var all = document.querySelectorAll('*')
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
538
|
+
var all = Array.from(document.querySelectorAll('*')).filter(function(candidate) {
|
|
539
|
+
return (candidate.textContent || '').includes(scopeText);
|
|
540
|
+
});
|
|
541
|
+
root = all[scopeNth];
|
|
399
542
|
}
|
|
400
543
|
if (!root) return {type: 'noscope'};
|
|
401
544
|
|
|
@@ -403,6 +546,11 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
403
546
|
var tag = root.tagName;
|
|
404
547
|
if (tag === 'INPUT' || tag === 'TEXTAREA') {
|
|
405
548
|
var val = root.value || '';
|
|
549
|
+
var inputType = tag === 'INPUT' ? (root.type || 'text').toLowerCase() : 'textarea';
|
|
550
|
+
var selectionCapable = tag === 'TEXTAREA' || ['text', 'search', 'tel', 'url', 'password'].indexOf(inputType) !== -1;
|
|
551
|
+
if (!selectionCapable) {
|
|
552
|
+
return {type: 'unsupported', tag: tag.toLowerCase(), inputType: inputType, currentValue: String(val).slice(0, 160)};
|
|
553
|
+
}
|
|
406
554
|
var pos = -1;
|
|
407
555
|
for (var n = 0; n <= nth; n++) {
|
|
408
556
|
pos = val.indexOf(findText, pos + (n > 0 ? 1 : 0));
|
|
@@ -410,9 +558,7 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
410
558
|
}
|
|
411
559
|
// 原子化:定位到 input 同时完成 focus + setSelectionRange,避免外层 mouseClick 聚焦不可靠
|
|
412
560
|
root.focus();
|
|
413
|
-
|
|
414
|
-
root.setSelectionRange(pos, pos + findText.length);
|
|
415
|
-
}
|
|
561
|
+
root.setSelectionRange(pos, pos + findText.length);
|
|
416
562
|
return {type: 'input', selectionStart: pos, selectionEnd: pos + findText.length};
|
|
417
563
|
}
|
|
418
564
|
|
|
@@ -421,16 +567,19 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
421
567
|
for (var k = 0; k < inputs.length; k++) {
|
|
422
568
|
var inp = inputs[k];
|
|
423
569
|
var v = inp.value || '';
|
|
570
|
+
var childType = inp.tagName === 'INPUT' ? (inp.type || 'text').toLowerCase() : 'textarea';
|
|
571
|
+
var childSelectionCapable = inp.tagName === 'TEXTAREA' || ['text', 'search', 'tel', 'url', 'password'].indexOf(childType) !== -1;
|
|
424
572
|
var ip = -1;
|
|
425
573
|
for (var n2 = 0; n2 <= nth; n2++) {
|
|
426
574
|
ip = v.indexOf(findText, ip + (n2 > 0 ? 1 : 0));
|
|
427
575
|
if (ip === -1) break;
|
|
428
576
|
}
|
|
577
|
+
if (ip !== -1 && !childSelectionCapable) {
|
|
578
|
+
return {type: 'unsupported', tag: inp.tagName.toLowerCase(), inputType: childType, currentValue: String(v).slice(0, 160)};
|
|
579
|
+
}
|
|
429
580
|
if (ip !== -1) {
|
|
430
581
|
inp.focus();
|
|
431
|
-
|
|
432
|
-
inp.setSelectionRange(ip, ip + findText.length);
|
|
433
|
-
}
|
|
582
|
+
inp.setSelectionRange(ip, ip + findText.length);
|
|
434
583
|
return {type: 'input', selectionStart: ip, selectionEnd: ip + findText.length};
|
|
435
584
|
}
|
|
436
585
|
}
|
|
@@ -487,6 +636,19 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
487
636
|
formatted = startNode.tagName.toLowerCase();
|
|
488
637
|
}
|
|
489
638
|
|
|
639
|
+
// contenteditable 使用 DOM Range 建立精确选区;CDP 的 Shift+Click 会把整个编辑区选中。
|
|
640
|
+
if (root instanceof HTMLElement && root.isContentEditable) {
|
|
641
|
+
var selection = window.getSelection();
|
|
642
|
+
if (!selection) return {type: 'notfound'};
|
|
643
|
+
var selectionRange = document.createRange();
|
|
644
|
+
selectionRange.setStart(s.node, s.offset);
|
|
645
|
+
selectionRange.setEnd(e.node, Math.min(e.offset + 1, e.node.textContent.length));
|
|
646
|
+
root.focus();
|
|
647
|
+
selection.removeAllRanges();
|
|
648
|
+
selection.addRange(selectionRange);
|
|
649
|
+
return {type: 'range', formatted: formatted || undefined};
|
|
650
|
+
}
|
|
651
|
+
|
|
490
652
|
return {
|
|
491
653
|
type: 'coords',
|
|
492
654
|
startX: sr.x + 1,
|
|
@@ -495,13 +657,22 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
495
657
|
endY: er.y + er.height / 2,
|
|
496
658
|
formatted: formatted || undefined
|
|
497
659
|
};
|
|
498
|
-
}`, undefined, timeout, [findText, nth, scopeSelector, scopeText, scopeXpath]);
|
|
660
|
+
}`, undefined, timeout, [findText, nth, scopeNth, scopeSelector, scopeText, scopeXpath]);
|
|
499
661
|
if (!result || result.type === 'noscope') {
|
|
500
|
-
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
662
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeNth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
501
663
|
throw new StructuredToolError('TEXT_SCOPE_NOT_FOUND', `未找到目标元素: ${JSON.stringify(scopeTarget)}`, '请检查 target 是否能定位到包含目标文本的元素,或先用 extract(state) 查看可交互元素', diagnostics);
|
|
502
664
|
}
|
|
665
|
+
if (result.type === 'unsupported') {
|
|
666
|
+
throw new StructuredToolError('UNSUPPORTED_SELECTION', `${result.tag}[type=${result.inputType}] 不支持文本 selection`, 'select 只支持 textarea 和 text/search/tel/url/password input;请使用 type mode="controlled" 设置完整 value', {
|
|
667
|
+
elementTag: result.tag,
|
|
668
|
+
inputType: result.inputType,
|
|
669
|
+
currentValue: result.currentValue,
|
|
670
|
+
requestedFind: findText,
|
|
671
|
+
recommendedMode: 'controlled',
|
|
672
|
+
});
|
|
673
|
+
}
|
|
503
674
|
if (result.type === 'notfound') {
|
|
504
|
-
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
675
|
+
const diagnostics = await collectTextSelectionDiagnostics(unifiedSession, findText, scopeTarget, nth, scopeNth, scopeSelector, scopeText, scopeXpath, timeout);
|
|
505
676
|
throw new StructuredToolError('TEXT_NOT_FOUND', scopeTarget
|
|
506
677
|
? `目标元素内未找到文本 "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`
|
|
507
678
|
: `未找到文本: "${findText}"${nth > 0 ? `(第 ${nth} 个匹配)` : ''}`, '请检查 find 文本、nth 序号和 target 范围;context.candidates 提供了当前范围内的候选文本和 selector', diagnostics);
|
|
@@ -510,6 +681,9 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
510
681
|
// 注入脚本已完成 focus + setSelectionRange(原子化,避免外层 mouseClick 聚焦不可靠)
|
|
511
682
|
return undefined;
|
|
512
683
|
}
|
|
684
|
+
if (result.type === 'range') {
|
|
685
|
+
return result.formatted;
|
|
686
|
+
}
|
|
513
687
|
// DOM 文本节点:鼠标选择
|
|
514
688
|
const coords = result;
|
|
515
689
|
// iframe 坐标修正(precise 模式需要视口绝对坐标)
|
|
@@ -529,59 +703,6 @@ async function selectText(unifiedSession, findText, scopeTarget, nth = 0, timeou
|
|
|
529
703
|
await unifiedSession.keyUp('Shift');
|
|
530
704
|
return coords.formatted;
|
|
531
705
|
}
|
|
532
|
-
async function executeEditingCommands(unifiedSession, commands, timeout) {
|
|
533
|
-
const result = await unifiedSession.evaluate(`function(commands) {
|
|
534
|
-
function summarizeSelection() {
|
|
535
|
-
var active = document.activeElement;
|
|
536
|
-
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA')) {
|
|
537
|
-
return {
|
|
538
|
-
type: active.tagName.toLowerCase(),
|
|
539
|
-
selectionStart: active.selectionStart,
|
|
540
|
-
selectionEnd: active.selectionEnd,
|
|
541
|
-
selectedText: String(active.value || '').slice(active.selectionStart || 0, active.selectionEnd || 0)
|
|
542
|
-
};
|
|
543
|
-
}
|
|
544
|
-
var selection = window.getSelection();
|
|
545
|
-
return {
|
|
546
|
-
type: 'document',
|
|
547
|
-
text: selection ? selection.toString() : '',
|
|
548
|
-
collapsed: selection ? selection.isCollapsed : true
|
|
549
|
-
};
|
|
550
|
-
}
|
|
551
|
-
function commandSelectAll() {
|
|
552
|
-
var active = document.activeElement;
|
|
553
|
-
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA') && typeof active.select === 'function') {
|
|
554
|
-
active.select();
|
|
555
|
-
active.dispatchEvent(new Event('select', {bubbles: true}));
|
|
556
|
-
return true;
|
|
557
|
-
}
|
|
558
|
-
if (active && active.isContentEditable) {
|
|
559
|
-
var range = document.createRange();
|
|
560
|
-
range.selectNodeContents(active);
|
|
561
|
-
var selection = window.getSelection();
|
|
562
|
-
if (!selection) return false;
|
|
563
|
-
selection.removeAllRanges();
|
|
564
|
-
selection.addRange(range);
|
|
565
|
-
return true;
|
|
566
|
-
}
|
|
567
|
-
return document.execCommand('selectAll');
|
|
568
|
-
}
|
|
569
|
-
var executed = commands.map(function(command) {
|
|
570
|
-
if (command === 'selectAll') {
|
|
571
|
-
return {command: command, success: commandSelectAll()};
|
|
572
|
-
}
|
|
573
|
-
if (['copy', 'cut', 'paste', 'undo', 'redo'].indexOf(command) !== -1) {
|
|
574
|
-
return {command: command, success: document.execCommand(command)};
|
|
575
|
-
}
|
|
576
|
-
return {command: command, success: false};
|
|
577
|
-
});
|
|
578
|
-
return {success: executed.every(function(item) { return item.success; }), executed: executed, selection: summarizeSelection()};
|
|
579
|
-
}`, undefined, timeout, [commands]);
|
|
580
|
-
if (!result.success) {
|
|
581
|
-
throw new StructuredToolError('EDIT_COMMAND_FAILED', `编辑命令执行失败: ${commands.join(', ')}`, '请确认当前页面已有可编辑焦点,或给 keydown 事件提供 target 参数', result);
|
|
582
|
-
}
|
|
583
|
-
return result;
|
|
584
|
-
}
|
|
585
706
|
/**
|
|
586
707
|
* 验证事件参数(两种执行模式共享),避免在 Extension/CDP 两个 switch 中重复校验
|
|
587
708
|
*/
|
|
@@ -652,13 +773,13 @@ function validateEvent(event) {
|
|
|
652
773
|
break;
|
|
653
774
|
}
|
|
654
775
|
}
|
|
655
|
-
async function executeUnifiedEvent(
|
|
776
|
+
async function executeUnifiedEvent(context, event) {
|
|
656
777
|
validateEvent(event);
|
|
657
778
|
const handler = unifiedInputHandlers[event.type];
|
|
658
779
|
if (!handler) {
|
|
659
780
|
throw new Error(`未知事件类型: ${event.type}`);
|
|
660
781
|
}
|
|
661
|
-
return handler(
|
|
782
|
+
return handler(context, event);
|
|
662
783
|
}
|
|
663
784
|
const unifiedInputHandlers = {
|
|
664
785
|
keydown: handleUnifiedKeyDown,
|
|
@@ -680,27 +801,21 @@ const unifiedInputHandlers = {
|
|
|
680
801
|
editorCommand: handleUnifiedEditorCommand,
|
|
681
802
|
drag: handleUnifiedDrag,
|
|
682
803
|
};
|
|
683
|
-
async function handleUnifiedKeyDown({ unifiedSession, timeout }, event) {
|
|
804
|
+
async function handleUnifiedKeyDown({ unifiedSession, timeout, frame }, event) {
|
|
684
805
|
if (unifiedSession.getInputMode() === 'stealth' && event.commands && event.commands.length > 0) {
|
|
685
806
|
throw new Error('commands 参数不支持 stealth 输入模式,请先调用 manage action=inputMode inputMode=precise 切换后重试');
|
|
686
807
|
}
|
|
687
808
|
if (event.commands && event.commands.length > 0 && event.target) {
|
|
688
|
-
|
|
689
|
-
if (!focused) {
|
|
690
|
-
throw new Error('commands 目标未找到或未成功聚焦');
|
|
691
|
-
}
|
|
809
|
+
await focusTargetForCommands(unifiedSession, event.target, timeout, frame);
|
|
692
810
|
}
|
|
693
811
|
await unifiedSession.keyDown(event.key, event.commands);
|
|
694
|
-
if (event.commands && event.commands.length > 0) {
|
|
695
|
-
return await executeEditingCommands(unifiedSession, event.commands, timeout);
|
|
696
|
-
}
|
|
697
812
|
return undefined;
|
|
698
813
|
}
|
|
699
814
|
async function handleUnifiedKeyUp({ unifiedSession }, event) {
|
|
700
815
|
await unifiedSession.keyUp(event.key);
|
|
701
816
|
return undefined;
|
|
702
817
|
}
|
|
703
|
-
async function handleUnifiedClick({ unifiedSession, timeout }, event) {
|
|
818
|
+
async function handleUnifiedClick({ unifiedSession, timeout, frame }, event) {
|
|
704
819
|
const button = event.button ?? 'left';
|
|
705
820
|
const clickCount = event.clickCount ?? 1;
|
|
706
821
|
if (!event.target) {
|
|
@@ -708,7 +823,7 @@ async function handleUnifiedClick({ unifiedSession, timeout }, event) {
|
|
|
708
823
|
return undefined;
|
|
709
824
|
}
|
|
710
825
|
if ('x' in event.target && 'y' in event.target) {
|
|
711
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
826
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
712
827
|
await unifiedSession.mouseMove(point.x, point.y);
|
|
713
828
|
await unifiedSession.mouseClick(button, clickCount);
|
|
714
829
|
return undefined;
|
|
@@ -726,14 +841,14 @@ async function handleUnifiedClick({ unifiedSession, timeout }, event) {
|
|
|
726
841
|
return undefined;
|
|
727
842
|
}
|
|
728
843
|
}
|
|
729
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
844
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
730
845
|
await unifiedSession.mouseMove(point.x, point.y);
|
|
731
846
|
await unifiedSession.mouseClick(button, clickCount, typeof point.refId === 'string' ? point.refId : undefined);
|
|
732
847
|
return undefined;
|
|
733
848
|
}
|
|
734
|
-
async function handleUnifiedMouseDown({ unifiedSession, timeout }, event) {
|
|
849
|
+
async function handleUnifiedMouseDown({ unifiedSession, timeout, frame }, event) {
|
|
735
850
|
if (event.target) {
|
|
736
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
851
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
737
852
|
await unifiedSession.mouseMove(point.x, point.y);
|
|
738
853
|
}
|
|
739
854
|
await unifiedSession.mouseDown(event.button ?? 'left');
|
|
@@ -743,8 +858,8 @@ async function handleUnifiedMouseUp({ unifiedSession }, event) {
|
|
|
743
858
|
await unifiedSession.mouseUp(event.button ?? 'left');
|
|
744
859
|
return undefined;
|
|
745
860
|
}
|
|
746
|
-
async function handleUnifiedMouseMove({ unifiedSession, humanize, timeout }, event) {
|
|
747
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
861
|
+
async function handleUnifiedMouseMove({ unifiedSession, humanize, timeout, frame }, event) {
|
|
862
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
748
863
|
if (humanize && event.steps && event.steps > 1) {
|
|
749
864
|
const path = generateBezierPath(unifiedSession.getMousePosition(), point, event.steps);
|
|
750
865
|
for (const p of path) {
|
|
@@ -757,7 +872,7 @@ async function handleUnifiedMouseMove({ unifiedSession, humanize, timeout }, eve
|
|
|
757
872
|
}
|
|
758
873
|
return undefined;
|
|
759
874
|
}
|
|
760
|
-
async function handleUnifiedWheel({ unifiedSession, timeout }, event) {
|
|
875
|
+
async function handleUnifiedWheel({ unifiedSession, timeout, frame }, event) {
|
|
761
876
|
if (event.target) {
|
|
762
877
|
const { selector, text, xpath, nth: nthParam } = targetToFindParams(event.target);
|
|
763
878
|
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
@@ -766,19 +881,19 @@ async function handleUnifiedWheel({ unifiedSession, timeout }, event) {
|
|
|
766
881
|
await unifiedSession.scroll(event.deltaX ?? 0, event.deltaY ?? 0, elements[nth].refId);
|
|
767
882
|
return undefined;
|
|
768
883
|
}
|
|
769
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
884
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
770
885
|
await unifiedSession.mouseMove(point.x, point.y);
|
|
771
886
|
}
|
|
772
887
|
await unifiedSession.mouseWheel(event.deltaX ?? 0, event.deltaY ?? 0);
|
|
773
888
|
return undefined;
|
|
774
889
|
}
|
|
775
|
-
async function handleUnifiedTouchStart({ unifiedSession, timeout }, event) {
|
|
776
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
890
|
+
async function handleUnifiedTouchStart({ unifiedSession, timeout, frame }, event) {
|
|
891
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
777
892
|
await unifiedSession.touchStart(point.x, point.y);
|
|
778
893
|
return undefined;
|
|
779
894
|
}
|
|
780
|
-
async function handleUnifiedTouchMove({ unifiedSession, timeout }, event) {
|
|
781
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
895
|
+
async function handleUnifiedTouchMove({ unifiedSession, timeout, frame }, event) {
|
|
896
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
782
897
|
await unifiedSession.touchMove(point.x, point.y);
|
|
783
898
|
return undefined;
|
|
784
899
|
}
|
|
@@ -786,13 +901,13 @@ async function handleUnifiedTouchEnd({ unifiedSession }) {
|
|
|
786
901
|
await unifiedSession.touchEnd();
|
|
787
902
|
return undefined;
|
|
788
903
|
}
|
|
789
|
-
async function handleUnifiedType({ unifiedSession, humanize, timeout }, event) {
|
|
904
|
+
async function handleUnifiedType({ unifiedSession, humanize, timeout, frame }, event) {
|
|
790
905
|
if (event.mode === 'controlled' || event.dispatch) {
|
|
791
|
-
await inputControlled(unifiedSession, event, timeout);
|
|
906
|
+
await inputControlled(unifiedSession, event, timeout, frame);
|
|
792
907
|
return undefined;
|
|
793
908
|
}
|
|
794
909
|
if (event.target) {
|
|
795
|
-
const point = await getTargetPointExtension(unifiedSession, event.target, timeout);
|
|
910
|
+
const point = await getTargetPointExtension(unifiedSession, event.target, timeout, frame);
|
|
796
911
|
await unifiedSession.mouseMove(point.x, point.y);
|
|
797
912
|
await unifiedSession.mouseClick('left');
|
|
798
913
|
}
|
|
@@ -826,13 +941,80 @@ async function handleUnifiedWait({ timeout }, event) {
|
|
|
826
941
|
await new Promise((resolve) => setTimeout(resolve, event.ms));
|
|
827
942
|
return undefined;
|
|
828
943
|
}
|
|
829
|
-
async function handleUnifiedSelect({ unifiedSession, timeout }, event) {
|
|
830
|
-
await focusTargetIfNeeded(unifiedSession, event.target,
|
|
944
|
+
async function handleUnifiedSelect({ unifiedSession, session, mode, timeout, frame }, event) {
|
|
945
|
+
await focusTargetIfNeeded(unifiedSession, session, mode, event.target, timeout, frame);
|
|
946
|
+
const selectionTarget = await unifiedSession.evaluate(`(() => {
|
|
947
|
+
const el = document.activeElement;
|
|
948
|
+
if (!(el instanceof HTMLInputElement)) return null;
|
|
949
|
+
return {tag: 'input', inputType: (el.type || 'text').toLowerCase(), value: String(el.value || '').slice(0, 160)};
|
|
950
|
+
})()`, undefined, timeout);
|
|
951
|
+
if (selectionTarget && !supportsTextSelection(selectionTarget.tag, selectionTarget.inputType)) {
|
|
952
|
+
throw new StructuredToolError('UNSUPPORTED_SELECTION', `input[type=${selectionTarget.inputType}] 不支持文本 selection`, '请改用 type mode="controlled" 设置完整 value,replace 事件会对这类 input 使用完整 value 更新路径', {
|
|
953
|
+
elementTag: selectionTarget.tag,
|
|
954
|
+
inputType: selectionTarget.inputType,
|
|
955
|
+
currentValue: selectionTarget.value,
|
|
956
|
+
requestedFind: event.find,
|
|
957
|
+
recommendedMode: 'controlled',
|
|
958
|
+
});
|
|
959
|
+
}
|
|
831
960
|
await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
832
961
|
return undefined;
|
|
833
962
|
}
|
|
834
|
-
async function handleUnifiedReplace({ unifiedSession, timeout }, event) {
|
|
835
|
-
await focusTargetIfNeeded(unifiedSession, event.target,
|
|
963
|
+
async function handleUnifiedReplace({ unifiedSession, session, mode, timeout, frame }, event) {
|
|
964
|
+
await focusTargetIfNeeded(unifiedSession, session, mode, event.target, timeout, frame);
|
|
965
|
+
const directResult = await unifiedSession.evaluate(`function(findText, replacementText, nth) {
|
|
966
|
+
var el = document.activeElement;
|
|
967
|
+
if (!(el instanceof HTMLInputElement)) return {handled: false};
|
|
968
|
+
var type = (el.type || 'text').toLowerCase();
|
|
969
|
+
if (['text', 'search', 'tel', 'url', 'password'].indexOf(type) !== -1) return {handled: false};
|
|
970
|
+
var current = String(el.value || '');
|
|
971
|
+
var pos = -1;
|
|
972
|
+
for (var i = 0; i <= nth; i++) {
|
|
973
|
+
pos = current.indexOf(findText, pos + (i > 0 ? 1 : 0));
|
|
974
|
+
if (pos === -1) return {handled: true, success: false, tag: 'input', inputType: type, currentValue: current.slice(0, 160), reason: 'text_not_found'};
|
|
975
|
+
}
|
|
976
|
+
if (el.disabled || el.readOnly) return {handled: true, success: false, tag: 'input', inputType: type, currentValue: current.slice(0, 160), reason: 'not_editable'};
|
|
977
|
+
var requested = current.slice(0, pos) + replacementText + current.slice(pos + findText.length);
|
|
978
|
+
var setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
|
|
979
|
+
if (!setter) return {handled: true, success: false, tag: 'input', inputType: type, currentValue: current.slice(0, 160), requestedValue: requested.slice(0, 160), reason: 'no_native_setter'};
|
|
980
|
+
setter.call(el, requested);
|
|
981
|
+
el.dispatchEvent(new Event('input', {bubbles: true}));
|
|
982
|
+
el.dispatchEvent(new Event('change', {bubbles: true}));
|
|
983
|
+
var actual = String(el.value || '');
|
|
984
|
+
return {handled: true, success: actual === requested, tag: 'input', inputType: type, currentValue: current.slice(0, 160), requestedValue: requested.slice(0, 160), actualValue: actual.slice(0, 160), reason: actual === requested ? undefined : 'value_not_applied'};
|
|
985
|
+
}`, undefined, timeout, [event.find, event.text, event.nth ?? 0]);
|
|
986
|
+
if (directResult.handled) {
|
|
987
|
+
if (!directResult.success) {
|
|
988
|
+
const valueNotApplied = directResult.reason === 'value_not_applied';
|
|
989
|
+
throw new StructuredToolError(directResult.reason === 'text_not_found'
|
|
990
|
+
? 'TEXT_NOT_FOUND'
|
|
991
|
+
: valueNotApplied
|
|
992
|
+
? 'VALUE_NOT_APPLIED'
|
|
993
|
+
: 'UNSUPPORTED_SELECTION', directResult.reason === 'text_not_found'
|
|
994
|
+
? `目标元素内未找到文本 "${event.find}"`
|
|
995
|
+
: valueNotApplied
|
|
996
|
+
? '浏览器没有接受 replace 计算出的完整 value'
|
|
997
|
+
: '目标 input 不支持文本 selection,且无法安全更新完整 value', valueNotApplied
|
|
998
|
+
? '请根据 actualValue 提供该 input type 接受的完整值,或使用 controlled type 设置合法值'
|
|
999
|
+
: '请使用 mode="controlled" 的 type 事件设置完整 value,或改用支持文本 selection 的 input 类型', {
|
|
1000
|
+
elementTag: directResult.tag,
|
|
1001
|
+
inputType: directResult.inputType,
|
|
1002
|
+
currentValue: directResult.currentValue,
|
|
1003
|
+
requestedFind: event.find,
|
|
1004
|
+
requestedValue: directResult.requestedValue,
|
|
1005
|
+
actualValue: directResult.actualValue,
|
|
1006
|
+
recommendedMode: 'controlled',
|
|
1007
|
+
reason: directResult.reason,
|
|
1008
|
+
});
|
|
1009
|
+
}
|
|
1010
|
+
return {
|
|
1011
|
+
elementTag: directResult.tag,
|
|
1012
|
+
inputType: directResult.inputType,
|
|
1013
|
+
requestedValue: directResult.requestedValue,
|
|
1014
|
+
actualValue: directResult.actualValue,
|
|
1015
|
+
normalized: directResult.actualValue !== directResult.requestedValue,
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
836
1018
|
const formatted = await selectText(unifiedSession, event.find, event.target, event.nth, timeout);
|
|
837
1019
|
let selectionConfirmed = false;
|
|
838
1020
|
for (let i = 0; i < 25; i++) {
|
|
@@ -883,18 +1065,18 @@ async function handleUnifiedReplace({ unifiedSession, timeout }, event) {
|
|
|
883
1065
|
}
|
|
884
1066
|
return undefined;
|
|
885
1067
|
}
|
|
886
|
-
async function handleUnifiedEditorContext({ unifiedSession, timeout }, event) {
|
|
887
|
-
return (await editorAction(unifiedSession, 'context', event, timeout));
|
|
1068
|
+
async function handleUnifiedEditorContext({ unifiedSession, timeout, frame }, event) {
|
|
1069
|
+
return (await editorAction(unifiedSession, 'context', event, timeout, frame));
|
|
888
1070
|
}
|
|
889
|
-
async function handleUnifiedEditorInsert({ unifiedSession, timeout }, event) {
|
|
890
|
-
await editorAction(unifiedSession, 'insert', event, timeout);
|
|
1071
|
+
async function handleUnifiedEditorInsert({ unifiedSession, timeout, frame }, event) {
|
|
1072
|
+
await editorAction(unifiedSession, 'insert', event, timeout, frame);
|
|
891
1073
|
return undefined;
|
|
892
1074
|
}
|
|
893
|
-
async function handleUnifiedEditorCommand({ unifiedSession, timeout }, event) {
|
|
894
|
-
await editorAction(unifiedSession, 'command', event, timeout);
|
|
1075
|
+
async function handleUnifiedEditorCommand({ unifiedSession, timeout, frame }, event) {
|
|
1076
|
+
await editorAction(unifiedSession, 'command', event, timeout, frame);
|
|
895
1077
|
return undefined;
|
|
896
1078
|
}
|
|
897
|
-
async function handleUnifiedDrag({ unifiedSession, timeout }, event) {
|
|
1079
|
+
async function handleUnifiedDrag({ unifiedSession, timeout, frame }, event) {
|
|
898
1080
|
if (!event.target) {
|
|
899
1081
|
throw new Error('drag 事件需要 target 参数');
|
|
900
1082
|
}
|
|
@@ -915,10 +1097,10 @@ async function handleUnifiedDrag({ unifiedSession, timeout }, event) {
|
|
|
915
1097
|
const srcEls = await unifiedSession.find(srcParams.selector, srcParams.text, srcParams.xpath, timeout);
|
|
916
1098
|
const dstEls = await unifiedSession.find(dstParams.selector, dstParams.text, dstParams.xpath, timeout);
|
|
917
1099
|
if (srcEls.length <= srcNth) {
|
|
918
|
-
throw new
|
|
1100
|
+
throw new StructuredToolError(srcEls.length === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', `drag 源元素未找到: ${JSON.stringify(event.target)}`, '请检查 drag 源 target、frame 和 nth', await buildTargetDiagnosticContext(unifiedSession, event.target, srcEls.length, srcNth, timeout, 'drag 源元素未找到', frame));
|
|
919
1101
|
}
|
|
920
1102
|
if (dstEls.length <= dstNth) {
|
|
921
|
-
throw new
|
|
1103
|
+
throw new StructuredToolError(dstEls.length === 0 ? 'TARGET_NOT_FOUND' : 'TARGET_INDEX_OUT_OF_RANGE', `drag 目标元素未找到: ${JSON.stringify(event.to)}`, '请检查 drag 目标 to、frame 和 nth', await buildTargetDiagnosticContext(unifiedSession, event.to, dstEls.length, dstNth, timeout, 'drag 目标元素未找到', frame));
|
|
922
1104
|
}
|
|
923
1105
|
return unifiedSession.dragAndDrop(srcEls[srcNth].refId, dstEls[dstNth].refId);
|
|
924
1106
|
};
|
|
@@ -945,7 +1127,7 @@ async function handleUnifiedDrag({ unifiedSession, timeout }, event) {
|
|
|
945
1127
|
* - precise 模式直接使用
|
|
946
1128
|
* - stealth 模式需减 offset 转为 iframe 相对坐标
|
|
947
1129
|
*/
|
|
948
|
-
async function inputControlled(unifiedSession, event, timeout) {
|
|
1130
|
+
async function inputControlled(unifiedSession, event, timeout, frame) {
|
|
949
1131
|
if (!event.target) {
|
|
950
1132
|
throw new Error('controlled 输入需要 target 参数定位输入元素');
|
|
951
1133
|
}
|
|
@@ -956,62 +1138,25 @@ async function inputControlled(unifiedSession, event, timeout) {
|
|
|
956
1138
|
const elements = await unifiedSession.find(selector, searchText, xpath, timeout);
|
|
957
1139
|
const nth = nthParam ?? 0;
|
|
958
1140
|
if (elements.length === 0 || nth >= elements.length) {
|
|
959
|
-
throw new StructuredToolError('TARGET_NOT_FOUND', `controlled 输入目标未找到: ${JSON.stringify(event.target)}`, '请检查 target 是否能定位到 input、textarea、select 或 contenteditable 元素;context.candidates 提供当前页面候选元素', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout));
|
|
1141
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', `controlled 输入目标未找到: ${JSON.stringify(event.target)}`, '请检查 target 是否能定位到 input、textarea、select 或 contenteditable 元素;context.candidates 提供当前页面候选元素', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout, 'controlled 输入目标未找到', frame));
|
|
960
1142
|
}
|
|
961
1143
|
const result = await unifiedSession.dispatchInput(elements[nth].refId, event.text ?? '');
|
|
962
1144
|
if (!result.success) {
|
|
963
|
-
throw new StructuredToolError('CONTROLLED_INPUT_FAILED', result.error || 'controlled 输入失败', '请检查目标元素是否可编辑、未 disabled,并确认当前 frame 与 target 匹配', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout, result.error || 'controlled 输入失败'));
|
|
1145
|
+
throw new StructuredToolError('CONTROLLED_INPUT_FAILED', result.error || 'controlled 输入失败', '请检查目标元素是否可编辑、未 disabled,并确认当前 frame 与 target 匹配', await buildTargetDiagnosticContext(unifiedSession, event.target, elements.length, nth, timeout, result.error || 'controlled 输入失败', frame));
|
|
964
1146
|
}
|
|
965
1147
|
}
|
|
966
|
-
async function buildTargetDiagnosticContext(unifiedSession, target, matchCount, nth, timeout, reason = '目标元素未找到') {
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
activeElement: active ? {
|
|
975
|
-
tag: active.tagName.toLowerCase(),
|
|
976
|
-
id: active.id || undefined,
|
|
977
|
-
className: typeof active.className === 'string' ? active.className : undefined,
|
|
978
|
-
text: (active.textContent || '').trim().slice(0, 80),
|
|
979
|
-
value: 'value' in active ? active.value : undefined,
|
|
980
|
-
selectionStart: 'selectionStart' in active ? active.selectionStart : undefined,
|
|
981
|
-
selectionEnd: 'selectionEnd' in active ? active.selectionEnd : undefined
|
|
982
|
-
} : null,
|
|
983
|
-
selection: {
|
|
984
|
-
text: selection ? selection.toString().slice(0, 160) : '',
|
|
985
|
-
collapsed: selection ? selection.isCollapsed : true,
|
|
986
|
-
anchorNode: selection && selection.anchorNode ? selection.anchorNode.nodeName : null,
|
|
987
|
-
focusNode: selection && selection.focusNode ? selection.focusNode.nodeName : null
|
|
988
|
-
},
|
|
989
|
-
candidates: candidates.map(function(el) {
|
|
990
|
-
const rect = el.getBoundingClientRect();
|
|
991
|
-
return {
|
|
992
|
-
tag: el.tagName.toLowerCase(),
|
|
993
|
-
id: el.id || undefined,
|
|
994
|
-
role: el.getAttribute('role') || undefined,
|
|
995
|
-
text: (el.textContent || el.getAttribute('aria-label') || el.getAttribute('placeholder') || '').trim().slice(0, 80),
|
|
996
|
-
visible: rect.width > 0 && rect.height > 0,
|
|
997
|
-
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
|
|
998
|
-
bounds: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}
|
|
999
|
-
};
|
|
1000
|
-
})
|
|
1001
|
-
};
|
|
1002
|
-
})()`, undefined, timeout);
|
|
1003
|
-
}
|
|
1004
|
-
catch (err) {
|
|
1005
|
-
page = { diagnosticError: err instanceof Error ? err.message : String(err) };
|
|
1006
|
-
}
|
|
1007
|
-
return { reason, target, matchCount, nth, page };
|
|
1148
|
+
async function buildTargetDiagnosticContext(unifiedSession, target, matchCount, nth, timeout, reason = '目标元素未找到', frame) {
|
|
1149
|
+
return buildTargetDiagnostics(unifiedSession, target, {
|
|
1150
|
+
nth,
|
|
1151
|
+
timeout,
|
|
1152
|
+
frame,
|
|
1153
|
+
matchCount,
|
|
1154
|
+
lastState: reason,
|
|
1155
|
+
});
|
|
1008
1156
|
}
|
|
1009
|
-
async function editorAction(unifiedSession, action, event, timeout) {
|
|
1157
|
+
async function editorAction(unifiedSession, action, event, timeout, frame) {
|
|
1010
1158
|
if (event.target) {
|
|
1011
|
-
|
|
1012
|
-
if (!focused) {
|
|
1013
|
-
throw new Error('editor 目标未找到或未成功聚焦');
|
|
1014
|
-
}
|
|
1159
|
+
await focusTargetForCommands(unifiedSession, event.target, timeout, frame);
|
|
1015
1160
|
}
|
|
1016
1161
|
return unifiedSession.evaluate(`function(action, text, command) {
|
|
1017
1162
|
var active = document.activeElement;
|
|
@@ -1074,7 +1219,7 @@ async function editorAction(unifiedSession, action, event, timeout) {
|
|
|
1074
1219
|
return {success: true, command: command};
|
|
1075
1220
|
}`, undefined, timeout, [action, event.text ?? '', event.command ?? '']);
|
|
1076
1221
|
}
|
|
1077
|
-
async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
1222
|
+
async function getTargetPointExtension(unifiedSession, target, timeout, frame) {
|
|
1078
1223
|
const frameOffset = unifiedSession.getFrameOffset();
|
|
1079
1224
|
const isStealth = unifiedSession.getInputMode() === 'stealth';
|
|
1080
1225
|
// 原始坐标:用户意图为 iframe 相对坐标
|
|
@@ -1089,10 +1234,10 @@ async function getTargetPointExtension(unifiedSession, target, timeout) {
|
|
|
1089
1234
|
const elements = await unifiedSession.find(selector, text, xpath, timeout);
|
|
1090
1235
|
const nth = nthParam ?? 0;
|
|
1091
1236
|
if (elements.length === 0) {
|
|
1092
|
-
throw new StructuredToolError('TARGET_NOT_FOUND', `未找到目标元素: ${JSON.stringify(target)}`, '请检查 target 是否正确,或先用 extract type=state 查看 interactiveElements 候选', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout));
|
|
1237
|
+
throw new StructuredToolError('TARGET_NOT_FOUND', `未找到目标元素: ${JSON.stringify(target)}`, '请检查 target 是否正确,或先用 extract type=state 查看 interactiveElements 候选', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout, '目标元素未找到', frame));
|
|
1093
1238
|
}
|
|
1094
1239
|
if (nth >= elements.length) {
|
|
1095
|
-
throw new StructuredToolError('TARGET_INDEX_OUT_OF_RANGE', `第 ${nth} 个匹配元素不存在(共 ${elements.length} 个)`, '请降低 nth,或先用 extract type=state 查看当前匹配到的候选元素', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout, '目标元素序号越界'));
|
|
1240
|
+
throw new StructuredToolError('TARGET_INDEX_OUT_OF_RANGE', `第 ${nth} 个匹配元素不存在(共 ${elements.length} 个)`, '请降低 nth,或先用 extract type=state 查看当前匹配到的候选元素', await buildTargetDiagnosticContext(unifiedSession, target, elements.length, nth, timeout, '目标元素序号越界', frame));
|
|
1096
1241
|
}
|
|
1097
1242
|
// 视口外时滚动后重新取 rect:与 actionableClick (left+single) 行为对齐,
|
|
1098
1243
|
// 否则非左键 / 多击的坐标路径在视口外坐标 dispatch,浏览器找不到元素,事件丢失
|
|
@@ -1339,6 +1484,7 @@ export function registerInputTool(server) {
|
|
|
1339
1484
|
replace: {events:[{type:"replace",target:{css:"textarea"},find:"old",text:"new"}]}
|
|
1340
1485
|
注意:纯键盘事件(不带 commands)仅保证 JS keyboard event 可被监听,不保证触发浏览器原生编辑行为;
|
|
1341
1486
|
全选/复制/粘贴等语义用 commands;"全选并替换文本"用 select/replace 事件更简洁;
|
|
1487
|
+
commands 成功表示浏览器已接受命令分发,页面结果需用 postCondition 或 extract 验证;
|
|
1342
1488
|
commands 仅支持 inputMode=precise,stealth 模式下会报错`,
|
|
1343
1489
|
inputSchema: inputSchema,
|
|
1344
1490
|
}, (args) => handleInput(args));
|