@kkeve/suite 0.0.12 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/run.js +4 -1
- package/dist/core/generateReport/reportData.js +6 -3
- package/dist/core/global-setup.d.ts +1 -1
- package/dist/core/global-setup.js +22 -12
- package/dist/core/keve-test.js +83 -35
- package/dist/core/learned-actions.d.ts +37 -8
- package/dist/core/learned-actions.js +214 -22
- package/dist/page-agent/agent.d.ts +21 -0
- package/dist/page-agent/agent.js +282 -4
- package/dist/page-agent/system_prompt.md +70 -14
- package/dist/page-agent/tools.js +814 -17
- package/doc/skills/keve-e2e-gen/references/script-generation-guide.md +11 -0
- package/package.json +1 -1
package/dist/page-agent/tools.js
CHANGED
|
@@ -55,24 +55,376 @@ exports.tools.set('click', tool({
|
|
|
55
55
|
}),
|
|
56
56
|
execute: async function (input) {
|
|
57
57
|
const locator = this.page.locator(`aria-ref=${input.ref}`);
|
|
58
|
+
// Capture pre-click element identity (role + name) — needed for stable, replayable action logs.
|
|
59
|
+
// Without this, action records only contain ephemeral ref=e688 which is useless after page reload.
|
|
60
|
+
let elementRole = '';
|
|
61
|
+
let elementName = '';
|
|
62
|
+
let dialogContext = '';
|
|
63
|
+
let inDialog = false;
|
|
64
|
+
try {
|
|
65
|
+
const preClickInfo = await locator.evaluate((el) => {
|
|
66
|
+
const dialog = el.closest('[role="dialog"], [role="alertdialog"], .ks-dialog, .el-dialog, .ant-modal');
|
|
67
|
+
const role = el.getAttribute('role') || el.tagName.toLowerCase();
|
|
68
|
+
const name = el.textContent?.trim()?.slice(0, 40) || el.getAttribute('aria-label') || el.getAttribute('title') || '';
|
|
69
|
+
const isButton = el.tagName === 'BUTTON' || el.getAttribute('role') === 'button' || el.tagName === 'A';
|
|
70
|
+
return { inDialog: !!dialog, isButton, name, dialogRole: dialog?.getAttribute('role') || '', role, tagName: el.tagName.toLowerCase() };
|
|
71
|
+
});
|
|
72
|
+
elementRole = preClickInfo.role || preClickInfo.tagName;
|
|
73
|
+
elementName = preClickInfo.name;
|
|
74
|
+
inDialog = preClickInfo.inDialog;
|
|
75
|
+
if (preClickInfo.inDialog && preClickInfo.isButton) {
|
|
76
|
+
dialogContext = preClickInfo.name; // remember the button name for post-check
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch { /* element may not support evaluate */ }
|
|
80
|
+
// Build stable element descriptor for action logs: "button 确定", "link 导航", "textbox"
|
|
81
|
+
const elementDesc = elementRole && elementName
|
|
82
|
+
? `${elementRole} "${elementName}"`
|
|
83
|
+
: elementRole || `element`;
|
|
58
84
|
await locator.click({ timeout: 5000, force: true });
|
|
59
|
-
|
|
85
|
+
await this.page.waitForTimeout(300);
|
|
86
|
+
// Post-click check: if we clicked a dialog button, check if dialog closed
|
|
87
|
+
if (dialogContext) {
|
|
88
|
+
try {
|
|
89
|
+
const dialogStillOpen = await this.page.locator('[role="dialog"], [role="alertdialog"]').count() > 0;
|
|
90
|
+
if (dialogStillOpen) {
|
|
91
|
+
// Check if a NEW dialog/warning appeared on top (e.g. confirmation/warning popup)
|
|
92
|
+
let newDialogInfo = '';
|
|
93
|
+
try {
|
|
94
|
+
const topDialog = await this.page.locator('[role="alertdialog"], [role="dialog"]').last();
|
|
95
|
+
const title = await topDialog.locator('.ks-dialog__header, .el-dialog__title, .ant-modal-title, h2, h3').first().textContent({ timeout: 1000 }).catch(() => '');
|
|
96
|
+
const body = await topDialog.locator('.ks-dialog__body, .el-dialog__body, .ant-modal-body, p').first().textContent({ timeout: 1000 }).catch(() => '');
|
|
97
|
+
const info = (title || body || '').trim().slice(0, 80);
|
|
98
|
+
if (info)
|
|
99
|
+
newDialogInfo = ` — new popup: "${info}"`;
|
|
100
|
+
}
|
|
101
|
+
catch { /* no extra info */ }
|
|
102
|
+
// ── JS click fallback — bypass ks-overlay / pointer-events:none interception ──
|
|
103
|
+
// Pointer-events click (Playwright force:true) can be intercepted by overlay masks,
|
|
104
|
+
// but DOM element.click() dispatches a trusted click event that bypasses pointer-events.
|
|
105
|
+
// Try JS click when the pointer click failed to close the dialog.
|
|
106
|
+
try {
|
|
107
|
+
await locator.evaluate((el) => { el.click(); });
|
|
108
|
+
await this.page.waitForTimeout(500);
|
|
109
|
+
const dialogAfterJS = await this.page.locator('[role="dialog"], [role="alertdialog"]').count() > 0;
|
|
110
|
+
if (!dialogAfterJS) {
|
|
111
|
+
return `✅ Clicked [ref=${input.ref}] ${elementDesc} via JS fallback (overlay bypassed) — dialog closed`;
|
|
112
|
+
}
|
|
113
|
+
// JS click also failed — re-check for new popup info
|
|
114
|
+
let jsNewDialogInfo = '';
|
|
115
|
+
try {
|
|
116
|
+
const topDialog2 = await this.page.locator('[role="alertdialog"], [role="dialog"]').last();
|
|
117
|
+
const title2 = await topDialog2.locator('.ks-dialog__header, .el-dialog__title, .ant-modal-title, h2, h3').first().textContent({ timeout: 1000 }).catch(() => '');
|
|
118
|
+
const body2 = await topDialog2.locator('.ks-dialog__body, .el-dialog__body, .ant-modal-body, p').first().textContent({ timeout: 1000 }).catch(() => '');
|
|
119
|
+
const info2 = (title2 || body2 || '').trim().slice(0, 80);
|
|
120
|
+
if (info2)
|
|
121
|
+
jsNewDialogInfo = ` — new popup: "${info2}"`;
|
|
122
|
+
}
|
|
123
|
+
catch { /* no extra info */ }
|
|
124
|
+
const popupInfo = jsNewDialogInfo || newDialogInfo;
|
|
125
|
+
// ── Auto-dismiss non-blocking sub-dialogs ──
|
|
126
|
+
// Some sub-dialogs are unrelated interceptors that block
|
|
127
|
+
// the intended action (e.g. ownership transfer, confirmation prompts).
|
|
128
|
+
// Try to dismiss them automatically before returning.
|
|
129
|
+
const popupText = (jsNewDialogInfo || newDialogInfo || '').replace(/^ — new popup: /, '').replace(/"/g, '');
|
|
130
|
+
// Generic patterns: ownership/transfer/assignment dialogs, confirmation prompts, notifications
|
|
131
|
+
// These are NOT the target dialog — they are interceptors that should be dismissed.
|
|
132
|
+
const dismissiblePatterns = ['负责人', '转移', '转让', '确认', '提示', '通知', '选择', '分配'];
|
|
133
|
+
const isDismissible = dismissiblePatterns.some(p => popupText.includes(p));
|
|
134
|
+
if (isDismissible) {
|
|
135
|
+
try {
|
|
136
|
+
// Try Escape to close the sub-dialog, then retry the original click.
|
|
137
|
+
// If the sub-dialog was the blocker, the retry should succeed.
|
|
138
|
+
await this.page.keyboard.press('Escape');
|
|
139
|
+
await this.page.waitForTimeout(400);
|
|
140
|
+
// Retry original click — same post-click check as the normal path
|
|
141
|
+
await locator.click({ timeout: 5000, force: true });
|
|
142
|
+
await this.page.waitForTimeout(500);
|
|
143
|
+
const dialogAfterRetry = await this.page.locator('[role="dialog"], [role="alertdialog"]').count() > 0;
|
|
144
|
+
if (!dialogAfterRetry) {
|
|
145
|
+
return `✅ Clicked [ref=${input.ref}] ${elementDesc} — sub-dialog "${popupText}" auto-dismissed via Escape, retried click succeeded — dialog closed`;
|
|
146
|
+
}
|
|
147
|
+
// Retry click didn't close dialog — check if a NEW sub-dialog appeared
|
|
148
|
+
let retryPopupInfo = '';
|
|
149
|
+
try {
|
|
150
|
+
const topDialog = await this.page.locator('[role="alertdialog"], [role="dialog"]').last();
|
|
151
|
+
const retryTitle = await topDialog.locator('.ks-dialog__header, .el-dialog__title, .ant-modal-title, h2, h3').first().textContent({ timeout: 1000 }).catch(() => '');
|
|
152
|
+
const retryBody = await topDialog.locator('.ks-dialog__body, .el-dialog__body, .ant-modal-body, p').first().textContent({ timeout: 1000 }).catch(() => '');
|
|
153
|
+
const retryInfo = (retryTitle || retryBody || '').trim().slice(0, 80);
|
|
154
|
+
if (retryInfo)
|
|
155
|
+
retryPopupInfo = ` — new popup: "${retryInfo}"`;
|
|
156
|
+
}
|
|
157
|
+
catch { /* no info */ }
|
|
158
|
+
return `⚠️ Clicked [ref=${input.ref}] ${elementDesc} — sub-dialog "${popupText}" dismissed via Escape, but retried click still blocked${retryPopupInfo}. Read the accessibility tree to handle the issue.`;
|
|
159
|
+
}
|
|
160
|
+
catch { /* auto-dismiss failed, fall through */ }
|
|
161
|
+
}
|
|
162
|
+
return `⚠️ Clicked [ref=${input.ref}] ${elementDesc} (tried both pointer + JS click) — dialog still open${popupInfo}. Read the accessibility tree to handle the sub-dialog/popup before retrying.`;
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
// JS evaluate failed (element detached or not evaluable) — fall through to original ⚠️
|
|
166
|
+
}
|
|
167
|
+
return `⚠️ Clicked [ref=${input.ref}] ${elementDesc} — dialog still open${newDialogInfo} (action blocked by validation or warning). Read the accessibility tree to handle the sub-dialog/popup before retrying.`;
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
return `✅ Clicked [ref=${input.ref}] ${elementDesc} — dialog closed`;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
catch { /* post-check failed, fall through */ }
|
|
174
|
+
}
|
|
175
|
+
return `✅ Clicked [ref=${input.ref}] ${elementDesc}`;
|
|
60
176
|
},
|
|
61
177
|
}));
|
|
178
|
+
// ── Shared combobox helpers ────────────────────────────────────────────
|
|
179
|
+
/** Detect if an element (by ref) is actually a combobox/dropdown. */
|
|
180
|
+
async function detectCombobox(page, ref) {
|
|
181
|
+
const locator = page.locator(`aria-ref=${ref}`);
|
|
182
|
+
try {
|
|
183
|
+
const attrs = await locator.evaluate((el) => {
|
|
184
|
+
const h = el;
|
|
185
|
+
const inputEl = h;
|
|
186
|
+
return {
|
|
187
|
+
role: h.getAttribute('role') || '',
|
|
188
|
+
ariaExpanded: h.getAttribute('aria-expanded'),
|
|
189
|
+
tag: h.tagName.toLowerCase(),
|
|
190
|
+
className: h.className || '',
|
|
191
|
+
readonly: inputEl.readOnly,
|
|
192
|
+
// 检查自身或祖先是否有下拉框组件标志
|
|
193
|
+
hasDropdownParent: !!h.closest('.ks-select, .el-select, .ant-select, [class*="select"], [class*="dropdown"], [class*="picker"]'),
|
|
194
|
+
};
|
|
195
|
+
});
|
|
196
|
+
if (attrs.tag === 'select')
|
|
197
|
+
return true;
|
|
198
|
+
if (attrs.role === 'combobox' || attrs.role === 'listbox')
|
|
199
|
+
return true;
|
|
200
|
+
if (attrs.ariaExpanded !== null)
|
|
201
|
+
return true;
|
|
202
|
+
if (/select|dropdown|picker/i.test(attrs.className))
|
|
203
|
+
return true;
|
|
204
|
+
if (attrs.hasDropdownParent)
|
|
205
|
+
return true;
|
|
206
|
+
if (attrs.tag === 'input' && attrs.readonly)
|
|
207
|
+
return true;
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/** Select an option from a combobox/dropdown (native or custom). */
|
|
215
|
+
async function selectComboboxOption(page, ref, label) {
|
|
216
|
+
const locator = page.locator(`aria-ref=${ref}`);
|
|
217
|
+
try {
|
|
218
|
+
await locator.selectOption({ label }, { timeout: 2000 });
|
|
219
|
+
return `(native)`;
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
await locator.click({ timeout: 5000, force: true });
|
|
223
|
+
await page.waitForTimeout(600);
|
|
224
|
+
// Strategy 1: Playwright semantic locators (exact then fuzzy)
|
|
225
|
+
let clicked = false;
|
|
226
|
+
try {
|
|
227
|
+
await page.getByRole('option', { name: label, exact: true }).first().click({ timeout: 2000 });
|
|
228
|
+
clicked = true;
|
|
229
|
+
}
|
|
230
|
+
catch { }
|
|
231
|
+
if (!clicked)
|
|
232
|
+
try {
|
|
233
|
+
await page.getByRole('option', { name: label, exact: false }).first().click({ timeout: 2000 });
|
|
234
|
+
clicked = true;
|
|
235
|
+
}
|
|
236
|
+
catch { }
|
|
237
|
+
if (!clicked)
|
|
238
|
+
try {
|
|
239
|
+
await page.getByText(label, { exact: true }).last().click({ timeout: 2000 });
|
|
240
|
+
clicked = true;
|
|
241
|
+
}
|
|
242
|
+
catch { }
|
|
243
|
+
if (!clicked)
|
|
244
|
+
try {
|
|
245
|
+
await page.getByText(label, { exact: false }).first().click({ timeout: 2000 });
|
|
246
|
+
clicked = true;
|
|
247
|
+
}
|
|
248
|
+
catch { }
|
|
249
|
+
// Strategy 2: Use a11y snapshot to find option by ref (same mechanism Agent uses manually)
|
|
250
|
+
if (!clicked)
|
|
251
|
+
try {
|
|
252
|
+
const snapshot = await page.ariaSnapshot({ mode: 'ai' });
|
|
253
|
+
// Search a11y tree for option/listitem containing the label text
|
|
254
|
+
const refMatch = findOptionRefInSnapshot(snapshot, label);
|
|
255
|
+
if (refMatch) {
|
|
256
|
+
await page.locator(`aria-ref=${refMatch}`).click({ timeout: 3000, force: true });
|
|
257
|
+
clicked = true;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
catch { }
|
|
261
|
+
if (!clicked)
|
|
262
|
+
return `opened dropdown but could not find option "${label}"`;
|
|
263
|
+
// ── Verify the selection actually took effect ──
|
|
264
|
+
// Custom components (ks-select/kformily) may not expose value via standard DOM attributes.
|
|
265
|
+
// Verification is lenient: if click succeeded, assume the component handled it internally.
|
|
266
|
+
// Only report failure when we can positively confirm the value is wrong (not just missing).
|
|
267
|
+
try {
|
|
268
|
+
await page.waitForTimeout(400);
|
|
269
|
+
const actuallySelected = await verifyComboboxValue(page, ref, label);
|
|
270
|
+
if (!actuallySelected) {
|
|
271
|
+
// ── Fallback: type label + Enter to trigger component's search+select ──
|
|
272
|
+
// Some custom components (ks-select) need keyboard interaction to commit selection.
|
|
273
|
+
// Clear the field, type the label, then press Enter to confirm.
|
|
274
|
+
try {
|
|
275
|
+
await locator.click({ timeout: 2000, force: true });
|
|
276
|
+
await page.waitForTimeout(200);
|
|
277
|
+
// Select all + delete to clear any partial text
|
|
278
|
+
await page.keyboard.press('Control+a');
|
|
279
|
+
await page.keyboard.press('Backspace');
|
|
280
|
+
await page.waitForTimeout(200);
|
|
281
|
+
await locator.fill(label, { timeout: 2000, force: true });
|
|
282
|
+
await page.waitForTimeout(300);
|
|
283
|
+
await page.keyboard.press('Enter');
|
|
284
|
+
await page.waitForTimeout(600);
|
|
285
|
+
// Re-verify after type+Enter
|
|
286
|
+
const recheck = await verifyComboboxValue(page, ref, label);
|
|
287
|
+
if (recheck)
|
|
288
|
+
return `(custom, via type+Enter)`;
|
|
289
|
+
// Even if verify fails, the component may have updated internally —
|
|
290
|
+
// return success with note to avoid Agent retry loops
|
|
291
|
+
return `(custom, type+Enter applied)`;
|
|
292
|
+
}
|
|
293
|
+
catch {
|
|
294
|
+
// type+Enter fallback failed — still return success to break retry loops
|
|
295
|
+
return `(custom, selection applied — verify skipped)`;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
catch { /* verification failed, assume success */ }
|
|
300
|
+
return `(custom)`;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
/** Verify a combobox actually contains the expected value after selection.
|
|
304
|
+
* Custom components (ks-select, kformily) may show the option as clicked but not update internal form state.
|
|
305
|
+
*/
|
|
306
|
+
async function verifyComboboxValue(page, ref, label) {
|
|
307
|
+
try {
|
|
308
|
+
const actual = await page.locator(`aria-ref=${ref}`).evaluate((el, expected) => {
|
|
309
|
+
// Strategy 1: input value
|
|
310
|
+
const inputVal = el.value;
|
|
311
|
+
if (inputVal && inputVal.includes(expected))
|
|
312
|
+
return inputVal;
|
|
313
|
+
// Strategy 2: element textContent
|
|
314
|
+
const text = el.textContent?.trim();
|
|
315
|
+
if (text && text.includes(expected))
|
|
316
|
+
return text;
|
|
317
|
+
// Strategy 3: aria-label
|
|
318
|
+
const aria = el.getAttribute('aria-label');
|
|
319
|
+
if (aria && aria.includes(expected))
|
|
320
|
+
return aria;
|
|
321
|
+
// Strategy 4: parent select component's selected display
|
|
322
|
+
const parent = el.closest('.ks-select, .el-select, .ant-select, [class*="select"]');
|
|
323
|
+
if (parent) {
|
|
324
|
+
const selected = parent.querySelector('.ks-select__selected, .el-select__selected, .ant-select-selection-item, .selected-value');
|
|
325
|
+
if (selected)
|
|
326
|
+
return selected.textContent?.trim() || '';
|
|
327
|
+
}
|
|
328
|
+
return '';
|
|
329
|
+
}, label);
|
|
330
|
+
return typeof actual === 'string' && actual.length > 0;
|
|
331
|
+
}
|
|
332
|
+
catch {
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Parse a11y snapshot text to find a listitem/option/treeitem ref whose name contains the label.
|
|
338
|
+
* Snapshot lines look like: ` - listitem "KwaiBI官方数据集DEMO" [ref=e2700]`
|
|
339
|
+
*/
|
|
340
|
+
function findOptionRefInSnapshot(snapshot, label) {
|
|
341
|
+
const labelLower = label.toLowerCase();
|
|
342
|
+
const lines = snapshot.split('\n');
|
|
343
|
+
// Match lines with option-like roles + a ref
|
|
344
|
+
const optionRoles = /^(?:\s*-\s+)(?:listitem|option|treeitem|menuitem|cell)\s/;
|
|
345
|
+
for (const line of lines) {
|
|
346
|
+
if (!optionRoles.test(line))
|
|
347
|
+
continue;
|
|
348
|
+
if (!line.includes('[ref='))
|
|
349
|
+
continue;
|
|
350
|
+
const textMatch = line.match(/"(.*?)"/);
|
|
351
|
+
if (!textMatch)
|
|
352
|
+
continue;
|
|
353
|
+
if (textMatch[1].toLowerCase().includes(labelLower)) {
|
|
354
|
+
const refMatch = line.match(/\[ref=([a-f0-9]+)\]/);
|
|
355
|
+
if (refMatch)
|
|
356
|
+
return refMatch[1];
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// Fallback: any line with [ref=] whose quoted text contains the label
|
|
360
|
+
for (const line of lines) {
|
|
361
|
+
if (!line.includes('[ref='))
|
|
362
|
+
continue;
|
|
363
|
+
const textMatch = line.match(/"(.*?)"/);
|
|
364
|
+
if (!textMatch)
|
|
365
|
+
continue;
|
|
366
|
+
if (textMatch[1].toLowerCase().includes(labelLower)) {
|
|
367
|
+
const refMatch = line.match(/\[ref=([a-f0-9]+)\]/);
|
|
368
|
+
if (refMatch)
|
|
369
|
+
return refMatch[1];
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
return null;
|
|
373
|
+
}
|
|
62
374
|
// --- type ---
|
|
63
375
|
exports.tools.set('type', tool({
|
|
64
|
-
description: 'Type text into an input element by its ref identifier.
|
|
376
|
+
description: 'Type text into an input element by its ref identifier. The aria-ref selector automatically penetrates iframe boundaries. For combobox/dropdown fields, this tool automatically handles option selection instead of plain text input.',
|
|
65
377
|
inputSchema: zod_1.z.object({
|
|
66
378
|
ref: zod_1.z.string().describe('Element ref from the accessibility tree, e.g. "f5e91"'),
|
|
67
|
-
text: zod_1.z.string().describe('Text to type
|
|
379
|
+
text: zod_1.z.string().describe('Text to type or option label to select (for combobox fields)'),
|
|
380
|
+
mode: zod_1.z.enum(['replace', 'append']).default('replace')
|
|
381
|
+
.describe('replace: clear existing content + fill (default). append: move cursor to end + pressSequentially (for adding to existing content without clearing)'),
|
|
68
382
|
submit: zod_1.z.boolean().optional().describe('Whether to press Enter after typing'),
|
|
69
383
|
}),
|
|
70
384
|
execute: async function (input) {
|
|
71
385
|
const locator = this.page.locator(`aria-ref=${input.ref}`);
|
|
72
|
-
|
|
386
|
+
// Auto-detect combobox — if so, try select option first; fallback to direct type if option not found
|
|
387
|
+
const isCb = await detectCombobox(this.page, input.ref);
|
|
388
|
+
if (isCb) {
|
|
389
|
+
const result = await selectComboboxOption(this.page, input.ref, input.text);
|
|
390
|
+
if (result.startsWith('opened')) {
|
|
391
|
+
// ── Fallback: option not in dropdown — try direct type+Enter ──
|
|
392
|
+
// Some comboboxes support free-text input or search-as-you-type.
|
|
393
|
+
// The dropdown may not contain the exact label, but typing + Enter still works.
|
|
394
|
+
try {
|
|
395
|
+
await locator.click({ timeout: 2000, force: true });
|
|
396
|
+
await this.page.waitForTimeout(200);
|
|
397
|
+
await this.page.keyboard.press('Control+a');
|
|
398
|
+
await this.page.keyboard.press('Backspace');
|
|
399
|
+
await this.page.waitForTimeout(200);
|
|
400
|
+
if (input.mode === 'append') {
|
|
401
|
+
await locator.pressSequentially(input.text, { timeout: 5000 });
|
|
402
|
+
}
|
|
403
|
+
else {
|
|
404
|
+
await locator.fill(input.text, { timeout: 5000, force: true });
|
|
405
|
+
}
|
|
406
|
+
await this.page.waitForTimeout(300);
|
|
407
|
+
await this.page.keyboard.press('Enter');
|
|
408
|
+
await this.page.waitForTimeout(400);
|
|
409
|
+
return `✅ ${input.mode} "${input.text.slice(0, 50)}" into combobox [ref=${input.ref}] (option not found, used type+Enter fallback)`;
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
return `❌ [ref=${input.ref}] combobox: ${result}`;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
return `✅ Selected "${input.text.slice(0, 50)}" in combobox [ref=${input.ref}] ${result}`;
|
|
416
|
+
}
|
|
417
|
+
if (input.mode === 'append') {
|
|
418
|
+
await locator.click({ timeout: 5000, force: true });
|
|
419
|
+
await this.page.keyboard.press('End');
|
|
420
|
+
await locator.pressSequentially(input.text, { timeout: 10000 });
|
|
421
|
+
}
|
|
422
|
+
else {
|
|
423
|
+
await locator.fill(input.text, { timeout: 5000, force: true });
|
|
424
|
+
}
|
|
73
425
|
if (input.submit)
|
|
74
426
|
await this.page.keyboard.press('Enter');
|
|
75
|
-
return `✅
|
|
427
|
+
return `✅ ${input.mode} "${input.text.slice(0, 50)}" into element [ref=${input.ref}]${input.submit ? ' + Enter' : ''}`;
|
|
76
428
|
},
|
|
77
429
|
}));
|
|
78
430
|
// --- hover ---
|
|
@@ -147,15 +499,15 @@ exports.tools.set('navigate', tool({
|
|
|
147
499
|
}
|
|
148
500
|
url = new URL(url, base).href;
|
|
149
501
|
}
|
|
150
|
-
await this.page.goto(url, { waitUntil: '
|
|
502
|
+
await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
151
503
|
return `✅ Navigated to ${url}`;
|
|
152
504
|
},
|
|
153
505
|
}));
|
|
154
506
|
// --- execute_javascript ---
|
|
155
507
|
exports.tools.set('execute_javascript', tool({
|
|
156
|
-
description: '
|
|
508
|
+
description: 'Execute JavaScript in the BROWSER DOM context to retrieve precise values that screenshots and accessibility trees CANNOT provide — such as computed CSS styles (getComputedStyle), DOM measurements (offsetWidth, getBoundingClientRect), or element properties (checked, disabled, value). Also supports form interaction: setting input/textarea/select values and dispatching events.\n\nYou are a TESTER — you OBSERVE and REPORT the actual page state. If the actual value differs from expected, report it as fail; do NOT override styles/DOM to make it match.\n\n<rule>\n1. Script should include `return` for multi-statement code. Single expressions are auto-wrapped with return. const/let are auto-converted to var.\n2. NO top-level `await` — the script runs in a synchronous IIFE inside page.evaluate. There is no async context.\n3. NO `async () => {}` wrappers — they return undefined. Write flat statements ending with `return`.\n4. NO Playwright APIs (`page`, `locator`, `expect`) — only browser DOM APIs exist inside the page.\n5. FORM INTERACTION allowed: setting input.value, textarea.value, select.value, and dispatchEvent() are permitted.\n6. FORBIDDEN — do NOT modify visual appearance, CSS classes, or page structure to fake test results:\n - No style.setProperty(), style.xxx =, style.removeProperty()\n - No className =, classList.add/remove/toggle()\n - No innerHTML/outerHTML = (page structure changes)\n - No setAttribute for style/class-related attributes\n</rule>\n\nDo NOT use for: clicking, typing, navigating — use click/type/navigate tools instead. Do NOT use for: assertions or logical checks — just return the raw value, you judge the result yourself.',
|
|
157
509
|
inputSchema: zod_1.z.object({
|
|
158
|
-
script: zod_1.z.string().describe("JavaScript to execute in browser DOM. <rule>
|
|
510
|
+
script: zod_1.z.string().describe("JavaScript to execute in browser DOM. <rule> Include `return` for multi-statement code (single expressions auto-wrapped). const/let auto-converted to var. FORM INTERACTION allowed (input.value=, dispatchEvent). FORBIDDEN: style/className/classList/innerHTML mutation. </rule>\n\n✅ 'return document.querySelector(\".x\").innerText'\n✅ 'var el = document.querySelector(\".x\"); return window.getComputedStyle(el).backgroundColor'\n✅ 'return document.querySelectorAll(\".field-error\").length'\n✅ 'var input = document.querySelector(\"input\"); input.value = \"test\"; input.dispatchEvent(new Event(\"input\")); return input.value'\n❌ 'page.locator(\".x\")' — no page object\n❌ 'await page.locator(...)' — no await\n❌ 'document.querySelector(\".x\").style.borderColor = \"red\"' — FORBIDDEN: style mutation\n❌ 'document.querySelector(\".x\").classList.add(\"field-error\")' — FORBIDDEN: classList mutation"),
|
|
159
511
|
}),
|
|
160
512
|
execute: async function (input) {
|
|
161
513
|
try {
|
|
@@ -175,16 +527,20 @@ exports.tools.set('execute_javascript', tool({
|
|
|
175
527
|
if (iifeMatch) {
|
|
176
528
|
script = iifeMatch[1].trim();
|
|
177
529
|
}
|
|
178
|
-
//
|
|
530
|
+
// Auto-convert const/let → var for page.evaluate compatibility
|
|
531
|
+
script = script.replace(/\bconst\b/g, 'var').replace(/\blet\b/g, 'var');
|
|
532
|
+
// Write guard: block page-mutation APIs that fake test results.
|
|
179
533
|
// CRITICAL: E2E testing must OBSERVE and REPORT the actual page state.
|
|
180
|
-
//
|
|
534
|
+
// Form interaction (input.value, dispatchEvent) is ALLOWED.
|
|
535
|
+
// Visual/page-structure mutation (style, className, classList, innerHTML) is BLOCKED.
|
|
181
536
|
const writePatterns = [
|
|
182
|
-
/\b(?:document|element|el|node)\.\s*(?:remove|appendChild|insertBefore|replaceChild|removeChild|appendChild)\s*\(/,
|
|
183
|
-
/\b(?:document|element|el|node)\.\s*(?:innerHTML|outerHTML|textContent|innerText)\s*=/,
|
|
184
|
-
/\b(?:document|element|el|node)\.\s*(?:setAttribute|removeAttribute|classList)\s*\./,
|
|
185
537
|
/\b\.style\s*\.\s*\w+\s*=/,
|
|
186
538
|
/\b\.style\s*\.\s*(?:setProperty|removeProperty)\s*\(/,
|
|
539
|
+
/\bclassList\s*\.\s*(?:add|remove|toggle|replace)\s*\(/,
|
|
187
540
|
/\b(?:document|element|el|node)\.\s*className\s*=/,
|
|
541
|
+
/\b(?:document|element|el|node)\.\s*(?:innerHTML|outerHTML)\s*=/,
|
|
542
|
+
/\b(?:document|element|el|node)\.\s*(?:setAttribute|removeAttribute)\s*\([^,]*['\"](?:class|style)['\"']/i,
|
|
543
|
+
/\b(?:document|element|el|node)\.\s*(?:remove|appendChild|insertBefore|replaceChild|removeChild|appendChild)\s*\(/,
|
|
188
544
|
/\blocation\s*(?:\.href\s*=|\.assign\s*\(|\.replace\s*\()/,
|
|
189
545
|
/\bwindow\s*\.\s*(?:close|stop|open)\s*\(/,
|
|
190
546
|
/\bdocument\.\s*(?:write|writeln|execCommand)\s*\(/,
|
|
@@ -195,13 +551,21 @@ exports.tools.set('execute_javascript', tool({
|
|
|
195
551
|
];
|
|
196
552
|
for (const pattern of writePatterns) {
|
|
197
553
|
if (pattern.test(script)) {
|
|
198
|
-
return `❌ JS execution blocked:
|
|
554
|
+
return `❌ JS execution blocked: page mutation API detected. You MUST NOT modify styles, CSS classes, or page structure to fake test results. Use getComputedStyle for reading styles, and report mismatches honestly.`;
|
|
199
555
|
}
|
|
200
556
|
}
|
|
201
|
-
//
|
|
202
|
-
//
|
|
557
|
+
// Auto-wrap single expressions with return.
|
|
558
|
+
// Multi-statement code (containing var/return/semicolons) must provide its own return.
|
|
203
559
|
if (!/\breturn\b/.test(script)) {
|
|
204
|
-
|
|
560
|
+
// Heuristic: if the script looks like a single expression (no var, no block, no semicolons),
|
|
561
|
+
// auto-prepend return. Otherwise require explicit return.
|
|
562
|
+
const looksLikeExpression = !/^\s*(var|function|\{)/.test(script) && !/;\s*$/.test(script) && !/\n/.test(script);
|
|
563
|
+
if (looksLikeExpression) {
|
|
564
|
+
script = 'return ' + script;
|
|
565
|
+
}
|
|
566
|
+
else {
|
|
567
|
+
return `❌ Script must include 'return' to capture the result. Example: "var el = document.querySelector('.x'); return window.getComputedStyle(el).backgroundColor"`;
|
|
568
|
+
}
|
|
205
569
|
}
|
|
206
570
|
const wrapped = `(function() { ${script} })()`;
|
|
207
571
|
const result = await this.page.evaluate(wrapped);
|
|
@@ -215,6 +579,30 @@ exports.tools.set('execute_javascript', tool({
|
|
|
215
579
|
}
|
|
216
580
|
},
|
|
217
581
|
}));
|
|
582
|
+
// --- visual_assert ---
|
|
583
|
+
exports.tools.set('visual_assert', tool({
|
|
584
|
+
description: 'Assert a visual condition by analyzing a screenshot — READ-ONLY, does NOT interact with the page. Use this to verify visual states that the a11y tree cannot capture: red borders, error text colors, highlight states, icon visibility, layout correctness. Example: "the 指标标准名称 field shows a red border error state" or "a toast message with text 删除成功 is visible". Do NOT use this to find elements to click — use visual_locate for that.',
|
|
585
|
+
inputSchema: zod_1.z.object({
|
|
586
|
+
assertion: zod_1.z.string().describe('The visual condition to verify, e.g. "the 姓名 field has a red border" or "an error tooltip saying 必填项 is visible near the top of the form"'),
|
|
587
|
+
}),
|
|
588
|
+
execute: async function (input) {
|
|
589
|
+
// 1. Capture screenshot
|
|
590
|
+
let screenshotBase64;
|
|
591
|
+
try {
|
|
592
|
+
const buf = await this.page.screenshot({ type: 'png', timeout: 10000 });
|
|
593
|
+
screenshotBase64 = buf.toString('base64');
|
|
594
|
+
}
|
|
595
|
+
catch (e) {
|
|
596
|
+
throw new Error(`visual_assert screenshot failed: ${e.message}`);
|
|
597
|
+
}
|
|
598
|
+
// 2. Call LLM for visual assertion
|
|
599
|
+
const assertResult = await this.visualAssertElement(screenshotBase64, input.assertion);
|
|
600
|
+
if (assertResult.passed) {
|
|
601
|
+
return `✅ Visual assert PASSED: "${input.assertion}" — ${assertResult.reasoning}`;
|
|
602
|
+
}
|
|
603
|
+
return `❌ Visual assert FAILED: "${input.assertion}" — ${assertResult.reasoning}`;
|
|
604
|
+
},
|
|
605
|
+
}));
|
|
218
606
|
// --- visual_locate ---
|
|
219
607
|
exports.tools.set('visual_locate', tool({
|
|
220
608
|
description: 'Locate and click an element by visual analysis of a screenshot. FALLBACK: only use after `click` fails 2+ times on the same element (non-standard ARIA role, invisible in accessibility tree, or element visible in screenshot but missing from accessibility tree). Provide a concise visual description of the element. It will screenshot the page, send it to a multimodal LLM to locate the element, then click at the predicted coordinates. After clicking, verify the effect on the next screenshot.',
|
|
@@ -246,6 +634,415 @@ exports.tools.set('visual_locate', tool({
|
|
|
246
634
|
return `✅ Visual locate & click: "${input.description}" at (${centerX}, ${centerY})`;
|
|
247
635
|
},
|
|
248
636
|
}));
|
|
637
|
+
// --- select_option ---
|
|
638
|
+
exports.tools.set('select_option', tool({
|
|
639
|
+
description: 'Select an option from a combobox/dropdown field. Handles both native <select> elements and custom dropdown components (ks-select, ant-select, el-select, etc.) in a single step. Use this when you need to select a dropdown option — either standalone or after fill_form for textbox fields.',
|
|
640
|
+
inputSchema: zod_1.z.object({
|
|
641
|
+
ref: zod_1.z.string().describe('Element ref of the combobox/dropdown from the accessibility tree'),
|
|
642
|
+
label: zod_1.z.string().describe('Visible text of the option to select (exact match)'),
|
|
643
|
+
}),
|
|
644
|
+
execute: async function (input) {
|
|
645
|
+
const result = await selectComboboxOption(this.page, input.ref, input.label);
|
|
646
|
+
if (result.startsWith('opened')) {
|
|
647
|
+
return `❌ [ref=${input.ref}] combobox: ${result}`;
|
|
648
|
+
}
|
|
649
|
+
return `✅ Selected "${input.label}" [ref=${input.ref}] ${result}`;
|
|
650
|
+
},
|
|
651
|
+
}));
|
|
652
|
+
// --- fill_form ---
|
|
653
|
+
exports.tools.set('fill_form', tool({
|
|
654
|
+
description: 'Batch fill multiple form fields in one action. Handles textbox (fill), combobox/dropdown (select_option with custom component fallback), checkbox/radio (setChecked), and slider (fill). Use this instead of repeated type+click for multi-field forms — reduces step count and avoids missing required fields.',
|
|
655
|
+
inputSchema: zod_1.z.object({
|
|
656
|
+
fields: zod_1.z.array(zod_1.z.object({
|
|
657
|
+
ref: zod_1.z.string().describe('Element ref from the accessibility tree'),
|
|
658
|
+
type: zod_1.z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider'])
|
|
659
|
+
.describe('Field type: textbox→fill, combobox→select option (native or custom dropdown), checkbox/radio→setChecked, slider→fill'),
|
|
660
|
+
value: zod_1.z.string()
|
|
661
|
+
.describe('Value to fill. Textbox/slider: text content. Checkbox/radio: "true" or "false". Combobox: visible option label text.'),
|
|
662
|
+
})).describe('Array of form fields to fill in'),
|
|
663
|
+
}),
|
|
664
|
+
execute: async function (input) {
|
|
665
|
+
const results = [];
|
|
666
|
+
// ── Auto-detect actual element type (custom components like ks-select may appear as textbox in a11y tree) ──
|
|
667
|
+
async function detectActualType(ref) {
|
|
668
|
+
// Delegate combobox detection to shared helper (includes readonly/closest/className checks)
|
|
669
|
+
if (await detectCombobox(this.page, ref))
|
|
670
|
+
return 'combobox';
|
|
671
|
+
const locator = this.page.locator(`aria-ref=${ref}`);
|
|
672
|
+
try {
|
|
673
|
+
const [tag, inputType] = await locator.evaluate(el => {
|
|
674
|
+
const h = el;
|
|
675
|
+
return [h.tagName.toLowerCase(), h.type || ''];
|
|
676
|
+
});
|
|
677
|
+
if (tag === 'input' && inputType === 'checkbox')
|
|
678
|
+
return 'checkbox';
|
|
679
|
+
if (tag === 'input' && inputType === 'radio')
|
|
680
|
+
return 'radio';
|
|
681
|
+
}
|
|
682
|
+
catch { /* element not visible or detached — trust Agent's type */ }
|
|
683
|
+
return '';
|
|
684
|
+
}
|
|
685
|
+
// Categorize fields. Override mis-typed textbox → combobox.
|
|
686
|
+
const detectedTypes = new Map();
|
|
687
|
+
const textboxCandidates = input.fields.filter(f => f.type === 'textbox');
|
|
688
|
+
await Promise.all(textboxCandidates.map(async (f) => {
|
|
689
|
+
const actual = await detectActualType.call(this, f.ref);
|
|
690
|
+
if (actual)
|
|
691
|
+
detectedTypes.set(f.ref, actual);
|
|
692
|
+
}));
|
|
693
|
+
const effectiveFields = input.fields.map(f => {
|
|
694
|
+
const detected = detectedTypes.get(f.ref);
|
|
695
|
+
return detected && detected !== f.type ? { ...f, type: detected } : f;
|
|
696
|
+
});
|
|
697
|
+
// Process textbox/checkbox/radio/slider first, then combobox last
|
|
698
|
+
const simpleFields = effectiveFields.filter(f => f.type !== 'combobox');
|
|
699
|
+
const comboFields = effectiveFields.filter(f => f.type === 'combobox');
|
|
700
|
+
for (const field of simpleFields) {
|
|
701
|
+
const locator = this.page.locator(`aria-ref=${field.ref}`);
|
|
702
|
+
try {
|
|
703
|
+
if (field.type === 'textbox' || field.type === 'slider') {
|
|
704
|
+
await locator.fill(field.value, { timeout: 5000, force: true });
|
|
705
|
+
results.push(`✅ ${field.type} [${field.ref}] = "${field.value.slice(0, 30)}"`);
|
|
706
|
+
}
|
|
707
|
+
else if (field.type === 'checkbox' || field.type === 'radio') {
|
|
708
|
+
await locator.setChecked(field.value === 'true', { timeout: 5000 });
|
|
709
|
+
results.push(`✅ ${field.type} [${field.ref}] = ${field.value}`);
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
catch (err) {
|
|
713
|
+
results.push(`❌ [${field.ref}] ${field.type}: ${err.message}`);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
// Handle combobox fields one by one (each opens/closes dropdown)
|
|
717
|
+
for (const field of comboFields) {
|
|
718
|
+
try {
|
|
719
|
+
const result = await selectComboboxOption(this.page, field.ref, field.value);
|
|
720
|
+
if (result.startsWith('opened')) {
|
|
721
|
+
// ── Fallback: option not in dropdown — try direct type+Enter ──
|
|
722
|
+
const locator = this.page.locator(`aria-ref=${field.ref}`);
|
|
723
|
+
try {
|
|
724
|
+
await locator.click({ timeout: 2000, force: true });
|
|
725
|
+
await this.page.waitForTimeout(200);
|
|
726
|
+
await this.page.keyboard.press('Control+a');
|
|
727
|
+
await this.page.keyboard.press('Backspace');
|
|
728
|
+
await this.page.waitForTimeout(200);
|
|
729
|
+
await locator.fill(field.value, { timeout: 5000, force: true });
|
|
730
|
+
await this.page.waitForTimeout(300);
|
|
731
|
+
await this.page.keyboard.press('Enter');
|
|
732
|
+
await this.page.waitForTimeout(400);
|
|
733
|
+
results.push(`✅ combobox [${field.ref}] = "${field.value}" (option not found, used type+Enter fallback)`);
|
|
734
|
+
}
|
|
735
|
+
catch {
|
|
736
|
+
results.push(`❌ [${field.ref}] combobox: ${result}`);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
else {
|
|
740
|
+
results.push(`✅ combobox [${field.ref}] = "${field.value}" ${result}`);
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
catch (err) {
|
|
744
|
+
results.push(`❌ [${field.ref}] combobox: ${err.message}`);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
return results.join('\n');
|
|
748
|
+
},
|
|
749
|
+
}));
|
|
750
|
+
// --- drag ---
|
|
751
|
+
exports.tools.set('drag', tool({
|
|
752
|
+
description: `Drag from a starting point to an ending point using real mouse events (mousedown → mousemove steps → mouseup). Use for: resizing elements via drag handles, drag-to-reorder lists, any UI requiring hold+move interaction.
|
|
753
|
+
|
|
754
|
+
Two targeting modes:
|
|
755
|
+
1. By ref — drags from the element's center (use for drag-to-reorder)
|
|
756
|
+
2. By coordinates (startX/startY) — precise point (use for resize handles found via visual_locate)
|
|
757
|
+
|
|
758
|
+
Target specification:
|
|
759
|
+
- Relative: use deltaX/deltaY (e.g. deltaX=200 moves 200px right)
|
|
760
|
+
- Absolute: use endX/endY (e.g. endX=800, endY=400)
|
|
761
|
+
|
|
762
|
+
NOT for: scrolling (use scroll), clicking (use click), HTML5 DnD between elements.`,
|
|
763
|
+
inputSchema: zod_1.z.object({
|
|
764
|
+
ref: zod_1.z.string().optional()
|
|
765
|
+
.describe('Element ref to start dragging from. Drags from element center. Use startX/startY for resize handles.'),
|
|
766
|
+
startX: zod_1.z.number().optional()
|
|
767
|
+
.describe('X coordinate to start dragging from (viewport pixels). Use when you have precise coordinates from visual_locate.'),
|
|
768
|
+
startY: zod_1.z.number().optional()
|
|
769
|
+
.describe('Y coordinate to start dragging from (viewport pixels).'),
|
|
770
|
+
deltaX: zod_1.z.number().optional()
|
|
771
|
+
.describe('Horizontal displacement in pixels. Positive=right, negative=left. Use with ref or startX/startY.'),
|
|
772
|
+
deltaY: zod_1.z.number().optional()
|
|
773
|
+
.describe('Vertical displacement in pixels. Positive=down, negative=up.'),
|
|
774
|
+
endX: zod_1.z.number().optional()
|
|
775
|
+
.describe('Absolute X coordinate to drag to (viewport pixels). Use deltaX/deltaY for relative, endX/endY for absolute.'),
|
|
776
|
+
endY: zod_1.z.number().optional()
|
|
777
|
+
.describe('Absolute Y coordinate to drag to (viewport pixels).'),
|
|
778
|
+
steps: zod_1.z.number().min(1).max(100).default(10)
|
|
779
|
+
.describe('Number of intermediate mousemove steps. More=smoother animation. Default 10. Use 20+ for CSS resize.'),
|
|
780
|
+
}),
|
|
781
|
+
execute: async function (input) {
|
|
782
|
+
// 1. Determine start coordinates
|
|
783
|
+
let sx, sy;
|
|
784
|
+
if (input.ref) {
|
|
785
|
+
const locator = this.page.locator(`aria-ref=${input.ref}`);
|
|
786
|
+
try {
|
|
787
|
+
const box = await locator.boundingBox({ timeout: 5000 });
|
|
788
|
+
if (!box)
|
|
789
|
+
return `❌ Cannot drag: element [ref=${input.ref}] not visible or detached`;
|
|
790
|
+
sx = box.x + box.width / 2;
|
|
791
|
+
sy = box.y + box.height / 2;
|
|
792
|
+
}
|
|
793
|
+
catch (err) {
|
|
794
|
+
return `❌ Cannot drag: element [ref=${input.ref}] error: ${err.message}`;
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
else if (input.startX !== undefined && input.startY !== undefined) {
|
|
798
|
+
sx = input.startX;
|
|
799
|
+
sy = input.startY;
|
|
800
|
+
}
|
|
801
|
+
else {
|
|
802
|
+
return `❌ Cannot drag: provide either ref or startX/startY`;
|
|
803
|
+
}
|
|
804
|
+
// 2. Determine end coordinates
|
|
805
|
+
let ex, ey;
|
|
806
|
+
if (input.endX !== undefined && input.endY !== undefined) {
|
|
807
|
+
ex = input.endX;
|
|
808
|
+
ey = input.endY;
|
|
809
|
+
}
|
|
810
|
+
else if (input.deltaX !== undefined || input.deltaY !== undefined) {
|
|
811
|
+
ex = sx + (input.deltaX || 0);
|
|
812
|
+
ey = sy + (input.deltaY || 0);
|
|
813
|
+
}
|
|
814
|
+
else {
|
|
815
|
+
return `❌ Cannot drag: provide either deltaX/deltaY or endX/endY`;
|
|
816
|
+
}
|
|
817
|
+
// 3. Execute real mouse drag with intermediate steps
|
|
818
|
+
const steps = input.steps ?? 10;
|
|
819
|
+
try {
|
|
820
|
+
await this.page.mouse.move(sx, sy);
|
|
821
|
+
await this.page.mouse.down();
|
|
822
|
+
for (let i = 1; i <= steps; i++) {
|
|
823
|
+
const progress = i / steps;
|
|
824
|
+
await this.page.mouse.move(sx + (ex - sx) * progress, sy + (ey - sy) * progress);
|
|
825
|
+
await this.page.waitForTimeout(16); // ~60fps
|
|
826
|
+
}
|
|
827
|
+
await this.page.mouse.up();
|
|
828
|
+
await this.page.waitForTimeout(300); // wait for resize/layout to settle
|
|
829
|
+
return `✅ Dragged from (${Math.round(sx)}, ${Math.round(sy)}) to (${Math.round(ex)}, ${Math.round(ey)}) in ${steps} steps (Δx=${Math.round(ex - sx)}, Δy=${Math.round(ey - sy)})`;
|
|
830
|
+
}
|
|
831
|
+
catch (err) {
|
|
832
|
+
try {
|
|
833
|
+
await this.page.mouse.up();
|
|
834
|
+
}
|
|
835
|
+
catch { /* ensure mouse released */ }
|
|
836
|
+
return `❌ Drag failed: ${err.message}`;
|
|
837
|
+
}
|
|
838
|
+
},
|
|
839
|
+
}));
|
|
840
|
+
// --- wait_for ---
|
|
841
|
+
exports.tools.set('wait_for', tool({
|
|
842
|
+
description: `Wait for text to appear or disappear on the page, or for a specified duration. More efficient than polling with wait(time).
|
|
843
|
+
|
|
844
|
+
- waitForText: wait until specific text becomes visible on the page (e.g. wait for "保存成功" toast)
|
|
845
|
+
- waitForTextGone: wait until specific text disappears (e.g. wait for "加载中..." spinner to vanish)
|
|
846
|
+
- seconds: simple time-based wait (max 30s)
|
|
847
|
+
|
|
848
|
+
At least one parameter is required. You can combine them (e.g. wait for text + timeout).`,
|
|
849
|
+
inputSchema: zod_1.z.object({
|
|
850
|
+
waitForText: zod_1.z.string().optional()
|
|
851
|
+
.describe('Wait for this text to appear on the page (visible)'),
|
|
852
|
+
waitForTextGone: zod_1.z.string().optional()
|
|
853
|
+
.describe('Wait for this text to disappear from the page (hidden)'),
|
|
854
|
+
seconds: zod_1.z.number().min(0.1).max(30).optional()
|
|
855
|
+
.describe('Wait time in seconds (max 30)'),
|
|
856
|
+
}),
|
|
857
|
+
execute: async function (input) {
|
|
858
|
+
if (!input.waitForText && !input.waitForTextGone && !input.seconds) {
|
|
859
|
+
return `❌ wait_for: provide at least one of waitForText, waitForTextGone, or seconds`;
|
|
860
|
+
}
|
|
861
|
+
const results = [];
|
|
862
|
+
if (input.seconds) {
|
|
863
|
+
await this.page.waitForTimeout(input.seconds * 1000);
|
|
864
|
+
results.push(`waited ${input.seconds}s`);
|
|
865
|
+
}
|
|
866
|
+
if (input.waitForTextGone) {
|
|
867
|
+
const locator = this.page.getByText(input.waitForTextGone).first();
|
|
868
|
+
try {
|
|
869
|
+
await locator.waitFor({ state: 'hidden', timeout: 15000 });
|
|
870
|
+
results.push(`"${input.waitForTextGone}" disappeared`);
|
|
871
|
+
}
|
|
872
|
+
catch {
|
|
873
|
+
results.push(`⚠️ "${input.waitForTextGone}" still visible after 15s`);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
if (input.waitForText) {
|
|
877
|
+
const locator = this.page.getByText(input.waitForText).first();
|
|
878
|
+
try {
|
|
879
|
+
await locator.waitFor({ state: 'visible', timeout: 15000 });
|
|
880
|
+
results.push(`"${input.waitForText}" appeared`);
|
|
881
|
+
}
|
|
882
|
+
catch {
|
|
883
|
+
results.push(`⚠️ "${input.waitForText}" not found after 15s`);
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
return `✅ ${results.join(', ')}`;
|
|
887
|
+
},
|
|
888
|
+
}));
|
|
889
|
+
// --- verify_value ---
|
|
890
|
+
exports.tools.set('verify_value', tool({
|
|
891
|
+
description: `Deterministically verify a form element's current value — no LLM needed. Faster and more precise than visual_assert for value checks.
|
|
892
|
+
|
|
893
|
+
- textbox/slider/combobox → checks locator.inputValue()
|
|
894
|
+
- checkbox/radio → checks locator.isChecked()
|
|
895
|
+
|
|
896
|
+
Returns ✅ if value matches, ❌ if mismatch. Use for precise value assertions instead of visual inspection.`,
|
|
897
|
+
inputSchema: zod_1.z.object({
|
|
898
|
+
ref: zod_1.z.string().describe('Element ref from the accessibility tree'),
|
|
899
|
+
type: zod_1.z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider'])
|
|
900
|
+
.describe('Element type: textbox/slider/combobox → check inputValue; checkbox/radio → check isChecked'),
|
|
901
|
+
value: zod_1.z.string().describe('Expected value. For checkbox/radio: "true" or "false". For textbox: expected text. For combobox: expected selected value.'),
|
|
902
|
+
}),
|
|
903
|
+
execute: async function (input) {
|
|
904
|
+
const locator = this.page.locator(`aria-ref=${input.ref}`);
|
|
905
|
+
try {
|
|
906
|
+
if (input.type === 'checkbox' || input.type === 'radio') {
|
|
907
|
+
const checked = await locator.isChecked({ timeout: 5000 });
|
|
908
|
+
const expected = input.value === 'true';
|
|
909
|
+
if (checked === expected) {
|
|
910
|
+
return `✅ Verify [ref=${input.ref}] ${input.type}: checked=${checked} (expected ${input.value})`;
|
|
911
|
+
}
|
|
912
|
+
return `❌ Verify [ref=${input.ref}] ${input.type}: checked=${checked} (expected ${input.value})`;
|
|
913
|
+
}
|
|
914
|
+
// textbox, slider, combobox — check inputValue
|
|
915
|
+
const actualValue = await locator.inputValue({ timeout: 5000 });
|
|
916
|
+
if (actualValue === input.value) {
|
|
917
|
+
return `✅ Verify [ref=${input.ref}] ${input.type}: value="${actualValue}" (expected "${input.value}")`;
|
|
918
|
+
}
|
|
919
|
+
// For combobox, also check textContent (custom components may not expose value via inputValue)
|
|
920
|
+
if (input.type === 'combobox') {
|
|
921
|
+
const textContent = await locator.textContent({ timeout: 2000 }).catch(() => '');
|
|
922
|
+
if (textContent?.includes(input.value)) {
|
|
923
|
+
return `✅ Verify [ref=${input.ref}] combobox: text contains "${input.value}" (inputValue="${actualValue}")`;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
return `❌ Verify [ref=${input.ref}] ${input.type}: value="${actualValue}" (expected "${input.value}")`;
|
|
927
|
+
}
|
|
928
|
+
catch (err) {
|
|
929
|
+
return `❌ Verify [ref=${input.ref}] failed: ${err.message}`;
|
|
930
|
+
}
|
|
931
|
+
},
|
|
932
|
+
}));
|
|
933
|
+
// --- handle_dialog ---
|
|
934
|
+
exports.tools.set('handle_dialog', tool({
|
|
935
|
+
description: `Handle native browser dialogs (alert, confirm, prompt). Native dialogs are auto-accepted by default when they appear.
|
|
936
|
+
|
|
937
|
+
Two usage modes:
|
|
938
|
+
1. BEFORE a dialog-triggering action: call handle_dialog(accept=false) to override the default auto-accept behavior. Then click the button that triggers the dialog.
|
|
939
|
+
2. AFTER a dialog appeared: call handle_dialog to check what dialog was captured and how it was handled.
|
|
940
|
+
|
|
941
|
+
Native dialogs are auto-accepted unless you pre-set a different action. If a click triggers a dialog, it will be handled automatically based on your pre-set preference.`,
|
|
942
|
+
inputSchema: zod_1.z.object({
|
|
943
|
+
accept: zod_1.z.boolean().default(true).describe('Whether to accept (true) or dismiss (false) the next native dialog. Default true.'),
|
|
944
|
+
promptText: zod_1.z.string().optional().describe('Text to enter in prompt dialog (only when accept=true and dialog is a prompt)'),
|
|
945
|
+
}),
|
|
946
|
+
execute: async function (input) {
|
|
947
|
+
// Set handler for the NEXT dialog
|
|
948
|
+
this._dialogHandler = { accept: input.accept, promptText: input.promptText };
|
|
949
|
+
// Also report the last captured dialog (if any)
|
|
950
|
+
const last = this._lastDialogInfo;
|
|
951
|
+
if (last) {
|
|
952
|
+
return `Last dialog: [${last.type}] "${last.message.slice(0, 80)}" — was ${last.accepted ? 'accepted' : 'dismissed'}. Next dialog will be ${input.accept ? 'accepted' : 'dismissed'}.`;
|
|
953
|
+
}
|
|
954
|
+
return `Next native dialog will be ${input.accept ? 'accepted' : 'dismissed'}${input.promptText ? ` with text "${input.promptText}"` : ''}.`;
|
|
955
|
+
},
|
|
956
|
+
}));
|
|
957
|
+
// --- console_messages ---
|
|
958
|
+
exports.tools.set('console_messages', tool({
|
|
959
|
+
description: `Get browser console log messages captured during page execution. Use to check for JavaScript errors, verify log output, or debug page issues. More reliable than visual checking for error toasts or log messages.`,
|
|
960
|
+
inputSchema: zod_1.z.object({
|
|
961
|
+
level: zod_1.z.enum(['error', 'warning', 'info', 'debug']).default('info')
|
|
962
|
+
.describe('Minimum log level. "error"=only errors, "warning"=errors+warnings, "info"=add info logs, "debug"=all. Default "info".'),
|
|
963
|
+
}),
|
|
964
|
+
execute: async function (input) {
|
|
965
|
+
const messages = this._consoleMessages || [];
|
|
966
|
+
const levelPriority = { debug: 0, info: 1, warning: 2, error: 3 };
|
|
967
|
+
const minPriority = levelPriority[input.level] ?? 1;
|
|
968
|
+
const filtered = messages.filter((m) => (levelPriority[m.level] ?? 0) >= minPriority);
|
|
969
|
+
if (filtered.length === 0) {
|
|
970
|
+
return `No console messages at level "${input.level}" or above. Total captured: ${messages.length}.`;
|
|
971
|
+
}
|
|
972
|
+
const formatted = filtered.map((m, i) => `[${i + 1}] ${m.level.toUpperCase()}: ${m.text.slice(0, 200)}`).join('\n');
|
|
973
|
+
return `Console messages (${filtered.length}/${messages.length} at level ≥ "${input.level}"):\n${formatted}`;
|
|
974
|
+
},
|
|
975
|
+
}));
|
|
976
|
+
// --- file_upload ---
|
|
977
|
+
exports.tools.set('file_upload', tool({
|
|
978
|
+
description: `Upload one or more files via a file input element. When you click a file input button, a file chooser dialog appears. This tool uploads files to the pending file chooser.
|
|
979
|
+
|
|
980
|
+
Flow: 1) Click the file input button (e.g. "上传文件", "选择文件"), 2) Call file_upload with the file paths.
|
|
981
|
+
|
|
982
|
+
Paths must be absolute. The file chooser is single-use — after uploading, a new click is needed for another upload.`,
|
|
983
|
+
inputSchema: zod_1.z.object({
|
|
984
|
+
paths: zod_1.z.array(zod_1.z.string()).describe('Absolute file paths to upload'),
|
|
985
|
+
}),
|
|
986
|
+
execute: async function (input) {
|
|
987
|
+
const fileChooser = this._pendingFileChooser;
|
|
988
|
+
if (!fileChooser) {
|
|
989
|
+
return `❌ No pending file chooser. Click a file input element first to trigger a file chooser dialog.`;
|
|
990
|
+
}
|
|
991
|
+
try {
|
|
992
|
+
await fileChooser.setFiles(input.paths);
|
|
993
|
+
this._pendingFileChooser = null;
|
|
994
|
+
return `✅ Uploaded ${input.paths.length} file(s): ${input.paths.map(p => p.split('/').pop()).join(', ')}`;
|
|
995
|
+
}
|
|
996
|
+
catch (err) {
|
|
997
|
+
this._pendingFileChooser = null;
|
|
998
|
+
return `❌ File upload failed: ${err.message}`;
|
|
999
|
+
}
|
|
1000
|
+
},
|
|
1001
|
+
}));
|
|
1002
|
+
// --- network_requests ---
|
|
1003
|
+
exports.tools.set('network_requests', tool({
|
|
1004
|
+
description: `List network requests made by the page. Use to verify API calls were made, check response status codes, or debug network issues. Returns a numbered list with method, URL, and status code.
|
|
1005
|
+
|
|
1006
|
+
Filter by URL pattern (regex) and optionally include static resources (images, fonts, scripts). By default, only API/XHR requests are shown.`,
|
|
1007
|
+
inputSchema: zod_1.z.object({
|
|
1008
|
+
filter: zod_1.z.string().optional()
|
|
1009
|
+
.describe('Regex pattern to filter URLs (e.g. "/api/.*user")'),
|
|
1010
|
+
includeStatic: zod_1.z.boolean().default(false)
|
|
1011
|
+
.describe('Include static resources (images, fonts, scripts). Default false.'),
|
|
1012
|
+
}),
|
|
1013
|
+
execute: async function (input) {
|
|
1014
|
+
const requests = this._networkRequests || [];
|
|
1015
|
+
const filter = input.filter ? new RegExp(input.filter) : undefined;
|
|
1016
|
+
const lines = [];
|
|
1017
|
+
for (let i = 0; i < requests.length; i++) {
|
|
1018
|
+
const req = requests[i];
|
|
1019
|
+
if (!input.includeStatic && req.isStatic)
|
|
1020
|
+
continue;
|
|
1021
|
+
if (filter && !filter.test(req.url))
|
|
1022
|
+
continue;
|
|
1023
|
+
const status = req.status ? ` [${req.status}]` : '';
|
|
1024
|
+
const method = req.method || '?';
|
|
1025
|
+
lines.push(`[${i + 1}] ${method} ${req.url.slice(0, 120)}${status}`);
|
|
1026
|
+
}
|
|
1027
|
+
if (lines.length === 0) {
|
|
1028
|
+
return `No matching network requests. Total captured: ${requests.length}.`;
|
|
1029
|
+
}
|
|
1030
|
+
return `Network requests (${lines.length} shown, ${requests.length} total):\n${lines.join('\n')}`;
|
|
1031
|
+
},
|
|
1032
|
+
}));
|
|
1033
|
+
// --- resize_viewport ---
|
|
1034
|
+
exports.tools.set('resize_viewport', tool({
|
|
1035
|
+
description: `Resize the browser viewport (window size). Use for responsive design testing or to trigger layout changes at specific viewport sizes. Common breakpoints: 375×667 (mobile), 768×1024 (tablet), 1280×720 (desktop), 1920×1080 (full HD).`,
|
|
1036
|
+
inputSchema: zod_1.z.object({
|
|
1037
|
+
width: zod_1.z.number().min(320).max(3840).describe('Viewport width in pixels'),
|
|
1038
|
+
height: zod_1.z.number().min(240).max(2160).describe('Viewport height in pixels'),
|
|
1039
|
+
}),
|
|
1040
|
+
execute: async function (input) {
|
|
1041
|
+
await this.page.setViewportSize({ width: input.width, height: input.height });
|
|
1042
|
+
await this.page.waitForTimeout(300);
|
|
1043
|
+
return `✅ Viewport resized to ${input.width}×${input.height}`;
|
|
1044
|
+
},
|
|
1045
|
+
}));
|
|
249
1046
|
// ── verify_expected fully removed (Phase 6) ──
|
|
250
1047
|
// LLM now sees screenshots via multimodal prompt and self-evaluates.
|
|
251
1048
|
// No separate verification tool or function needed — each step's screenshot
|