chrometools-mcp 3.2.10 → 3.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +169 -0
- package/README.md +14 -5
- package/angular-tools.js +9 -3
- package/bridge/bridge-client.js +62 -7
- package/bridge/bridge-service.js +80 -2
- package/browser/page-manager.js +83 -0
- package/extension/background.js +117 -0
- package/extension/content.js +3 -1
- package/extension/manifest.json +2 -1
- package/index.js +284 -48
- package/models/TextInputModel.js +56 -5
- package/models/index.js +20 -6
- package/nul +0 -0
- package/package.json +1 -1
- package/pom/apom-tree-converter.js +308 -39
- package/server/tool-definitions.js +3 -1
- package/server/tool-schemas.js +5 -0
- package/utils/hints-generator.js +46 -4
- package/utils/post-click-diagnostics.js +146 -47
package/models/index.js
CHANGED
|
@@ -38,18 +38,32 @@ const MODEL_REGISTRY = [
|
|
|
38
38
|
TextInputModel, // Default fallback for text-like inputs
|
|
39
39
|
];
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Wrap operation with timeout to prevent hanging
|
|
43
|
+
*/
|
|
44
|
+
async function withTimeout(operation, timeoutMs, operationName) {
|
|
45
|
+
const timeoutPromise = new Promise((_, reject) =>
|
|
46
|
+
setTimeout(() => reject(new Error(`${operationName} timed out after ${timeoutMs}ms`)), timeoutMs)
|
|
47
|
+
);
|
|
48
|
+
return Promise.race([operation(), timeoutPromise]);
|
|
49
|
+
}
|
|
50
|
+
|
|
41
51
|
/**
|
|
42
52
|
* Factory class for creating appropriate input models
|
|
43
53
|
*/
|
|
44
54
|
export class InputModelFactory {
|
|
45
55
|
/**
|
|
46
|
-
* Get element info (tagName, inputType)
|
|
56
|
+
* Get element info (tagName, inputType) with timeout
|
|
47
57
|
*/
|
|
48
|
-
static async getElementInfo(element) {
|
|
49
|
-
return await
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
58
|
+
static async getElementInfo(element, timeoutMs = 5000) {
|
|
59
|
+
return await withTimeout(
|
|
60
|
+
() => element.evaluate(el => ({
|
|
61
|
+
tagName: el.tagName.toLowerCase(),
|
|
62
|
+
inputType: el.type?.toLowerCase() || null,
|
|
63
|
+
})),
|
|
64
|
+
timeoutMs,
|
|
65
|
+
'getElementInfo'
|
|
66
|
+
);
|
|
53
67
|
}
|
|
54
68
|
|
|
55
69
|
/**
|
package/nul
ADDED
|
File without changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "chrometools-mcp",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.8",
|
|
4
4
|
"description": "MCP (Model Context Protocol) server for Chrome automation using Puppeteer. Persistent browser sessions, UI framework detection (MUI, Ant Design, etc.), Page Object support, visual testing, Figma comparison. Works seamlessly in WSL, Linux, macOS, and Windows.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
@@ -10,9 +10,10 @@
|
|
|
10
10
|
* Runs in browser context via page.evaluate()
|
|
11
11
|
*
|
|
12
12
|
* @param {boolean} interactiveOnly - Only include interactive elements and their parents
|
|
13
|
+
* @param {boolean} viewportOnly - Only include elements visible in current viewport
|
|
13
14
|
* @returns {Object} APOM tree structure
|
|
14
15
|
*/
|
|
15
|
-
function buildAPOMTree(interactiveOnly = true) {
|
|
16
|
+
function buildAPOMTree(interactiveOnly = true, viewportOnly = false) {
|
|
16
17
|
const pageId = `page_${btoa(window.location.href).replace(/[^a-zA-Z0-9]/g, '').substring(0, 20)}_${Date.now()}`;
|
|
17
18
|
|
|
18
19
|
const result = {
|
|
@@ -26,7 +27,14 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
26
27
|
interactiveCount: 0,
|
|
27
28
|
formCount: 0,
|
|
28
29
|
modalCount: 0,
|
|
29
|
-
maxDepth: 0
|
|
30
|
+
maxDepth: 0,
|
|
31
|
+
viewportOnly: viewportOnly || undefined,
|
|
32
|
+
viewport: viewportOnly ? {
|
|
33
|
+
width: window.innerWidth || document.documentElement.clientWidth,
|
|
34
|
+
height: window.innerHeight || document.documentElement.clientHeight,
|
|
35
|
+
scrollX: window.scrollX || window.pageXOffset,
|
|
36
|
+
scrollY: window.scrollY || window.pageYOffset
|
|
37
|
+
} : undefined
|
|
30
38
|
}
|
|
31
39
|
};
|
|
32
40
|
|
|
@@ -51,8 +59,122 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
51
59
|
// Collect radio and checkbox groups for easier agent access
|
|
52
60
|
result.groups = collectInputGroups(result.tree);
|
|
53
61
|
|
|
62
|
+
// Detect page alerts, warnings, errors, and restrictions
|
|
63
|
+
result.alerts = detectPageAlerts();
|
|
64
|
+
|
|
54
65
|
return result;
|
|
55
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Detect page alerts, warnings, errors, toasts, and restrictions
|
|
69
|
+
* Uses heuristics to find notification elements
|
|
70
|
+
*/
|
|
71
|
+
function detectPageAlerts() {
|
|
72
|
+
const alerts = [];
|
|
73
|
+
const seenTexts = new Set();
|
|
74
|
+
const MIN_TEXT_LENGTH = 10; // Ignore very short texts like "19", "OK"
|
|
75
|
+
|
|
76
|
+
// Helper to add alert avoiding duplicates and substrings
|
|
77
|
+
function addAlert(type, text, source) {
|
|
78
|
+
text = text?.trim();
|
|
79
|
+
if (!text || text.length < MIN_TEXT_LENGTH || text.length > 300) return;
|
|
80
|
+
|
|
81
|
+
// Normalize text
|
|
82
|
+
const normalizedText = text.substring(0, 200);
|
|
83
|
+
|
|
84
|
+
// Skip if already seen or is substring of existing
|
|
85
|
+
if (seenTexts.has(normalizedText)) return;
|
|
86
|
+
for (const seen of seenTexts) {
|
|
87
|
+
if (seen.includes(normalizedText) || normalizedText.includes(seen)) return;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
seenTexts.add(normalizedText);
|
|
91
|
+
alerts.push({ type, text: normalizedText, source });
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// 1. Elements with role="alert" or aria-live (highest priority)
|
|
95
|
+
document.querySelectorAll('[role="alert"], [aria-live="assertive"]').forEach(el => {
|
|
96
|
+
const text = el.textContent?.trim();
|
|
97
|
+
addAlert('alert', text, 'aria');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// 2. Classes containing restriction keywords (high priority - affects functionality)
|
|
101
|
+
const restrictionSelector = '[class*="deactivat"], [class*="disabled"], [class*="blocked"], [class*="restrict"], [class*="suspend"], [class*="inactive"]';
|
|
102
|
+
document.querySelectorAll(restrictionSelector).forEach(el => {
|
|
103
|
+
if (!isVisibleForAlert(el)) return;
|
|
104
|
+
const text = el.textContent?.trim();
|
|
105
|
+
addAlert('restriction', text, 'class');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
// 3. Error classes
|
|
109
|
+
document.querySelectorAll('[class*="error"], [class*="danger"], [class*="invalid"]').forEach(el => {
|
|
110
|
+
if (!isVisibleForAlert(el)) return;
|
|
111
|
+
const text = el.textContent?.trim();
|
|
112
|
+
addAlert('error', text, 'class');
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
// 4. Warning classes
|
|
116
|
+
document.querySelectorAll('[class*="warning"], [class*="warn"], [class*="caution"]').forEach(el => {
|
|
117
|
+
if (!isVisibleForAlert(el)) return;
|
|
118
|
+
const text = el.textContent?.trim();
|
|
119
|
+
addAlert('warning', text, 'class');
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
// 5. Toast/Snackbar notifications (usually important messages)
|
|
123
|
+
document.querySelectorAll('[class*="toast"], [class*="snackbar"], [class*="alert-banner"]').forEach(el => {
|
|
124
|
+
if (!isVisibleForAlert(el)) return;
|
|
125
|
+
const text = el.textContent?.trim();
|
|
126
|
+
addAlert('notification', text, 'toast');
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
// 6. SVG icons with warning colors - only for restriction/deactivated contexts
|
|
130
|
+
const warningColors = ['#FF3B30', '#FAB32F', '#FFCC00', '#FF9500', '#F44336'];
|
|
131
|
+
document.querySelectorAll('svg').forEach(svg => {
|
|
132
|
+
const svgHtml = svg.outerHTML.toLowerCase();
|
|
133
|
+
const hasWarningColor = warningColors.some(c => svgHtml.includes(c.toLowerCase()));
|
|
134
|
+
if (!hasWarningColor) return;
|
|
135
|
+
|
|
136
|
+
// Only consider if parent has restriction-related class or text
|
|
137
|
+
let parent = svg.parentElement;
|
|
138
|
+
let attempts = 0;
|
|
139
|
+
while (parent && attempts < 3) {
|
|
140
|
+
const className = parent.className?.toLowerCase() || '';
|
|
141
|
+
const text = parent.textContent?.trim() || '';
|
|
142
|
+
|
|
143
|
+
// Check if context suggests restriction/warning
|
|
144
|
+
const isRestrictionContext =
|
|
145
|
+
/deactivat|disabled|blocked|restrict|suspend|приостановлен|заблокирован/.test(className + ' ' + text.toLowerCase());
|
|
146
|
+
|
|
147
|
+
if (isRestrictionContext && text.length >= MIN_TEXT_LENGTH) {
|
|
148
|
+
addAlert('restriction', text, 'icon-color');
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
parent = parent.parentElement;
|
|
152
|
+
attempts++;
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
// 7. Elements with semantic key/name attributes
|
|
157
|
+
document.querySelectorAll('[key*="error"], [key*="warning"], [key*="deactivat"], [key*="block"]').forEach(el => {
|
|
158
|
+
const parent = el.closest('div, span, p');
|
|
159
|
+
if (parent) {
|
|
160
|
+
const text = parent.textContent?.trim();
|
|
161
|
+
addAlert('warning', text, 'semantic-attr');
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
// Return only if there are meaningful alerts
|
|
166
|
+
return alerts.length > 0 ? alerts : undefined;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Check if element is visible (for alert detection)
|
|
171
|
+
*/
|
|
172
|
+
function isVisibleForAlert(el) {
|
|
173
|
+
if (el.offsetWidth === 0 || el.offsetHeight === 0) return false;
|
|
174
|
+
const style = window.getComputedStyle(el);
|
|
175
|
+
return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0';
|
|
176
|
+
}
|
|
177
|
+
|
|
56
178
|
/**
|
|
57
179
|
* Collect radio and checkbox groups from the tree
|
|
58
180
|
*/
|
|
@@ -148,6 +270,32 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
148
270
|
return node;
|
|
149
271
|
}
|
|
150
272
|
|
|
273
|
+
/**
|
|
274
|
+
* Filter out null, undefined, and empty string values from object
|
|
275
|
+
* to reduce JSON output size
|
|
276
|
+
*/
|
|
277
|
+
function filterNullValues(obj) {
|
|
278
|
+
if (!obj || typeof obj !== 'object') return obj;
|
|
279
|
+
|
|
280
|
+
const result = {};
|
|
281
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
282
|
+
// Skip null, undefined, and empty strings
|
|
283
|
+
if (value === null || value === undefined || value === '') continue;
|
|
284
|
+
// Skip false for boolean fields (keep true)
|
|
285
|
+
if (value === false) continue;
|
|
286
|
+
// Recursively filter nested objects (but not arrays)
|
|
287
|
+
if (typeof value === 'object' && !Array.isArray(value)) {
|
|
288
|
+
const filtered = filterNullValues(value);
|
|
289
|
+
if (Object.keys(filtered).length > 0) {
|
|
290
|
+
result[key] = filtered;
|
|
291
|
+
}
|
|
292
|
+
} else {
|
|
293
|
+
result[key] = value;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
return result;
|
|
297
|
+
}
|
|
298
|
+
|
|
151
299
|
/**
|
|
152
300
|
* Check if cursor:pointer is explicitly set (not inherited)
|
|
153
301
|
*/
|
|
@@ -229,6 +377,23 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
229
377
|
interactiveElements.add(document.body);
|
|
230
378
|
}
|
|
231
379
|
|
|
380
|
+
/**
|
|
381
|
+
* Check if element is in viewport
|
|
382
|
+
*/
|
|
383
|
+
function isInViewport(el) {
|
|
384
|
+
const rect = el.getBoundingClientRect();
|
|
385
|
+
const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
|
|
386
|
+
const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
|
|
387
|
+
|
|
388
|
+
// Element is in viewport if any part of it is visible
|
|
389
|
+
return (
|
|
390
|
+
rect.bottom > 0 &&
|
|
391
|
+
rect.right > 0 &&
|
|
392
|
+
rect.top < viewportHeight &&
|
|
393
|
+
rect.left < viewportWidth
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
|
|
232
397
|
/**
|
|
233
398
|
* Check if element is visible
|
|
234
399
|
* More reliable check that works with position:fixed elements (Angular Material, etc.)
|
|
@@ -248,6 +413,11 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
248
413
|
// For body element, always consider visible if dimensions > 0
|
|
249
414
|
if (el === document.body) return true;
|
|
250
415
|
|
|
416
|
+
// Check viewport if viewportOnly mode is enabled
|
|
417
|
+
if (viewportOnly && !isInViewport(el)) {
|
|
418
|
+
return false;
|
|
419
|
+
}
|
|
420
|
+
|
|
251
421
|
// Additional check: element should be in viewport or have offsetParent
|
|
252
422
|
// This handles elements inside position:fixed containers (Angular Material)
|
|
253
423
|
return el.offsetParent !== null || style.position === 'fixed' || style.position === 'sticky';
|
|
@@ -297,19 +467,23 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
297
467
|
node = {
|
|
298
468
|
id,
|
|
299
469
|
tag: element.tagName.toLowerCase(),
|
|
300
|
-
position,
|
|
301
470
|
type: elementType.type,
|
|
302
471
|
children: []
|
|
303
472
|
};
|
|
304
473
|
|
|
474
|
+
// Only include position if not static (position is null for static)
|
|
475
|
+
if (position) {
|
|
476
|
+
node.position = position;
|
|
477
|
+
}
|
|
478
|
+
|
|
305
479
|
// Add selector only in includeAll mode
|
|
306
480
|
if (!interactiveOnly) {
|
|
307
481
|
node.selector = selector;
|
|
308
482
|
}
|
|
309
483
|
|
|
310
|
-
// Add metadata for interactive elements
|
|
484
|
+
// Add metadata for interactive elements, filtering out null/undefined values
|
|
311
485
|
if (elementType.metadata) {
|
|
312
|
-
node.metadata = elementType.metadata;
|
|
486
|
+
node.metadata = filterNullValues(elementType.metadata);
|
|
313
487
|
}
|
|
314
488
|
} else {
|
|
315
489
|
// Containers: compact format "tag_id": [children] when interactiveOnly
|
|
@@ -340,8 +514,8 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
340
514
|
if (elementType.type === 'form') {
|
|
341
515
|
result.metadata.formCount++;
|
|
342
516
|
}
|
|
343
|
-
if (position.type === 'fixed' || position.type === 'absolute') {
|
|
344
|
-
if (position.zIndex >= 100) {
|
|
517
|
+
if (position && (position.type === 'fixed' || position.type === 'absolute')) {
|
|
518
|
+
if (position.zIndex && position.zIndex >= 100) {
|
|
345
519
|
result.metadata.modalCount++;
|
|
346
520
|
}
|
|
347
521
|
}
|
|
@@ -365,6 +539,11 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
365
539
|
return compactNode;
|
|
366
540
|
}
|
|
367
541
|
|
|
542
|
+
// Remove empty children array to reduce output size
|
|
543
|
+
if (node.children && node.children.length === 0) {
|
|
544
|
+
delete node.children;
|
|
545
|
+
}
|
|
546
|
+
|
|
368
547
|
return node;
|
|
369
548
|
}
|
|
370
549
|
|
|
@@ -379,37 +558,30 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
379
558
|
|
|
380
559
|
/**
|
|
381
560
|
* Get positioning information
|
|
561
|
+
* Returns null for static position (default) to reduce output size
|
|
382
562
|
*/
|
|
383
563
|
function getPositionInfo(element) {
|
|
384
564
|
const style = window.getComputedStyle(element);
|
|
385
565
|
const position = style.position;
|
|
386
|
-
const zIndex = style.zIndex === 'auto' ? 'auto' : parseInt(style.zIndex, 10);
|
|
387
566
|
|
|
388
|
-
//
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
(position === 'absolute' && zIndex !== 'auto') ||
|
|
393
|
-
(position === 'relative' && zIndex !== 'auto') ||
|
|
394
|
-
parseFloat(style.opacity) < 1 ||
|
|
395
|
-
style.transform !== 'none' ||
|
|
396
|
-
style.filter !== 'none' ||
|
|
397
|
-
style.perspective !== 'none' ||
|
|
398
|
-
style.clipPath !== 'none' ||
|
|
399
|
-
style.mask !== 'none' ||
|
|
400
|
-
style.mixBlendMode !== 'normal' ||
|
|
401
|
-
style.isolation === 'isolate';
|
|
567
|
+
// Skip static position (default) - no need to include
|
|
568
|
+
if (position === 'static') {
|
|
569
|
+
return null;
|
|
570
|
+
}
|
|
402
571
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
hasBackdrop: style.backgroundColor !== 'rgba(0, 0, 0, 0)' &&
|
|
409
|
-
(position === 'fixed' || position === 'absolute'),
|
|
410
|
-
isFullscreen: element.offsetWidth >= window.innerWidth * 0.9 &&
|
|
411
|
-
element.offsetHeight >= window.innerHeight * 0.9
|
|
572
|
+
const zIndex = style.zIndex === 'auto' ? 'auto' : parseInt(style.zIndex, 10);
|
|
573
|
+
|
|
574
|
+
// Only include non-default values
|
|
575
|
+
const result = {
|
|
576
|
+
type: position
|
|
412
577
|
};
|
|
578
|
+
|
|
579
|
+
// Only include zIndex if it's not 'auto'
|
|
580
|
+
if (zIndex !== 'auto') {
|
|
581
|
+
result.zIndex = zIndex;
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
return result;
|
|
413
585
|
}
|
|
414
586
|
|
|
415
587
|
/**
|
|
@@ -446,6 +618,83 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
446
618
|
}
|
|
447
619
|
}
|
|
448
620
|
|
|
621
|
+
/**
|
|
622
|
+
* Check if tag is a custom element (Web Component / Framework component)
|
|
623
|
+
*/
|
|
624
|
+
function isCustomElement(tag) {
|
|
625
|
+
// Custom elements must contain a hyphen
|
|
626
|
+
return tag.includes('-');
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/**
|
|
630
|
+
* Check if element has explicit click binding (framework attributes or addEventListener)
|
|
631
|
+
* Uses monkey-patched addEventListener tracker for framework detection (Angular, React, Vue)
|
|
632
|
+
*/
|
|
633
|
+
function hasExplicitClickBinding(element) {
|
|
634
|
+
// Check for framework-specific click bindings (attributes)
|
|
635
|
+
const attrs = element.attributes;
|
|
636
|
+
for (let i = 0; i < attrs.length; i++) {
|
|
637
|
+
const name = attrs[i].name.toLowerCase();
|
|
638
|
+
// Angular: (click), Angular.js: ng-click
|
|
639
|
+
// Vue: @click, v-on:click
|
|
640
|
+
// React: onClick (but this is a property, not attribute)
|
|
641
|
+
if (name === '(click)' || name === 'ng-click' ||
|
|
642
|
+
name === '@click' || name === 'v-on:click' ||
|
|
643
|
+
name.startsWith('on') && name.includes('click')) {
|
|
644
|
+
return true;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
// Check onclick property
|
|
648
|
+
if (element.onclick) return true;
|
|
649
|
+
// Check onclick attribute
|
|
650
|
+
if (element.hasAttribute('onclick')) return true;
|
|
651
|
+
|
|
652
|
+
// Check for click listeners added via addEventListener (framework detection)
|
|
653
|
+
// This is injected by page-manager.js via evaluateOnNewDocument
|
|
654
|
+
if (typeof window.__hasClickListener === 'function' && window.__hasClickListener(element)) {
|
|
655
|
+
return true;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* Find click handler for interactive elements (bubbling search)
|
|
663
|
+
* Searches UP from element to find ancestor with explicit click binding
|
|
664
|
+
* Returns "tag:id" format like "div:container_19" or null if not found
|
|
665
|
+
*/
|
|
666
|
+
function findClickHandler(element) {
|
|
667
|
+
const tag = element.tagName.toLowerCase();
|
|
668
|
+
|
|
669
|
+
// Check self first - native interactive elements handle their own clicks
|
|
670
|
+
if (hasExplicitClickBinding(element) ||
|
|
671
|
+
['button', 'a', 'input', 'select'].includes(tag) ||
|
|
672
|
+
element.getAttribute('role') === 'button') {
|
|
673
|
+
return null; // element handles its own click, no need for clickTarget
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
// Search UP (like event bubbling) - find ancestor with explicit click handler
|
|
677
|
+
let parent = element.parentElement;
|
|
678
|
+
let depth = 0;
|
|
679
|
+
const maxDepth = 5; // limit to prevent hanging
|
|
680
|
+
|
|
681
|
+
while (parent && depth < maxDepth) {
|
|
682
|
+
if (hasExplicitClickBinding(parent)) {
|
|
683
|
+
// Found real click handler
|
|
684
|
+
const parentId = elementIds.get(parent);
|
|
685
|
+
if (parentId) {
|
|
686
|
+
const parentTag = parent.tagName.toLowerCase();
|
|
687
|
+
return `${parentTag}:${parentId}`;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
parent = parent.parentElement;
|
|
691
|
+
depth++;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// No click handler found - return null (no fallback!)
|
|
695
|
+
return null;
|
|
696
|
+
}
|
|
697
|
+
|
|
449
698
|
/**
|
|
450
699
|
* Check if element is interactive based on various signals
|
|
451
700
|
*/
|
|
@@ -680,12 +929,20 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
680
929
|
|
|
681
930
|
// Generic container - check for JavaScript interactivity
|
|
682
931
|
const interactivityCheck = checkInteractivity(element);
|
|
932
|
+
|
|
933
|
+
// For ALL interactive elements, search for click handler (bubbling)
|
|
934
|
+
// Returns "tag:id" format or null if not found
|
|
935
|
+
const clickTarget = interactivityCheck.isInteractive
|
|
936
|
+
? findClickHandler(element)
|
|
937
|
+
: null;
|
|
938
|
+
|
|
683
939
|
return {
|
|
684
940
|
type: 'container',
|
|
685
941
|
isInteractive: interactivityCheck.isInteractive,
|
|
686
942
|
metadata: interactivityCheck.isInteractive ? {
|
|
687
943
|
text: element.textContent?.trim().substring(0, 100) || '',
|
|
688
|
-
interactivityReason: interactivityCheck.reason
|
|
944
|
+
interactivityReason: interactivityCheck.reason,
|
|
945
|
+
clickTarget: clickTarget || undefined
|
|
689
946
|
} : null
|
|
690
947
|
};
|
|
691
948
|
}
|
|
@@ -711,12 +968,17 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
711
968
|
// Try to find stable class name (excluding framework-specific dynamic classes)
|
|
712
969
|
const stableClass = getStableClassName(element);
|
|
713
970
|
if (stableClass) {
|
|
714
|
-
const
|
|
971
|
+
const escapedClass = CSS.escape(stableClass);
|
|
972
|
+
const classSelector = `.${escapedClass}`;
|
|
715
973
|
// Verify it's unique within parent context
|
|
716
974
|
if (element.parentElement) {
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
975
|
+
try {
|
|
976
|
+
const matches = element.parentElement.querySelectorAll(classSelector);
|
|
977
|
+
if (matches.length === 1 && matches[0] === element) {
|
|
978
|
+
return classSelector;
|
|
979
|
+
}
|
|
980
|
+
} catch (e) {
|
|
981
|
+
// Invalid selector, continue to path-based approach
|
|
720
982
|
}
|
|
721
983
|
}
|
|
722
984
|
}
|
|
@@ -728,10 +990,10 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
728
990
|
while (current && current !== document.body) {
|
|
729
991
|
let selector = current.tagName.toLowerCase();
|
|
730
992
|
|
|
731
|
-
// Add stable class if available
|
|
993
|
+
// Add stable class if available (escaped for CSS selector safety)
|
|
732
994
|
const stableClass = getStableClassName(current);
|
|
733
995
|
if (stableClass) {
|
|
734
|
-
selector += `.${stableClass}`;
|
|
996
|
+
selector += `.${CSS.escape(stableClass)}`;
|
|
735
997
|
}
|
|
736
998
|
|
|
737
999
|
// Add nth-of-type if needed
|
|
@@ -754,6 +1016,7 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
754
1016
|
|
|
755
1017
|
/**
|
|
756
1018
|
* Get stable class name excluding framework-specific dynamic classes
|
|
1019
|
+
* and Tailwind CSS utility classes with special characters
|
|
757
1020
|
* Returns first stable class or null
|
|
758
1021
|
*/
|
|
759
1022
|
function getStableClassName(element) {
|
|
@@ -763,8 +1026,14 @@ function buildAPOMTree(interactiveOnly = true) {
|
|
|
763
1026
|
|
|
764
1027
|
const classes = element.className.split(/\s+/).filter(c => c);
|
|
765
1028
|
|
|
766
|
-
// Filter out framework-specific classes
|
|
1029
|
+
// Filter out framework-specific classes and Tailwind utilities
|
|
767
1030
|
const stableClasses = classes.filter(className => {
|
|
1031
|
+
// Tailwind CSS: classes with special characters that break CSS selectors
|
|
1032
|
+
// Colons for variants (hover:, focus:, md:, etc.)
|
|
1033
|
+
// Slashes for fractions (w-1/2)
|
|
1034
|
+
// Brackets for arbitrary values (bg-[#1da1f2])
|
|
1035
|
+
if (/[:\/\[\]]/.test(className)) return false;
|
|
1036
|
+
|
|
768
1037
|
// React: CSS Modules, Styled Components, Emotion
|
|
769
1038
|
if (/^[a-zA-Z0-9_-]+-[a-zA-Z0-9_-]{5,}$/.test(className)) return false;
|
|
770
1039
|
if (/^css-[a-z0-9]+(-[a-z0-9]+)?$/i.test(className)) return false;
|
|
@@ -477,7 +477,7 @@ export const toolDefinitions = [
|
|
|
477
477
|
},
|
|
478
478
|
{
|
|
479
479
|
name: "analyzePage",
|
|
480
|
-
description: "PRIMARY tool for reading page state. Returns APOM tree: {tree, metadata, groups}. Compact format (default): containers as \"tag_id\":[children] keys, interactive elements as {id, tag, type, position, metadata} without selectors. Use element IDs (e.g., button_45, input_20) with click/type tools. Selectors registered internally for resolution. Use refresh:true after clicks. Efficient: 8-10k tokens vs screenshot 15-25k.",
|
|
480
|
+
description: "PRIMARY tool for reading page state. Returns APOM tree: {tree, metadata, groups}. Compact format (default): containers as \"tag_id\":[children] keys, interactive elements as {id, tag, type, position, metadata} without selectors. Use element IDs (e.g., button_45, input_20) with click/type tools. Selectors registered internally for resolution. Use refresh:true after clicks. Efficient: 8-10k tokens vs screenshot 15-25k. Legend: clickTarget format is \"tag:id\" (e.g., \"kp-chats-list-item:container_58\") - use the id part for clicking. No clickTarget = element handles its own click.",
|
|
481
481
|
inputSchema: {
|
|
482
482
|
type: "object",
|
|
483
483
|
properties: {
|
|
@@ -486,6 +486,8 @@ export const toolDefinitions = [
|
|
|
486
486
|
useLegacyFormat: { type: "boolean", description: "Return legacy format instead of APOM (default: false - APOM is now default)" },
|
|
487
487
|
registerElements: { type: "boolean", description: "Auto-register elements in selector resolver (default: true)" },
|
|
488
488
|
groupBy: { type: "string", description: "Group elements: 'type' or 'flat' (default: 'type')", enum: ["type", "flat"] },
|
|
489
|
+
viewportOnly: { type: "boolean", description: "Only analyze elements in current viewport (default: false). Reduces output for long pages." },
|
|
490
|
+
diff: { type: "boolean", description: "Return only changes since last analysis: {added, removed, changed} (default: false)." },
|
|
489
491
|
},
|
|
490
492
|
},
|
|
491
493
|
},
|
package/server/tool-schemas.js
CHANGED
|
@@ -21,6 +21,8 @@ export const ClickSchema = z.object({
|
|
|
21
21
|
waitAfter: z.number().optional().describe("Milliseconds to wait after click (default: 1500)"),
|
|
22
22
|
screenshot: z.boolean().optional().describe("Capture screenshot after click (default: false for performance)"),
|
|
23
23
|
timeout: z.number().optional().describe("Maximum time to wait for operation in ms (default: 30000)"),
|
|
24
|
+
skipNetworkWait: z.boolean().optional().describe("Skip waiting for network requests (default: false). Use for forms with long-polling/WebSockets to avoid timeouts."),
|
|
25
|
+
networkWaitTimeout: z.number().optional().describe("Maximum time to wait for network requests in ms (default: 3000). Only used if skipNetworkWait is false."),
|
|
24
26
|
}).refine(data => (data.id && !data.selector) || (!data.id && data.selector), {
|
|
25
27
|
message: "Either 'id' or 'selector' must be provided, but not both"
|
|
26
28
|
});
|
|
@@ -31,6 +33,7 @@ export const TypeSchema = z.object({
|
|
|
31
33
|
text: z.string().describe("Text to type"),
|
|
32
34
|
delay: z.number().optional().describe("Delay between keystrokes in ms (default: 30)"),
|
|
33
35
|
clearFirst: z.boolean().optional().describe("Clear field before typing (default: true)"),
|
|
36
|
+
timeout: z.number().optional().describe("Maximum time to wait for operation in ms (default: 30000)"),
|
|
34
37
|
}).refine(data => (data.id && !data.selector) || (!data.id && data.selector), {
|
|
35
38
|
message: "Either 'id' or 'selector' must be provided, but not both"
|
|
36
39
|
});
|
|
@@ -260,6 +263,8 @@ export const AnalyzePageSchema = z.object({
|
|
|
260
263
|
useLegacyFormat: z.boolean().optional().describe("Return legacy format instead of APOM (default: false - APOM is now the default format)"),
|
|
261
264
|
registerElements: z.boolean().optional().describe("Automatically register elements in selector resolver (default: true)"),
|
|
262
265
|
groupBy: z.enum(['type', 'flat']).optional().describe("Group elements by type or return flat structure (default: 'type')"),
|
|
266
|
+
viewportOnly: z.boolean().optional().describe("Only analyze elements visible in current viewport (default: false). Reduces output for long pages."),
|
|
267
|
+
diff: z.boolean().optional().describe("Return only changes since last analysis: {added, removed, changed} (default: false). Useful after clicks to see what changed."),
|
|
263
268
|
});
|
|
264
269
|
|
|
265
270
|
export const GetElementDetailsSchema = z.object({
|
package/utils/hints-generator.js
CHANGED
|
@@ -8,6 +8,20 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export function generateNavigationHints(page, url) {
|
|
10
10
|
return page.evaluate(() => {
|
|
11
|
+
// Helper to get safe class selector (filters Tailwind special chars)
|
|
12
|
+
function getSafeClassSelector(element) {
|
|
13
|
+
if (!element.className || typeof element.className !== 'string') return null;
|
|
14
|
+
const classes = element.className.split(' ')
|
|
15
|
+
.filter(c => c && !/[:\/\[\]]/.test(c))
|
|
16
|
+
.slice(0, 1);
|
|
17
|
+
if (classes.length === 0) return null;
|
|
18
|
+
try {
|
|
19
|
+
return `.${CSS.escape(classes[0])}`;
|
|
20
|
+
} catch (e) {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
11
25
|
const hints = {
|
|
12
26
|
pageType: 'unknown',
|
|
13
27
|
availableActions: [],
|
|
@@ -64,7 +78,7 @@ export function generateNavigationHints(page, url) {
|
|
|
64
78
|
hints.keyElements.push({
|
|
65
79
|
type: 'primary-button',
|
|
66
80
|
text: mainButton.textContent.trim(),
|
|
67
|
-
selector: mainButton.id ? `#${mainButton.id}` :
|
|
81
|
+
selector: mainButton.id ? `#${CSS.escape(mainButton.id)}` : (getSafeClassSelector(mainButton) || 'button'),
|
|
68
82
|
});
|
|
69
83
|
}
|
|
70
84
|
|
|
@@ -74,7 +88,7 @@ export function generateNavigationHints(page, url) {
|
|
|
74
88
|
hints.keyElements.push({
|
|
75
89
|
type: 'notification',
|
|
76
90
|
text: alert.textContent.trim().substring(0, 100),
|
|
77
|
-
selector: alert
|
|
91
|
+
selector: getSafeClassSelector(alert) || '[role="alert"]',
|
|
78
92
|
});
|
|
79
93
|
}
|
|
80
94
|
});
|
|
@@ -91,6 +105,20 @@ export async function generateClickHints(page, selector) {
|
|
|
91
105
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
92
106
|
|
|
93
107
|
return page.evaluate((clickedSelector) => {
|
|
108
|
+
// Helper to get safe class selector (filters Tailwind special chars)
|
|
109
|
+
function getSafeClassSelector(element) {
|
|
110
|
+
if (!element.className || typeof element.className !== 'string') return null;
|
|
111
|
+
const classes = element.className.split(' ')
|
|
112
|
+
.filter(c => c && !/[:\/\[\]]/.test(c))
|
|
113
|
+
.slice(0, 1);
|
|
114
|
+
if (classes.length === 0) return null;
|
|
115
|
+
try {
|
|
116
|
+
return `.${CSS.escape(classes[0])}`;
|
|
117
|
+
} catch (e) {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
94
122
|
const hints = {
|
|
95
123
|
pageChanged: false,
|
|
96
124
|
newElements: [],
|
|
@@ -105,7 +133,7 @@ export async function generateClickHints(page, selector) {
|
|
|
105
133
|
hints.modalOpened = true;
|
|
106
134
|
hints.newElements.push({
|
|
107
135
|
type: 'modal',
|
|
108
|
-
selector: modal
|
|
136
|
+
selector: getSafeClassSelector(modal) || '[role="dialog"]',
|
|
109
137
|
});
|
|
110
138
|
hints.suggestedNext.push('Interact with modal or close it');
|
|
111
139
|
}
|
|
@@ -145,6 +173,20 @@ export async function generateFormSubmitHints(page) {
|
|
|
145
173
|
await new Promise(resolve => setTimeout(resolve, 500));
|
|
146
174
|
|
|
147
175
|
return page.evaluate(() => {
|
|
176
|
+
// Helper to get safe class selector (filters Tailwind special chars)
|
|
177
|
+
function getSafeClassSelector(element) {
|
|
178
|
+
if (!element.className || typeof element.className !== 'string') return null;
|
|
179
|
+
const classes = element.className.split(' ')
|
|
180
|
+
.filter(c => c && !/[:\/\[\]]/.test(c))
|
|
181
|
+
.slice(0, 1);
|
|
182
|
+
if (classes.length === 0) return null;
|
|
183
|
+
try {
|
|
184
|
+
return `.${CSS.escape(classes[0])}`;
|
|
185
|
+
} catch (e) {
|
|
186
|
+
return null;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
148
190
|
const hints = {
|
|
149
191
|
success: false,
|
|
150
192
|
errors: [],
|
|
@@ -173,7 +215,7 @@ export async function generateFormSubmitHints(page) {
|
|
|
173
215
|
if (el.offsetWidth > 0) {
|
|
174
216
|
hints.errors.push({
|
|
175
217
|
text: el.textContent.trim().substring(0, 100),
|
|
176
|
-
selector: el
|
|
218
|
+
selector: getSafeClassSelector(el) || '[aria-invalid="true"]',
|
|
177
219
|
});
|
|
178
220
|
}
|
|
179
221
|
});
|