@empiricalrun/test-gen 0.38.4 → 0.38.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/browser-injected-scripts/annotate-elements.js +541 -159
- package/browser-injected-scripts/annotate-elements.spec.ts +2 -2
- package/dist/actions/assert.js +3 -3
- package/dist/actions/click.js +4 -4
- package/dist/actions/fill.js +3 -3
- package/dist/actions/goto.d.ts.map +1 -1
- package/dist/actions/goto.js +4 -5
- package/dist/actions/hover.js +4 -4
- package/dist/actions/index.d.ts +4 -2
- package/dist/actions/index.d.ts.map +1 -1
- package/dist/actions/index.js +13 -1
- package/dist/actions/press.js +3 -3
- package/dist/actions/skill.d.ts.map +1 -1
- package/dist/actions/skill.js +25 -4
- package/dist/actions/text-content.js +3 -3
- package/dist/actions/utils/index.d.ts.map +1 -1
- package/dist/actions/utils/index.js +25 -0
- package/dist/agent/browsing/utils.d.ts.map +1 -1
- package/dist/agent/browsing/utils.js +71 -3
- package/dist/agent/codegen/skills-retriever.d.ts.map +1 -1
- package/dist/agent/codegen/skills-retriever.js +36 -1
- package/dist/agent/codegen/use-skill.d.ts +2 -1
- package/dist/agent/codegen/use-skill.d.ts.map +1 -1
- package/dist/agent/codegen/use-skill.js +3 -2
- package/dist/agent/master/run.d.ts.map +1 -1
- package/dist/agent/master/run.js +43 -24
- package/dist/agent/master/with-hints.d.ts +2 -2
- package/dist/agent/master/with-hints.d.ts.map +1 -1
- package/dist/agent/master/with-hints.js +2 -2
- package/dist/agent/planner/run-time-planner.d.ts +20 -0
- package/dist/agent/planner/run-time-planner.d.ts.map +1 -0
- package/dist/agent/planner/run-time-planner.js +121 -0
- package/dist/bin/utils/context.d.ts +1 -1
- package/dist/bin/utils/context.d.ts.map +1 -1
- package/dist/bin/utils/context.js +1 -1
- package/dist/bin/utils/platform/web/index.d.ts +1 -0
- package/dist/bin/utils/platform/web/index.d.ts.map +1 -1
- package/dist/bin/utils/platform/web/index.js +27 -1
- package/dist/browser-injected-scripts/annotate-elements.js +541 -159
- package/dist/browser-injected-scripts/annotate-elements.spec.ts +2 -2
- package/dist/evals/master-agent.evals.d.ts.map +1 -1
- package/dist/evals/master-agent.evals.js +2 -1
- package/dist/page/index.d.ts +11 -0
- package/dist/page/index.d.ts.map +1 -0
- package/dist/page/index.js +16 -0
- package/dist/types/index.d.ts +3 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -9,28 +9,20 @@
|
|
|
9
9
|
* @param {string} options.markerClass - CSS class to apply to hint markers.
|
|
10
10
|
* @returns {Object} An object containing annotations map and enable/disable methods.
|
|
11
11
|
*/
|
|
12
|
-
|
|
13
|
-
options = {},
|
|
14
|
-
) {
|
|
12
|
+
function annotateClickableElements(options = {}) {
|
|
15
13
|
const {
|
|
16
14
|
hintCharacterSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ", // Default set of characters for hints
|
|
17
15
|
maxHints = 1000, // Maximum number of hints to generate
|
|
18
16
|
markerClass = "hint-marker", // CSS class for markers
|
|
19
17
|
} = options;
|
|
20
18
|
|
|
19
|
+
const document = window.document;
|
|
21
20
|
const annotationsMap = {};
|
|
21
|
+
const usedHints = new Set();
|
|
22
22
|
let annotationsContainer = null;
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
* no other elements are covering it, regardless of its current
|
|
27
|
-
* viewport visibility.
|
|
28
|
-
*
|
|
29
|
-
* @param {Element} element - The DOM element to check.
|
|
30
|
-
* @returns {boolean} - Returns true if the element is clickable; otherwise, false.
|
|
31
|
-
*/
|
|
32
|
-
|
|
33
|
-
function isElementClickNotBlocked(element) {
|
|
24
|
+
// Check if the element is not blocked and visible for clicking
|
|
25
|
+
function isElementClickNotBlocked(element, windowToAnnotate) {
|
|
34
26
|
const rect = element.getBoundingClientRect();
|
|
35
27
|
|
|
36
28
|
// Calculate the center point of the element
|
|
@@ -41,21 +33,21 @@ window.annotateClickableElements = function annotateClickableElements(
|
|
|
41
33
|
if (
|
|
42
34
|
centerX < 0 ||
|
|
43
35
|
centerY < 0 ||
|
|
44
|
-
centerX > (
|
|
45
|
-
centerY > (
|
|
36
|
+
centerX > (windowToAnnotate.innerWidth || document.documentElement.clientWidth) ||
|
|
37
|
+
centerY > (windowToAnnotate.innerHeight || document.documentElement.clientHeight)
|
|
46
38
|
) {
|
|
47
39
|
// Determine the viewport dimensions
|
|
48
40
|
const viewportWidth =
|
|
49
|
-
|
|
41
|
+
windowToAnnotate.innerWidth || document.documentElement.clientWidth;
|
|
50
42
|
const viewportHeight =
|
|
51
|
-
|
|
43
|
+
windowToAnnotate.innerHeight || document.documentElement.clientHeight;
|
|
52
44
|
|
|
53
45
|
// Calculate the new scroll positions to bring the element into the center of the viewport
|
|
54
46
|
const newScrollX = centerX - viewportWidth / 2;
|
|
55
47
|
const newScrollY = centerY - viewportHeight / 2;
|
|
56
48
|
|
|
57
49
|
// Scroll the window to the new positions
|
|
58
|
-
|
|
50
|
+
windowToAnnotate.scrollTo({
|
|
59
51
|
top: newScrollY,
|
|
60
52
|
left: newScrollX,
|
|
61
53
|
});
|
|
@@ -67,20 +59,12 @@ window.annotateClickableElements = function annotateClickableElements(
|
|
|
67
59
|
return element.contains(topElement);
|
|
68
60
|
}
|
|
69
61
|
|
|
70
|
-
|
|
71
|
-
const topElement = document.elementFromPoint(centerX, centerY);
|
|
62
|
+
const topElement = windowToAnnotate.document.elementFromPoint(centerX, centerY);
|
|
72
63
|
|
|
73
64
|
// Check if the topmost element is the target element or one of its descendants
|
|
74
65
|
return element.contains(topElement);
|
|
75
66
|
}
|
|
76
67
|
|
|
77
|
-
/**
|
|
78
|
-
* Generates a sequence of hint strings based on the provided character set.
|
|
79
|
-
* For example, with 'abc' as the set and maxHints 5, it generates: 'a', 'b', 'c', 'aa', 'ab'
|
|
80
|
-
* @param {string} charset - Characters to use.
|
|
81
|
-
* @param {number} max - Maximum number of hints to generate.
|
|
82
|
-
* @returns {string[]} Array of hint strings.
|
|
83
|
-
*/
|
|
84
68
|
function generateHintStrings(charset, max) {
|
|
85
69
|
const hints = [];
|
|
86
70
|
let length = 1;
|
|
@@ -88,7 +72,11 @@ window.annotateClickableElements = function annotateClickableElements(
|
|
|
88
72
|
while (hints.length < max) {
|
|
89
73
|
const combos = cartesianProduct(Array(length).fill(charset.split("")));
|
|
90
74
|
for (const combo of combos) {
|
|
91
|
-
|
|
75
|
+
const hint = combo.join("");
|
|
76
|
+
if (!usedHints.has(hint)) {
|
|
77
|
+
hints.push(hint);
|
|
78
|
+
usedHints.add(hint);
|
|
79
|
+
}
|
|
92
80
|
if (hints.length >= max) break;
|
|
93
81
|
}
|
|
94
82
|
length++;
|
|
@@ -97,29 +85,19 @@ window.annotateClickableElements = function annotateClickableElements(
|
|
|
97
85
|
return hints;
|
|
98
86
|
}
|
|
99
87
|
|
|
100
|
-
/**
|
|
101
|
-
* Creates a Cartesian product of arrays.
|
|
102
|
-
* @param {Array<Array>} arrays - Array of arrays for Cartesian product.
|
|
103
|
-
* @returns {Array<Array>} Cartesian product.
|
|
104
|
-
*/
|
|
105
88
|
function cartesianProduct(arrays) {
|
|
106
89
|
return arrays.reduce(
|
|
107
90
|
(acc, curr) =>
|
|
108
91
|
acc
|
|
109
92
|
.map((a) => curr.map((b) => a.concat([b])))
|
|
110
93
|
.reduce((a, b) => a.concat(b), []),
|
|
111
|
-
[[]]
|
|
94
|
+
[[]]
|
|
112
95
|
);
|
|
113
96
|
}
|
|
114
97
|
|
|
115
|
-
|
|
116
|
-
* Checks if an element is visible and interactable.
|
|
117
|
-
* Enhanced version inspired by getLocalHintsForElement.
|
|
118
|
-
* @param {Element} element - DOM element.
|
|
119
|
-
* @returns {boolean} True if the element is clickable and visible.
|
|
120
|
-
*/
|
|
98
|
+
// Check if an element is clickable
|
|
121
99
|
function isElementClickable(element) {
|
|
122
|
-
if (!(element instanceof Element)) return false;
|
|
100
|
+
// if (!(element instanceof Element)) return false;
|
|
123
101
|
|
|
124
102
|
const tagName = element.tagName.toLowerCase();
|
|
125
103
|
let isClickable = false;
|
|
@@ -311,11 +289,6 @@ window.annotateClickableElements = function annotateClickableElements(
|
|
|
311
289
|
return isClickable;
|
|
312
290
|
}
|
|
313
291
|
|
|
314
|
-
/**
|
|
315
|
-
* Helper function to determine if an input is selectable (e.g., text, search).
|
|
316
|
-
* @param {Element} input - The input element.
|
|
317
|
-
* @returns {boolean} True if the input type is selectable.
|
|
318
|
-
*/
|
|
319
292
|
function isInputSelectable(input) {
|
|
320
293
|
const selectableTypes = [
|
|
321
294
|
"text",
|
|
@@ -330,68 +303,23 @@ window.annotateClickableElements = function annotateClickableElements(
|
|
|
330
303
|
return selectableTypes.includes(type);
|
|
331
304
|
}
|
|
332
305
|
|
|
333
|
-
|
|
334
|
-
* Generates a unique CSS selector path for a given element.
|
|
335
|
-
* @param {Element} element - The DOM element.
|
|
336
|
-
* @returns {string} A unique CSS selector string.
|
|
337
|
-
*/
|
|
338
|
-
function getUniqueSelector(element) {
|
|
339
|
-
if (element.id) {
|
|
340
|
-
return `#${CSS.escape(element.id)}`;
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
const parts = [];
|
|
344
|
-
while (element && element.nodeType === Node.ELEMENT_NODE) {
|
|
345
|
-
let part = element.nodeName.toLowerCase();
|
|
346
|
-
|
|
347
|
-
// Use classList instead of className to avoid errors
|
|
348
|
-
if (element.classList && element.classList.length > 0) {
|
|
349
|
-
const classes = Array.from(element.classList)
|
|
350
|
-
.filter((cls) => cls.length > 0)
|
|
351
|
-
.map((cls) => `.${CSS.escape(cls)}`)
|
|
352
|
-
.join("");
|
|
353
|
-
part += classes;
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
// Add nth-child if necessary
|
|
357
|
-
const parent = element.parentNode;
|
|
358
|
-
if (parent) {
|
|
359
|
-
const siblings = Array.from(parent.children).filter(
|
|
360
|
-
(sibling) => sibling.nodeName === element.nodeName,
|
|
361
|
-
);
|
|
362
|
-
if (siblings.length > 1) {
|
|
363
|
-
const index =
|
|
364
|
-
Array.prototype.indexOf.call(parent.children, element) + 1;
|
|
365
|
-
part += `:nth-child(${index})`;
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
parts.unshift(part);
|
|
370
|
-
element = element.parentElement;
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
return parts.join(" > ");
|
|
374
|
-
}
|
|
306
|
+
var parentElements = [];
|
|
375
307
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
* @param {Element} el - The clickable DOM element.
|
|
379
|
-
* @param {string} hint - The hint identifier.
|
|
380
|
-
* @param {string} nodePath - Unique CSS selector path for the element.
|
|
381
|
-
* @returns {HTMLElement} The created hint marker element.
|
|
382
|
-
*/
|
|
383
|
-
function createHintMarker(el, hint) {
|
|
308
|
+
// Create a hint marker
|
|
309
|
+
function createHintMarker(el, hint, parentElement) {
|
|
384
310
|
const rect = el.getBoundingClientRect();
|
|
385
|
-
|
|
311
|
+
const parentRect = parentElement.getBoundingClientRect();
|
|
312
|
+
|
|
313
|
+
// Create the marker element
|
|
386
314
|
const marker = document.createElement("div");
|
|
387
315
|
marker.textContent = hint;
|
|
388
316
|
marker.className = markerClass;
|
|
389
|
-
|
|
317
|
+
|
|
390
318
|
// Style the marker
|
|
391
319
|
Object.assign(marker.style, {
|
|
392
320
|
position: "absolute",
|
|
393
|
-
top: `${rect.top +
|
|
394
|
-
left: `${rect.left +
|
|
321
|
+
top: `${rect.top + parentRect.top}px`,
|
|
322
|
+
left: `${rect.left + parentRect.left}px`,
|
|
395
323
|
background:
|
|
396
324
|
"-webkit-gradient(linear, 0% 0%, 0% 100%, from(rgb(255, 247, 133)), to(rgb(255, 197, 66)))",
|
|
397
325
|
padding: "1px 3px 0px",
|
|
@@ -411,116 +339,570 @@ window.annotateClickableElements = function annotateClickableElements(
|
|
|
411
339
|
fontWeight: "bold",
|
|
412
340
|
textShadow: "rgba(255, 255, 255, 0.6) 0px 1px 0px",
|
|
413
341
|
});
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
342
|
+
|
|
343
|
+
// Attach the marker to the specified parent element
|
|
344
|
+
parentElement.appendChild(marker);
|
|
345
|
+
parentElements.push(parentElement);
|
|
417
346
|
return marker;
|
|
418
347
|
}
|
|
348
|
+
|
|
419
349
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
*/
|
|
350
|
+
// Clear existing annotations
|
|
351
|
+
//TODO: Handle clearing annotations
|
|
423
352
|
function clearAnnotations() {
|
|
353
|
+
parentElements.forEach((parentElement) => {
|
|
354
|
+
const markers = parentElement.querySelectorAll(`.${markerClass}`);
|
|
355
|
+
markers.forEach((marker) => marker.remove());
|
|
356
|
+
});
|
|
357
|
+
parentElements = [];
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// Initialize annotations for a given window (including iframes)
|
|
361
|
+
function initializeAnnotations(windowToAnnotate, parentHints, depth) {
|
|
362
|
+
const container =
|
|
363
|
+
parentHints?.nodeName === "IFRAME"
|
|
364
|
+
? parentHints.contentWindow.document.body
|
|
365
|
+
: annotationsContainer;
|
|
366
|
+
|
|
367
|
+
// Ensure the container exists
|
|
368
|
+
if (!container) return;
|
|
369
|
+
|
|
370
|
+
// Filter for clickable elements
|
|
371
|
+
const clickableElements = Array.from(windowToAnnotate.document.querySelectorAll("*")).filter((el) => {
|
|
372
|
+
return isElementClickable(el) && isElementClickNotBlocked(el, windowToAnnotate);
|
|
373
|
+
});
|
|
374
|
+
// Generate hint strings for the clickable elements
|
|
375
|
+
const hints = generateHintStrings(hintCharacterSet, Math.min(maxHints, clickableElements.length));
|
|
376
|
+
|
|
377
|
+
// Create markers for the elements
|
|
378
|
+
clickableElements.slice(0, maxHints).forEach((el, index) => {
|
|
379
|
+
const hint = hints[index];
|
|
380
|
+
const rect = el.getBoundingClientRect();
|
|
381
|
+
|
|
382
|
+
// Use createHintMarker with the specified container
|
|
383
|
+
createHintMarker(el, hint, container);
|
|
384
|
+
el.style.boxShadow = `inset 0 0 0px 2px red`;
|
|
385
|
+
|
|
386
|
+
// Add element details to the annotations map
|
|
387
|
+
annotationsMap[hint] = {
|
|
388
|
+
node: el,
|
|
389
|
+
rect: {
|
|
390
|
+
top: rect.top + windowToAnnotate.scrollY,
|
|
391
|
+
left: rect.left + windowToAnnotate.scrollX,
|
|
392
|
+
width: rect.width,
|
|
393
|
+
height: rect.height,
|
|
394
|
+
},
|
|
395
|
+
depth: [...depth ]
|
|
396
|
+
};
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
// Process iframes recursively
|
|
400
|
+
Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach((iframe) => {
|
|
401
|
+
try {
|
|
402
|
+
const frameWindow = iframe.contentWindow;
|
|
403
|
+
if (frameWindow && iframe.offsetWidth > 0 && iframe.offsetHeight > 0) {
|
|
404
|
+
initializeAnnotations(frameWindow, iframe, [...depth, iframe]);
|
|
405
|
+
}
|
|
406
|
+
} catch (e) {
|
|
407
|
+
console.warn("Cannot access iframe:", e);
|
|
408
|
+
}
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
// Initialize and enable annotations
|
|
414
|
+
function enable() {
|
|
415
|
+
clearAnnotations();
|
|
416
|
+
if (!annotationsContainer) {
|
|
417
|
+
annotationsContainer = document.createElement("div");
|
|
418
|
+
annotationsContainer.className = "annotations";
|
|
419
|
+
Object.assign(annotationsContainer.style, {
|
|
420
|
+
position: "absolute",
|
|
421
|
+
top: "0",
|
|
422
|
+
left: "0",
|
|
423
|
+
width: "100%",
|
|
424
|
+
height: "100%",
|
|
425
|
+
pointerEvents: "none",
|
|
426
|
+
zIndex: "9999",
|
|
427
|
+
});
|
|
428
|
+
document.body.appendChild(annotationsContainer);
|
|
429
|
+
initializeAnnotations(window, null, []);
|
|
430
|
+
} else {
|
|
431
|
+
annotationsContainer.style.display = "block";
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// Destroy annotations
|
|
436
|
+
function destroy() {
|
|
424
437
|
if (annotationsContainer) {
|
|
425
|
-
|
|
438
|
+
clearAnnotations();
|
|
439
|
+
Object.values(annotationsMap).forEach((annotation) => {
|
|
440
|
+
annotation.node.style.boxShadow = "none";
|
|
441
|
+
});
|
|
442
|
+
|
|
443
|
+
annotationsContainer.parentNode.removeChild(annotationsContainer);
|
|
426
444
|
annotationsContainer = null;
|
|
427
445
|
}
|
|
428
446
|
}
|
|
429
447
|
|
|
430
|
-
|
|
431
|
-
* Initializes the annotations by creating the container and adding markers.
|
|
432
|
-
*/
|
|
433
|
-
function initializeAnnotations() {
|
|
434
|
-
clearAnnotations(); // Clear any existing annotations
|
|
448
|
+
enable();
|
|
435
449
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
450
|
+
return {
|
|
451
|
+
annotations: annotationsMap,
|
|
452
|
+
destroy,
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
/* eslint-disable no-undef */
|
|
456
|
+
/**
|
|
457
|
+
* Annotates all clickable elements on the page with unique hint markers.
|
|
458
|
+
* Returns an object containing annotations and methods to enable/disable them.
|
|
459
|
+
*
|
|
460
|
+
* @param {Object} options - Configuration options for hint markers.
|
|
461
|
+
* @param {string} options.hintCharacterSet - Characters to use for generating hint identifiers.
|
|
462
|
+
* @param {number} options.maxHints - Maximum number of hints to generate.
|
|
463
|
+
* @param {string} options.markerClass - CSS class to apply to hint markers.
|
|
464
|
+
* @returns {Object} An object containing annotations map and enable/disable methods.
|
|
465
|
+
*/
|
|
466
|
+
function annotateClickableElements(options = {}) {
|
|
467
|
+
const {
|
|
468
|
+
hintCharacterSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ", // Default set of characters for hints
|
|
469
|
+
maxHints = 1000, // Maximum number of hints to generate
|
|
470
|
+
markerClass = "hint-marker", // CSS class for markers
|
|
471
|
+
} = options;
|
|
449
472
|
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
473
|
+
const document = window.document;
|
|
474
|
+
const annotationsMap = {};
|
|
475
|
+
const usedHints = new Set();
|
|
476
|
+
let annotationsContainer = null;
|
|
453
477
|
|
|
454
|
-
|
|
455
|
-
|
|
478
|
+
// Check if the element is not blocked and visible for clicking
|
|
479
|
+
function isElementClickNotBlocked(element, windowToAnnotate) {
|
|
480
|
+
const rect = element.getBoundingClientRect();
|
|
456
481
|
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
482
|
+
// Calculate the center point of the element
|
|
483
|
+
const centerX = rect.left + rect.width / 2;
|
|
484
|
+
const centerY = rect.top + rect.height / 2;
|
|
460
485
|
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
486
|
+
// check if element is within the viewport
|
|
487
|
+
if (
|
|
488
|
+
centerX < 0 ||
|
|
489
|
+
centerY < 0 ||
|
|
490
|
+
centerX > (windowToAnnotate.innerWidth || document.documentElement.clientWidth) ||
|
|
491
|
+
centerY > (windowToAnnotate.innerHeight || document.documentElement.clientHeight)
|
|
492
|
+
) {
|
|
493
|
+
// Determine the viewport dimensions
|
|
494
|
+
const viewportWidth =
|
|
495
|
+
windowToAnnotate.innerWidth || document.documentElement.clientWidth;
|
|
496
|
+
const viewportHeight =
|
|
497
|
+
windowToAnnotate.innerHeight || document.documentElement.clientHeight;
|
|
498
|
+
|
|
499
|
+
// Calculate the new scroll positions to bring the element into the center of the viewport
|
|
500
|
+
const newScrollX = centerX - viewportWidth / 2;
|
|
501
|
+
const newScrollY = centerY - viewportHeight / 2;
|
|
502
|
+
|
|
503
|
+
// Scroll the window to the new positions
|
|
504
|
+
windowToAnnotate.scrollTo({
|
|
505
|
+
top: newScrollY,
|
|
506
|
+
left: newScrollX,
|
|
507
|
+
});
|
|
508
|
+
|
|
509
|
+
const newRect = element.getBoundingClientRect();
|
|
510
|
+
const newCenterX = newRect.left + newRect.width / 2;
|
|
511
|
+
const newCenterY = newRect.top + newRect.height / 2;
|
|
512
|
+
const topElement = document.elementFromPoint(newCenterX, newCenterY);
|
|
513
|
+
return element.contains(topElement);
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const topElement = windowToAnnotate.document.elementFromPoint(centerX, centerY);
|
|
517
|
+
|
|
518
|
+
// Check if the topmost element is the target element or one of its descendants
|
|
519
|
+
return element.contains(topElement);
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function generateHintStrings(charset, max) {
|
|
523
|
+
const hints = [];
|
|
524
|
+
let length = 1;
|
|
525
|
+
|
|
526
|
+
while (hints.length < max) {
|
|
527
|
+
const combos = cartesianProduct(Array(length).fill(charset.split("")));
|
|
528
|
+
for (const combo of combos) {
|
|
529
|
+
const hint = combo.join("");
|
|
530
|
+
if (!usedHints.has(hint)) {
|
|
531
|
+
hints.push(hint);
|
|
532
|
+
usedHints.add(hint);
|
|
533
|
+
}
|
|
534
|
+
if (hints.length >= max) break;
|
|
535
|
+
}
|
|
536
|
+
length++;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
return hints;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
function cartesianProduct(arrays) {
|
|
543
|
+
return arrays.reduce(
|
|
544
|
+
(acc, curr) =>
|
|
545
|
+
acc
|
|
546
|
+
.map((a) => curr.map((b) => a.concat([b])))
|
|
547
|
+
.reduce((a, b) => a.concat(b), []),
|
|
548
|
+
[[]]
|
|
467
549
|
);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// Check if an element is clickable
|
|
553
|
+
function isElementClickable(element) {
|
|
554
|
+
// if (!(element instanceof Element)) return false;
|
|
555
|
+
|
|
556
|
+
const tagName = element.tagName.toLowerCase();
|
|
557
|
+
let isClickable = false;
|
|
558
|
+
|
|
559
|
+
// Check for aria-disabled
|
|
560
|
+
const ariaDisabled = element.getAttribute("aria-disabled");
|
|
561
|
+
if (ariaDisabled && ["", "true"].includes(ariaDisabled.toLowerCase())) {
|
|
562
|
+
return false; // Element should not be clickable if aria-disabled is true
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Check for visibility
|
|
566
|
+
const style = window.getComputedStyle(element);
|
|
567
|
+
if (
|
|
568
|
+
style.display === "none" ||
|
|
569
|
+
style.visibility === "hidden" ||
|
|
570
|
+
// This is done for cases where opacity is undefined
|
|
571
|
+
// parseFloat(style.opacity) === 0
|
|
572
|
+
style.pointerEvents === "none"
|
|
573
|
+
) {
|
|
574
|
+
return false;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// Check if element is disabled (for applicable elements)
|
|
578
|
+
if (element.disabled) return false;
|
|
579
|
+
|
|
580
|
+
// Check for AngularJS click handlers
|
|
581
|
+
if (!isElementClickable._checkForAngularJs) {
|
|
582
|
+
isElementClickable._checkForAngularJs = (function () {
|
|
583
|
+
const angularElements = document.getElementsByClassName("ng-scope");
|
|
584
|
+
if (angularElements.length === 0) {
|
|
585
|
+
return () => false;
|
|
586
|
+
} else {
|
|
587
|
+
const ngAttributes = [];
|
|
588
|
+
for (const prefix of ["", "data-", "x-"]) {
|
|
589
|
+
for (const separator of ["-", ":", "_"]) {
|
|
590
|
+
ngAttributes.push(`${prefix}ng${separator}click`);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
return function (el) {
|
|
594
|
+
for (const attribute of ngAttributes) {
|
|
595
|
+
if (el.hasAttribute(attribute)) return true;
|
|
596
|
+
}
|
|
597
|
+
return false;
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
})();
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
if (!isClickable && isElementClickable._checkForAngularJs(element)) {
|
|
604
|
+
isClickable = true;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
// Check for onclick attribute or listener
|
|
608
|
+
if (
|
|
609
|
+
element.hasAttribute("onclick") ||
|
|
610
|
+
typeof element.onclick === "function"
|
|
611
|
+
) {
|
|
612
|
+
isClickable = true;
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
// Check for jsaction attribute (commonly used in frameworks like Google's)
|
|
616
|
+
if (!isClickable && element.hasAttribute("jsaction")) {
|
|
617
|
+
const jsactionRules = element.getAttribute("jsaction").split(";");
|
|
618
|
+
for (const jsactionRule of jsactionRules) {
|
|
619
|
+
const ruleSplit = jsactionRule.trim().split(":");
|
|
620
|
+
if (ruleSplit.length >= 1 && ruleSplit.length <= 2) {
|
|
621
|
+
const [eventType] = ruleSplit[0].trim().split(".");
|
|
622
|
+
if (eventType === "click") {
|
|
623
|
+
isClickable = true;
|
|
624
|
+
break;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
// Check for role attributes that imply clickability
|
|
631
|
+
if (!isClickable) {
|
|
632
|
+
const role = element.getAttribute("role");
|
|
633
|
+
const clickableRoles = [
|
|
634
|
+
"button",
|
|
635
|
+
"tab",
|
|
636
|
+
"link",
|
|
637
|
+
"checkbox",
|
|
638
|
+
"menuitem",
|
|
639
|
+
"menuitemcheckbox",
|
|
640
|
+
"menuitemradio",
|
|
641
|
+
"radio",
|
|
642
|
+
"switch",
|
|
643
|
+
];
|
|
644
|
+
if (role && clickableRoles.includes(role.toLowerCase())) {
|
|
645
|
+
isClickable = true;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// Check for contentEditable
|
|
650
|
+
if (!isClickable) {
|
|
651
|
+
const contentEditable = element.getAttribute("contentEditable");
|
|
652
|
+
if (
|
|
653
|
+
contentEditable != null &&
|
|
654
|
+
["", "contenteditable", "true"].includes(contentEditable.toLowerCase())
|
|
655
|
+
) {
|
|
656
|
+
isClickable = true;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
// Tag-specific clickability
|
|
661
|
+
const focusableTags = [
|
|
662
|
+
"a",
|
|
663
|
+
"button",
|
|
664
|
+
"input",
|
|
665
|
+
"select",
|
|
666
|
+
"textarea",
|
|
667
|
+
"object",
|
|
668
|
+
"embed",
|
|
669
|
+
"label",
|
|
670
|
+
"details",
|
|
671
|
+
];
|
|
672
|
+
if (focusableTags.includes(tagName)) {
|
|
673
|
+
switch (tagName) {
|
|
674
|
+
case "a":
|
|
675
|
+
// Ensure it's not just a named anchor without href
|
|
676
|
+
if (element.hasAttribute("href")) {
|
|
677
|
+
isClickable = true;
|
|
678
|
+
}
|
|
679
|
+
break;
|
|
680
|
+
case "input": {
|
|
681
|
+
const type = (element.getAttribute("type") || "").toLowerCase();
|
|
682
|
+
if (
|
|
683
|
+
type !== "hidden" &&
|
|
684
|
+
!element.disabled &&
|
|
685
|
+
!(element.readOnly && isInputSelectable(element))
|
|
686
|
+
) {
|
|
687
|
+
isClickable = true;
|
|
688
|
+
}
|
|
689
|
+
break;
|
|
690
|
+
}
|
|
691
|
+
case "textarea":
|
|
692
|
+
if (!element.disabled && !element.readOnly) {
|
|
693
|
+
isClickable = true;
|
|
694
|
+
}
|
|
695
|
+
break;
|
|
696
|
+
case "button":
|
|
697
|
+
case "select":
|
|
698
|
+
if (!element.disabled) {
|
|
699
|
+
isClickable = true;
|
|
700
|
+
}
|
|
701
|
+
break;
|
|
702
|
+
case "object":
|
|
703
|
+
case "embed":
|
|
704
|
+
isClickable = true;
|
|
705
|
+
break;
|
|
706
|
+
case "label":
|
|
707
|
+
if (element.control && !element.control.disabled) {
|
|
708
|
+
isClickable = true;
|
|
709
|
+
}
|
|
710
|
+
break;
|
|
711
|
+
case "details":
|
|
712
|
+
isClickable = true;
|
|
713
|
+
break;
|
|
714
|
+
default:
|
|
715
|
+
break;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// Check for class names containing 'button' as a possible click target
|
|
720
|
+
if (!isClickable) {
|
|
721
|
+
const className = element.getAttribute("class");
|
|
722
|
+
if (className && className.toLowerCase().includes("button")) {
|
|
723
|
+
isClickable = true;
|
|
724
|
+
} else if (element.classList.contains("cursor-pointer")) {
|
|
725
|
+
isClickable = true;
|
|
726
|
+
} else if (element.classList.contains("v-list-item--link")) {
|
|
727
|
+
// vue specific click handling
|
|
728
|
+
isClickable = true;
|
|
729
|
+
} else if (element.style.cursor === "pointer") {
|
|
730
|
+
isClickable = true;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// Check for tabindex
|
|
735
|
+
if (!isClickable) {
|
|
736
|
+
const tabIndexValue = element.getAttribute("tabindex");
|
|
737
|
+
const tabIndex = tabIndexValue ? parseInt(tabIndexValue) : -1;
|
|
738
|
+
if (tabIndex >= 0 && !isNaN(tabIndex)) {
|
|
739
|
+
isClickable = true;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
return isClickable;
|
|
744
|
+
}
|
|
468
745
|
|
|
746
|
+
function isInputSelectable(input) {
|
|
747
|
+
const selectableTypes = [
|
|
748
|
+
"text",
|
|
749
|
+
"search",
|
|
750
|
+
"password",
|
|
751
|
+
"url",
|
|
752
|
+
"email",
|
|
753
|
+
"number",
|
|
754
|
+
"tel",
|
|
755
|
+
];
|
|
756
|
+
const type = (input.getAttribute("type") || "").toLowerCase();
|
|
757
|
+
return selectableTypes.includes(type);
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
var parentElements = [];
|
|
761
|
+
|
|
762
|
+
// Create a hint marker
|
|
763
|
+
function createHintMarker(el, hint, parentElement) {
|
|
764
|
+
const rect = el.getBoundingClientRect();
|
|
765
|
+
const parentRect = parentElement.getBoundingClientRect();
|
|
766
|
+
|
|
767
|
+
// Create the marker element
|
|
768
|
+
const marker = document.createElement("div");
|
|
769
|
+
marker.textContent = hint;
|
|
770
|
+
marker.className = markerClass;
|
|
771
|
+
|
|
772
|
+
// Style the marker
|
|
773
|
+
Object.assign(marker.style, {
|
|
774
|
+
position: "absolute",
|
|
775
|
+
top: `${rect.top + parentRect.top}px`,
|
|
776
|
+
left: `${rect.left + parentRect.left}px`,
|
|
777
|
+
background:
|
|
778
|
+
"-webkit-gradient(linear, 0% 0%, 0% 100%, from(rgb(255, 247, 133)), to(rgb(255, 197, 66)))",
|
|
779
|
+
padding: "1px 3px 0px",
|
|
780
|
+
borderRadius: "3px",
|
|
781
|
+
border: "1px solid rgb(227, 190, 35)",
|
|
782
|
+
fontSize: "11px",
|
|
783
|
+
pointerEvents: "none",
|
|
784
|
+
zIndex: "10000",
|
|
785
|
+
whiteSpace: "nowrap",
|
|
786
|
+
overflow: "hidden",
|
|
787
|
+
boxShadow: "rgba(0, 0, 0, 0.3) 0px 3px 7px 0px",
|
|
788
|
+
letterSpacing: 0,
|
|
789
|
+
minHeight: 0,
|
|
790
|
+
lineHeight: "100%",
|
|
791
|
+
color: "rgb(48, 37, 5)",
|
|
792
|
+
fontFamily: "Helvetica, Arial, sans-serif",
|
|
793
|
+
fontWeight: "bold",
|
|
794
|
+
textShadow: "rgba(255, 255, 255, 0.6) 0px 1px 0px",
|
|
795
|
+
});
|
|
796
|
+
|
|
797
|
+
// Attach the marker to the specified parent element
|
|
798
|
+
parentElement.appendChild(marker);
|
|
799
|
+
parentElements.push(parentElement);
|
|
800
|
+
return marker;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
// Clear existing annotations
|
|
805
|
+
//TODO: Handle clearing annotations
|
|
806
|
+
function clearAnnotations() {
|
|
807
|
+
parentElements.forEach((parentElement) => {
|
|
808
|
+
const markers = parentElement.querySelectorAll(`.${markerClass}`);
|
|
809
|
+
markers.forEach((marker) => marker.remove());
|
|
810
|
+
});
|
|
811
|
+
parentElements = [];
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
// Initialize annotations for a given window (including iframes)
|
|
815
|
+
function initializeAnnotations(windowToAnnotate, parentHints, depth) {
|
|
816
|
+
const container =
|
|
817
|
+
parentHints?.nodeName === "IFRAME"
|
|
818
|
+
? parentHints.contentWindow.document.body
|
|
819
|
+
: annotationsContainer;
|
|
820
|
+
|
|
821
|
+
// Ensure the container exists
|
|
822
|
+
if (!container) return;
|
|
823
|
+
|
|
824
|
+
// Filter for clickable elements
|
|
825
|
+
const clickableElements = Array.from(windowToAnnotate.document.querySelectorAll("*")).filter((el) => {
|
|
826
|
+
return isElementClickable(el) && isElementClickNotBlocked(el, windowToAnnotate);
|
|
827
|
+
});
|
|
828
|
+
// Generate hint strings for the clickable elements
|
|
829
|
+
const hints = generateHintStrings(hintCharacterSet, Math.min(maxHints, clickableElements.length));
|
|
830
|
+
|
|
831
|
+
// Create markers for the elements
|
|
469
832
|
clickableElements.slice(0, maxHints).forEach((el, index) => {
|
|
470
833
|
const hint = hints[index];
|
|
471
|
-
const nodePath = getUniqueSelector(el);
|
|
472
834
|
const rect = el.getBoundingClientRect();
|
|
473
|
-
|
|
474
|
-
//
|
|
475
|
-
createHintMarker(el, hint,
|
|
835
|
+
|
|
836
|
+
// Use createHintMarker with the specified container
|
|
837
|
+
createHintMarker(el, hint, container);
|
|
476
838
|
el.style.boxShadow = `inset 0 0 0px 2px red`;
|
|
477
839
|
|
|
478
|
-
//
|
|
840
|
+
// Add element details to the annotations map
|
|
479
841
|
annotationsMap[hint] = {
|
|
480
842
|
node: el,
|
|
481
|
-
nodePath: nodePath,
|
|
482
843
|
rect: {
|
|
483
|
-
top: rect.top +
|
|
484
|
-
left: rect.left +
|
|
844
|
+
top: rect.top + windowToAnnotate.scrollY,
|
|
845
|
+
left: rect.left + windowToAnnotate.scrollX,
|
|
485
846
|
width: rect.width,
|
|
486
847
|
height: rect.height,
|
|
487
|
-
right: rect.right + window.scrollX,
|
|
488
|
-
bottom: rect.bottom + window.scrollY,
|
|
489
848
|
},
|
|
849
|
+
depth: [...depth ]
|
|
490
850
|
};
|
|
491
851
|
});
|
|
852
|
+
|
|
853
|
+
// Process iframes recursively
|
|
854
|
+
Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach((iframe) => {
|
|
855
|
+
try {
|
|
856
|
+
const frameWindow = iframe.contentWindow;
|
|
857
|
+
if (frameWindow && iframe.offsetWidth > 0 && iframe.offsetHeight > 0) {
|
|
858
|
+
initializeAnnotations(frameWindow, iframe, [...depth, iframe]);
|
|
859
|
+
}
|
|
860
|
+
} catch (e) {
|
|
861
|
+
console.warn("Cannot access iframe:", e);
|
|
862
|
+
}
|
|
863
|
+
});
|
|
492
864
|
}
|
|
865
|
+
|
|
493
866
|
|
|
494
|
-
|
|
495
|
-
* Enables the annotations by making the annotations container visible.
|
|
496
|
-
*/
|
|
867
|
+
// Initialize and enable annotations
|
|
497
868
|
function enable() {
|
|
869
|
+
clearAnnotations();
|
|
498
870
|
if (!annotationsContainer) {
|
|
499
|
-
|
|
871
|
+
annotationsContainer = document.createElement("div");
|
|
872
|
+
annotationsContainer.className = "annotations";
|
|
873
|
+
Object.assign(annotationsContainer.style, {
|
|
874
|
+
position: "absolute",
|
|
875
|
+
top: "0",
|
|
876
|
+
left: "0",
|
|
877
|
+
width: "100%",
|
|
878
|
+
height: "100%",
|
|
879
|
+
pointerEvents: "none",
|
|
880
|
+
zIndex: "9999",
|
|
881
|
+
});
|
|
882
|
+
document.body.appendChild(annotationsContainer);
|
|
883
|
+
initializeAnnotations(window, null, []);
|
|
500
884
|
} else {
|
|
501
885
|
annotationsContainer.style.display = "block";
|
|
502
886
|
}
|
|
503
887
|
}
|
|
504
888
|
|
|
505
|
-
|
|
506
|
-
* removes all generated hints from DOM
|
|
507
|
-
*/
|
|
889
|
+
// Destroy annotations
|
|
508
890
|
function destroy() {
|
|
509
891
|
if (annotationsContainer) {
|
|
892
|
+
clearAnnotations();
|
|
510
893
|
Object.values(annotationsMap).forEach((annotation) => {
|
|
511
894
|
annotation.node.style.boxShadow = "none";
|
|
512
895
|
});
|
|
513
896
|
|
|
514
897
|
annotationsContainer.parentNode.removeChild(annotationsContainer);
|
|
898
|
+
annotationsContainer = null;
|
|
515
899
|
}
|
|
516
900
|
}
|
|
517
901
|
|
|
518
|
-
// Initialize annotations upon first run
|
|
519
902
|
enable();
|
|
520
903
|
|
|
521
|
-
// Return the desired object
|
|
522
904
|
return {
|
|
523
905
|
annotations: annotationsMap,
|
|
524
906
|
destroy,
|
|
525
907
|
};
|
|
526
|
-
}
|
|
908
|
+
}
|