@shiplightai/sdk 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +280 -0
- package/dist/agentHelpers-UCLT5EKK.js +1 -0
- package/dist/agentLogin-ARB3NEO4.js +1 -0
- package/dist/chunk-6H2NJBNL.js +1 -0
- package/dist/chunk-GDTCZALZ.js +192 -0
- package/dist/chunk-GPZJYXUG.js +3880 -0
- package/dist/chunk-KFC5I6R5.js +14 -0
- package/dist/chunk-QIBDXB3J.js +22 -0
- package/dist/chunk-UFLZ3URR.js +1 -0
- package/dist/chunk-UHZTPBZ3.js +197 -0
- package/dist/chunk-YR4E7JSB.js +3 -0
- package/dist/handler-TPOFKKIB.js +1 -0
- package/dist/index.d.ts +446 -0
- package/dist/index.js +44 -0
- package/dist/task-57MAWXLN.js +190 -0
- package/package.json +76 -0
|
@@ -0,0 +1,3880 @@
|
|
|
1
|
+
import{a as h}from"./chunk-YR4E7JSB.js";import{g as R}from"./chunk-UFLZ3URR.js";import{z as I}from"zod";var q=class{constructor(){this.tools=new Map}register(e){if(this.tools.has(e.name))throw new Error(`Tool '${e.name}' is already registered`);let i={name:e.name,description:e.description,schema:e.schema,execute:e.execute,usesElementIndex:e.usesElementIndex??!1,availability:{openai:e.availability?.openai??!0,mcp:e.availability?.mcp??!0}};this.tools.set(e.name,i)}get(e){return this.tools.get(e)}has(e){return this.tools.has(e)}getToolNames(){return Array.from(this.tools.keys())}getTools(){return Array.from(this.tools.values())}async execute(e,i,o){let l=this.tools.get(e);if(!l)throw new Error(`Tool not found: ${e}`);try{let t=i?.description,r={...i};delete r.description;let n=l.schema.parse(r),d={...o,actionDescription:t};return await l.execute(n,d)}catch(t){if(t instanceof I.ZodError){let r=t.issues.map(n=>`${n.path.join(".")}: ${n.message}`).join(", ");return{success:!1,error:`Invalid arguments for tool '${e}': ${r}`,actionEntity:{action_description:i?.description||`${e} (validation failed)`,action_data:{action_name:e,kwargs:i},feedback:`Validation error: ${r}`}}}throw t}}clear(){this.tools.clear()}size(){return this.tools.size}buildActionUnionSchema(){let e=this.getTools().filter(i=>i.availability.openai);return this.buildUnionSchemaFromTools(e)}buildActionUnionSchemaForTools(e){let i=new Set(e),o=this.getTools().filter(l=>i.has(l.name));return this.buildUnionSchemaFromTools(o)}buildUnionSchemaFromTools(e){if(e.length===0)return I.object({done:I.any()});let i=e.map(r=>{let n=r.schema;if(n instanceof I.ZodObject){let d=n._def.shape();Object.keys(d).length===0&&(n=I.object({_empty:I.boolean().optional()}))}return I.object({[r.name]:n})});if(i.length===1)return i[0];let[o,l,...t]=i;return I.union([o,l,...t])}},ie=new q;import{createAnthropic as K}from"@ai-sdk/anthropic";function H(e){let o=R().env?.ANTHROPIC_API_KEY;if(!o)throw new Error("ANTHROPIC_API_KEY not configured in SDK config");return h.debug(`Using Anthropic provider: model=${e}`),K({apiKey:o})(e)}function M(e){return{anthropic:{structuredOutputMode:"jsonTool"}}}import{createGoogleGenerativeAI as J}from"@ai-sdk/google";import{createVertex as Q}from"@ai-sdk/google-vertex";var O={MEDIA_RESOLUTION_HIGH:"MEDIA_RESOLUTION_HIGH",MEDIA_RESOLUTION_MEDIUM:"MEDIA_RESOLUTION_MEDIUM",MEDIA_RESOLUTION_LOW:"MEDIA_RESOLUTION_LOW"};function _(){let o=(R().env||{}).GOOGLE_GENAI_USE_VERTEXAI;return o==="True"||o==="true"}function P(e){let o=R().env||{};if(_()){let r=o.GOOGLE_CLOUD_PROJECT;if(!r)throw new Error("GOOGLE_CLOUD_PROJECT is required when using Vertex AI");let n=e==="gemini-3-flash-preview"?"global":o.GOOGLE_CLOUD_LOCATION;if(!n)throw new Error("GOOGLE_CLOUD_LOCATION is required when using Vertex AI");return h.debug(`Using Vertex AI provider: model=${e}, location=${n}`),Q({project:r,location:n})(e)}let l=o.GOOGLE_API_KEY||o.GOOGLE_GENERATIVE_AI_API_KEY;if(!l)throw new Error("Google API key is missing. Set GOOGLE_API_KEY or GOOGLE_GENERATIVE_AI_API_KEY in SDK config or environment.");return h.debug(`Using Google AI provider (API key): model=${e}`),J({apiKey:l})(e)}function W(e,i){let o={thinkingConfig:{thinkingBudget:512,includeThoughts:!0}},l={thinkingConfig:{thinkingLevel:"minimal",includeThoughts:!0},mediaResolution:O.MEDIA_RESOLUTION_HIGH},t;switch(i){case"gemini-3-flash-preview":t={...l};break;default:t={...o},e===1&&(t.mediaResolution=O.MEDIA_RESOLUTION_HIGH)}return _()?{vertex:t}:{google:t}}function be(e){if(e.startsWith("claude-"))return H(e);if(e.startsWith("gemini-"))return P(e);throw new Error(`Unsupported model: ${e}. Use 'claude-*' or 'gemini-*' models.`)}function ye(e,i){return e.startsWith("claude-")?M(e):W(i,e)}import S from"sharp";var F=768;async function D(e,i){let l=await S(e).metadata(),t=l.width||0,r=l.height||0;if(t===0||r===0)throw new Error("Invalid image dimensions");let n=r,d=0,c=Math.floor((t-n)/2),u=Math.max(0,t-n),a=(p,b)=>{let g=S(e).extract({left:p,top:0,width:b,height:n});return i?.resize&&(g=g.resize(F,F)),g.png().toBuffer()},[m,w,E]=await Promise.all([a(d,Math.min(n,t)),a(c,Math.min(n,t-c)),a(u,Math.min(n,t-u))]);return[m,w,E]}async function V(e){let i=S(e),o=await i.metadata(),l=o.width||0,t=o.height||0;if(l===0||t===0)throw new Error("Invalid image dimensions");let{data:r}=await i.grayscale().raw().toBuffer({resolveWithObject:!0}),n=[];for(let d=0;d<t;d++){n[d]=[];for(let c=0;c<l;c++)n[d][c]=r[d*l+c]}return{pixels:n,width:l,height:t}}var X=new Set(["button","link","textbox","checkbox","radio","combobox","listbox","menuitem","menuitemcheckbox","menuitemradio","option","tab","switch","slider","spinbutton","searchbox","scrollbar","treeitem","gridcell"]),$=new Set(["click","mousedown","mouseup","dblclick","pointerdown","pointerup","touchstart","touchend"]),B=["[onclick]","[onmousedown]","[ontouchstart]","div","span","li","tr","td","[role]",'[class*="btn"]','[class*="button"]','[class*="click"]',"[data-action]","[data-click]"],G=500;function Y(e,i){return e.length>i?e.slice(0,i)+"...":e}var z=["title","type","checked","name","role","value","placeholder","data-date-format","alt","aria-label","aria-expanded","data-state","aria-checked","data-id","data-testid","data-test-id","data-handlepos","data-item-id"],U={"react-flow__(\\S+)":"$1"};var j=class{constructor(e,i=null){this.isVisible=e,this.parent=i}},N=class extends j{constructor(e,i,o=null){super(i,o),this.text=e,this.type="TEXT_NODE"}hasParentWithHighlightIndex(){let e=this.parent;for(;e!==null;){if(e.highlightIndex!==null)return!0;e=e.parent}return!1}isParentInViewport(){return this.parent===null?!1:this.parent.isInViewport}isParentTopElement(){return this.parent===null?!1:this.parent.isTopElement}},C=class L extends j{constructor(i,o,l,t,r,n=!1,d=!1,c=!1,u=!1,a=!1,m=!1,w=null,E=null,p=null,b=null,g=null){super(r,g),this.tagName=i,this.xpath=o,this.attributes=l,this.children=t,this.isInteractive=n,this.isScrollable=d,this.markAsClickable=c,this.isTopElement=u,this.isInViewport=a,this.shadowRoot=m,this.highlightIndex=w,this.viewportCoordinates=E,this.pageCoordinates=p,this.viewportInfo=b,this.isNew=null}getAllTextTillNextClickableElement(i=-1){let o=[],l=(t,r)=>{if(!(i!==-1&&r>i)&&!(t instanceof L&&t!==this&&t.highlightIndex!==null)){if(t instanceof N)o.push(t.text);else if(t instanceof L)for(let n of t.children)l(n,r+1)}};return l(this,0),o.join(`
|
|
2
|
+
`).trim()}clickableElementsToString(i){let o=i?.includeAttributes??z,l=i?.includeClassesWithRename??U,t=[],r=(n,d)=>{let c=d,u=" ".repeat(d);if(n instanceof L){if(n.highlightIndex!==null){c+=1;let a=n.isScrollable?"":n.getAllTextTillNextClickableElement(),m=null;if(o.length>0){let s={};for(let y of Object.keys(n.attributes))if(o.includes(y)){let v=n.attributes[y].trim();v!==""&&(s[y]=v)}let f=o.filter(y=>y in s);if(f.length>1){let y=new Set,v={};for(let x of f){let k=s[x];k.length>5&&(k in v?y.add(x):v[k]=x)}for(let x of y)delete s[x]}n.tagName===s.role&&delete s.role;let T=["aria-label","placeholder","title"];for(let y of T)s[y]&&s[y].trim().toLowerCase()===a.trim().toLowerCase()&&delete s[y];Object.keys(s).length>0&&(m=Object.entries(s).map(([y,v])=>`${y}=${Y(v,200)}`).join(" "))}let w=n.isNew?`*[${n.highlightIndex}]`:`[${n.highlightIndex}]`,E=[];if(Object.keys(l).length>0&&n.attributes.class){let f=n.attributes.class.split(/\s+/);for(let T of f)for(let[y,v]of Object.entries(l))try{let x=new RegExp(`^${y}$`);if(T.match(x)){let A=T.replace(x,v);A&&E.push(A);break}}catch{continue}}let p=n.isScrollable?" (SCROLLABLE)":"",b=n.markAsClickable?" (CLICKABLE)":"",g=`${u}${w}${p}${b}<${n.tagName}`;if(E.length>0&&(g+=` ${E.join(" ")}`),m&&(g+=` ${m}`),a){let s=a.trim();m||(g+=" "),g+=`>${s}`}else m||(g+=" ");g+=" />",t.push(g)}for(let a of n.children)r(a,c)}else if(n instanceof N){if(n.hasParentWithHighlightIndex())return;n.parent&&n.parent.isVisible&&n.parent.isTopElement&&t.push(`${u}${n.text}`)}};return r(this,0),t.join(`
|
|
3
|
+
`)}};var Z=`(
|
|
4
|
+
args = {
|
|
5
|
+
doHighlightElements: true,
|
|
6
|
+
focusHighlightIndex: -1,
|
|
7
|
+
viewportExpansion: 0,
|
|
8
|
+
debugMode: false,
|
|
9
|
+
interactiveClassNames: [],
|
|
10
|
+
alwaysHighlightFileInput: false,
|
|
11
|
+
}
|
|
12
|
+
) => {
|
|
13
|
+
const EVENT_LISTENER_MAPPING = {
|
|
14
|
+
'onclick': 'click',
|
|
15
|
+
'onmousedown': 'mousedown',
|
|
16
|
+
'onmouseup': 'mouseup',
|
|
17
|
+
'ondblclick': 'dblclick',
|
|
18
|
+
'onmouseenter': 'mouseenter',
|
|
19
|
+
'onmouseleave': 'mouseleave',
|
|
20
|
+
'onmousemove': 'mousemove',
|
|
21
|
+
'onmouseout': 'mouseout',
|
|
22
|
+
'onmouseover': 'mouseover',
|
|
23
|
+
'onmouseup': 'mouseup',
|
|
24
|
+
'onmousewheel': 'mousewheel',
|
|
25
|
+
'onscroll': 'scroll',
|
|
26
|
+
'onselect': 'select',
|
|
27
|
+
'onchange': 'change',
|
|
28
|
+
'onfocus': 'focus',
|
|
29
|
+
'onblur': 'blur',
|
|
30
|
+
'onkeydown': 'keydown',
|
|
31
|
+
'onkeyup': 'keyup',
|
|
32
|
+
'onkeypress': 'keypress',
|
|
33
|
+
'oninput': 'input',
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const INTERACTION_EVENTS = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave'];
|
|
37
|
+
|
|
38
|
+
const {
|
|
39
|
+
doHighlightElements,
|
|
40
|
+
focusHighlightIndex,
|
|
41
|
+
viewportExpansion,
|
|
42
|
+
debugMode,
|
|
43
|
+
interactiveClassNames,
|
|
44
|
+
alwaysHighlightFileInput,
|
|
45
|
+
} = args;
|
|
46
|
+
|
|
47
|
+
const buttonClassNames = ['button', 'dropdown-toggle'];
|
|
48
|
+
const heuristicClassPattern = /\\b(btn|const clickable|menu|item|entry|link)\\b/i;
|
|
49
|
+
const containerSelectors = 'button,a,[role="button"],.menu,.dropdown,.list,.toolbar';
|
|
50
|
+
|
|
51
|
+
let highlightIndex = 0; // Reset highlight index
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Helper function to check if element has any of the specified class names.
|
|
55
|
+
*
|
|
56
|
+
* @param {HTMLElement} element - The element to check.
|
|
57
|
+
* @param {string[]} classNames - Array of class names to check for.
|
|
58
|
+
* @returns {boolean} Whether the element has any of the specified class names.
|
|
59
|
+
*/
|
|
60
|
+
function hasAnyClassName(element, classNames) {
|
|
61
|
+
if (!element.classList || !classNames || classNames.length === 0) return false;
|
|
62
|
+
return classNames.some(className => element.classList.contains(className));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Add caching mechanisms at the top level
|
|
66
|
+
const DOM_CACHE = {
|
|
67
|
+
boundingRects: new WeakMap(),
|
|
68
|
+
clientRects: new WeakMap(),
|
|
69
|
+
computedStyles: new WeakMap(),
|
|
70
|
+
nodeEventListeners: new WeakMap(),
|
|
71
|
+
clearCache: () => {
|
|
72
|
+
DOM_CACHE.boundingRects = new WeakMap();
|
|
73
|
+
DOM_CACHE.clientRects = new WeakMap();
|
|
74
|
+
DOM_CACHE.computedStyles = new WeakMap();
|
|
75
|
+
DOM_CACHE.nodeEventListeners = new WeakMap();
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Gets the cached bounding rect for an element.
|
|
81
|
+
*
|
|
82
|
+
* @param {HTMLElement} element - The element to get the bounding rect for.
|
|
83
|
+
* @returns {DOMRect | null} The cached bounding rect, or null if the element is not found.
|
|
84
|
+
*/
|
|
85
|
+
function getCachedBoundingRect(element) {
|
|
86
|
+
if (!element) return null;
|
|
87
|
+
|
|
88
|
+
if (DOM_CACHE.boundingRects.has(element)) {
|
|
89
|
+
return DOM_CACHE.boundingRects.get(element);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const rect = element.getBoundingClientRect();
|
|
93
|
+
|
|
94
|
+
if (rect) {
|
|
95
|
+
DOM_CACHE.boundingRects.set(element, rect);
|
|
96
|
+
}
|
|
97
|
+
return rect;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Gets the cached computed style for an element.
|
|
102
|
+
*
|
|
103
|
+
* @param {HTMLElement} element - The element to get the computed style for.
|
|
104
|
+
* @returns {CSSStyleDeclaration | null} The cached computed style, or null if the element is not found.
|
|
105
|
+
*/
|
|
106
|
+
function getCachedComputedStyle(element) {
|
|
107
|
+
if (!element) return null;
|
|
108
|
+
|
|
109
|
+
if (DOM_CACHE.computedStyles.has(element)) {
|
|
110
|
+
return DOM_CACHE.computedStyles.get(element);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const style = window.getComputedStyle(element);
|
|
114
|
+
|
|
115
|
+
if (style) {
|
|
116
|
+
DOM_CACHE.computedStyles.set(element, style);
|
|
117
|
+
}
|
|
118
|
+
return style;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Gets the cached client rects for an element.
|
|
123
|
+
*
|
|
124
|
+
* @param {HTMLElement} element - The element to get the client rects for.
|
|
125
|
+
* @returns {DOMRectList | null} The cached client rects, or null if the element is not found.
|
|
126
|
+
*/
|
|
127
|
+
function getCachedClientRects(element) {
|
|
128
|
+
if (!element) return null;
|
|
129
|
+
|
|
130
|
+
if (DOM_CACHE.clientRects.has(element)) {
|
|
131
|
+
return DOM_CACHE.clientRects.get(element);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const rects = element.getClientRects();
|
|
135
|
+
|
|
136
|
+
if (rects) {
|
|
137
|
+
DOM_CACHE.clientRects.set(element, rects);
|
|
138
|
+
}
|
|
139
|
+
return rects;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Gets the event listeners for a node.
|
|
144
|
+
*
|
|
145
|
+
* @param {HTMLElement} element - The element to get the event listeners for.
|
|
146
|
+
* @returns {string[]} The event listeners for the element.
|
|
147
|
+
*/
|
|
148
|
+
function getNodeEventListeners(element) {
|
|
149
|
+
const set = new Set();
|
|
150
|
+
try {
|
|
151
|
+
if (typeof getEventListeners === 'function') {
|
|
152
|
+
const listeners = getEventListeners(element);
|
|
153
|
+
for (const eventType in listeners) {
|
|
154
|
+
if (listeners[eventType] && listeners[eventType].length > 0) {
|
|
155
|
+
set.add(eventType);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
|
|
161
|
+
if (typeof getEventListenersForNode === 'function') {
|
|
162
|
+
const listeners = getEventListenersForNode(element);
|
|
163
|
+
for (const listener of listeners) {
|
|
164
|
+
if (listener.type) {
|
|
165
|
+
set.add(listener.type);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
for (const attr in EVENT_LISTENER_MAPPING) {
|
|
171
|
+
if (element.hasAttribute(attr) || typeof element[attr] === 'function') {
|
|
172
|
+
set.add(EVENT_LISTENER_MAPPING[attr]);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
} catch (error) {
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const listenedEvents = Array.from(set);
|
|
179
|
+
return listenedEvents;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function getCachedNodeEventListeners(element) {
|
|
183
|
+
if (!element) return null;
|
|
184
|
+
if (DOM_CACHE.nodeEventListeners.has(element)) {
|
|
185
|
+
return DOM_CACHE.nodeEventListeners.get(element);
|
|
186
|
+
}
|
|
187
|
+
const listenedEvents = getNodeEventListeners(element);
|
|
188
|
+
if (listenedEvents) {
|
|
189
|
+
DOM_CACHE.nodeEventListeners.set(element, listenedEvents);
|
|
190
|
+
}
|
|
191
|
+
return listenedEvents;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Hash map of DOM nodes indexed by their highlight index.
|
|
196
|
+
*
|
|
197
|
+
* @type {Object<string, any>}
|
|
198
|
+
*/
|
|
199
|
+
const DOM_HASH_MAP = {};
|
|
200
|
+
|
|
201
|
+
const ID = { current: 0 };
|
|
202
|
+
|
|
203
|
+
const HIGHLIGHT_CONTAINER_ID = "playwright-highlight-container";
|
|
204
|
+
|
|
205
|
+
// Add a WeakMap cache for XPath strings
|
|
206
|
+
const xpathCache = new WeakMap();
|
|
207
|
+
|
|
208
|
+
const existingLabelBoundingBoxes = [];
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Highlights an element in the DOM and returns the index of the next element.
|
|
212
|
+
*
|
|
213
|
+
* @param {HTMLElement} element - The element to highlight.
|
|
214
|
+
* @param {number} index - The index of the element.
|
|
215
|
+
* @param {HTMLElement | null} parentIframe - The parent iframe node.
|
|
216
|
+
* @returns {number} The index of the next element.
|
|
217
|
+
*/
|
|
218
|
+
function highlightElement(element, index, parentIframe = null) {
|
|
219
|
+
if (!element) return index;
|
|
220
|
+
|
|
221
|
+
const overlays = [];
|
|
222
|
+
/**
|
|
223
|
+
* @type {HTMLElement | null}
|
|
224
|
+
*/
|
|
225
|
+
let label = null;
|
|
226
|
+
let labelWidth = 20;
|
|
227
|
+
let labelHeight = 16;
|
|
228
|
+
let cleanupFn = null;
|
|
229
|
+
|
|
230
|
+
try {
|
|
231
|
+
// Create or get highlight container
|
|
232
|
+
let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
|
|
233
|
+
if (!container) {
|
|
234
|
+
container = document.createElement("div");
|
|
235
|
+
container.id = HIGHLIGHT_CONTAINER_ID;
|
|
236
|
+
container.style.position = "fixed";
|
|
237
|
+
container.style.pointerEvents = "none";
|
|
238
|
+
container.style.top = "0";
|
|
239
|
+
container.style.left = "0";
|
|
240
|
+
container.style.width = "100%";
|
|
241
|
+
container.style.height = "100%";
|
|
242
|
+
// Use the maximum valid value in zIndex to ensure the element is not blocked by overlapping elements.
|
|
243
|
+
container.style.zIndex = "2147483647";
|
|
244
|
+
container.style.backgroundColor = 'transparent';
|
|
245
|
+
document.body.appendChild(container);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Get element client rects
|
|
249
|
+
let rects = element.getClientRects(); // Use getClientRects()
|
|
250
|
+
|
|
251
|
+
if (!rects || rects.length === 0) return index; // Exit if no rects
|
|
252
|
+
|
|
253
|
+
// If element is inside an iframe, we need to transform the rects to main document coordinates
|
|
254
|
+
if (parentIframe) {
|
|
255
|
+
const transformedRects = [];
|
|
256
|
+
const iframeRect = parentIframe.getBoundingClientRect();
|
|
257
|
+
|
|
258
|
+
// Get iframe's content area offset and CSS transforms
|
|
259
|
+
const iframeStyle = window.getComputedStyle(parentIframe);
|
|
260
|
+
const borderLeft = parseFloat(iframeStyle.borderLeftWidth) || 0;
|
|
261
|
+
const borderTop = parseFloat(iframeStyle.borderTopWidth) || 0;
|
|
262
|
+
const paddingLeft = parseFloat(iframeStyle.paddingLeft) || 0;
|
|
263
|
+
const paddingTop = parseFloat(iframeStyle.paddingTop) || 0;
|
|
264
|
+
|
|
265
|
+
const contentOffsetX = borderLeft + paddingLeft;
|
|
266
|
+
const contentOffsetY = borderTop + paddingTop;
|
|
267
|
+
|
|
268
|
+
// Extract scale factor from CSS transform
|
|
269
|
+
let scaleX = 1, scaleY = 1;
|
|
270
|
+
const transform = iframeStyle.transform;
|
|
271
|
+
if (transform && transform !== 'none') {
|
|
272
|
+
// Parse scale from transform matrix or scale() function
|
|
273
|
+
const scaleMatch = transform.match(/scale\\(([^)]+)\\)/);
|
|
274
|
+
if (scaleMatch) {
|
|
275
|
+
const scaleValues = scaleMatch[1].split(',').map(v => parseFloat(v.trim()));
|
|
276
|
+
scaleX = scaleValues[0] || 1;
|
|
277
|
+
scaleY = scaleValues[1] || scaleX; // If only one value, use it for both X and Y
|
|
278
|
+
} else {
|
|
279
|
+
// Try to parse matrix() transform
|
|
280
|
+
const matrixMatch = transform.match(/matrix\\(([^)]+)\\)/);
|
|
281
|
+
if (matrixMatch) {
|
|
282
|
+
const values = matrixMatch[1].split(',').map(v => parseFloat(v.trim()));
|
|
283
|
+
if (values.length >= 6) {
|
|
284
|
+
scaleX = values[0]; // a value in matrix(a, b, c, d, e, f)
|
|
285
|
+
scaleY = values[3]; // d value in matrix(a, b, c, d, e, f)
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Get iframe scroll position
|
|
292
|
+
let scrollLeft = 0, scrollTop = 0;
|
|
293
|
+
try {
|
|
294
|
+
const iframeDoc = parentIframe.contentDocument || parentIframe.contentWindow?.document;
|
|
295
|
+
if (iframeDoc) {
|
|
296
|
+
scrollLeft = iframeDoc.documentElement?.scrollLeft || iframeDoc.body?.scrollLeft || 0;
|
|
297
|
+
scrollTop = iframeDoc.documentElement?.scrollTop || iframeDoc.body?.scrollTop || 0;
|
|
298
|
+
}
|
|
299
|
+
} catch (e) {
|
|
300
|
+
console.warn("Cannot access iframe scroll position (cross-origin):", e);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// Transform each rect accounting for iframe scaling
|
|
304
|
+
for (const rect of rects) {
|
|
305
|
+
// Apply scale factor to coordinates and dimensions
|
|
306
|
+
const scaledWidth = rect.width * scaleX;
|
|
307
|
+
const scaledHeight = rect.height * scaleY;
|
|
308
|
+
const scaledTop = rect.top * scaleY;
|
|
309
|
+
const scaledLeft = rect.left * scaleX;
|
|
310
|
+
const scaledScrollTop = scrollTop * scaleY;
|
|
311
|
+
const scaledScrollLeft = scrollLeft * scaleX;
|
|
312
|
+
|
|
313
|
+
const transformedRect = {
|
|
314
|
+
top: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop,
|
|
315
|
+
left: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
316
|
+
bottom: scaledTop + scaledHeight + iframeRect.top + contentOffsetY - scaledScrollTop,
|
|
317
|
+
right: scaledLeft + scaledWidth + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
318
|
+
width: scaledWidth,
|
|
319
|
+
height: scaledHeight,
|
|
320
|
+
x: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
321
|
+
y: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop
|
|
322
|
+
};
|
|
323
|
+
transformedRects.push(transformedRect);
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
rects = transformedRects;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// Generate a color based on the index
|
|
330
|
+
const colors = [
|
|
331
|
+
"#FF0000",
|
|
332
|
+
"#00FF00",
|
|
333
|
+
"#0000FF",
|
|
334
|
+
"#FFA500",
|
|
335
|
+
"#800080",
|
|
336
|
+
"#008080",
|
|
337
|
+
"#FF69B4",
|
|
338
|
+
"#4B0082",
|
|
339
|
+
"#FF4500",
|
|
340
|
+
"#2E8B57",
|
|
341
|
+
"#DC143C",
|
|
342
|
+
"#4682B4",
|
|
343
|
+
];
|
|
344
|
+
const colorIndex = index % colors.length;
|
|
345
|
+
const baseColor = colors[colorIndex];
|
|
346
|
+
const backgroundColor = baseColor + "1A"; // 10% opacity version of the color
|
|
347
|
+
|
|
348
|
+
// No need for iframe offset calculation since rects are already transformed to main document coordinates
|
|
349
|
+
const iframeOffset = { x: 0, y: 0 };
|
|
350
|
+
|
|
351
|
+
// Create fragment to hold overlay elements
|
|
352
|
+
const fragment = document.createDocumentFragment();
|
|
353
|
+
|
|
354
|
+
// Create highlight overlays for each client rect
|
|
355
|
+
for (const rect of rects) {
|
|
356
|
+
if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects
|
|
357
|
+
|
|
358
|
+
const overlay = document.createElement("div");
|
|
359
|
+
overlay.style.position = "fixed";
|
|
360
|
+
overlay.style.border = \`1px solid \${baseColor}\`;
|
|
361
|
+
overlay.style.backgroundColor = "none";
|
|
362
|
+
overlay.style.pointerEvents = "none";
|
|
363
|
+
overlay.style.boxSizing = "border-box";
|
|
364
|
+
|
|
365
|
+
const top = rect.top + iframeOffset.y;
|
|
366
|
+
const left = rect.left + iframeOffset.x;
|
|
367
|
+
|
|
368
|
+
overlay.style.top = \`\${top}px\`;
|
|
369
|
+
overlay.style.left = \`\${left}px\`;
|
|
370
|
+
overlay.style.width = \`\${rect.width}px\`;
|
|
371
|
+
overlay.style.height = \`\${rect.height}px\`;
|
|
372
|
+
|
|
373
|
+
fragment.appendChild(overlay);
|
|
374
|
+
overlays.push({ element: overlay, initialRect: rect }); // Store overlay and its rect
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// Create and position a single label relative to the first rect
|
|
378
|
+
const firstRect = rects[0];
|
|
379
|
+
label = document.createElement("div");
|
|
380
|
+
label.className = "playwright-highlight-label";
|
|
381
|
+
label.style.position = "fixed";
|
|
382
|
+
label.style.background = baseColor;
|
|
383
|
+
label.style.color = "white";
|
|
384
|
+
label.style.padding = "1px 4px";
|
|
385
|
+
label.style.borderRadius = "4px";
|
|
386
|
+
const fontSize = Math.min(index >= 100 ? 8 : 12, Math.max(8, firstRect.height / 2));
|
|
387
|
+
label.style.fontSize = \`\${fontSize}px\`;
|
|
388
|
+
label.textContent = index;
|
|
389
|
+
|
|
390
|
+
// labelWidth = label.offsetWidth > 0 ? label.offsetWidth : labelWidth; // Update actual width if possible
|
|
391
|
+
// labelHeight = label.offsetHeight > 0 ? label.offsetHeight : labelHeight; // Update actual height if possible
|
|
392
|
+
labelWidth *= fontSize / 12;
|
|
393
|
+
labelHeight *= fontSize / 12;
|
|
394
|
+
const digits = index.toString().length;
|
|
395
|
+
labelWidth += (digits - 2) * fontSize / 1.5;
|
|
396
|
+
labelHeight *= fontSize / 12;
|
|
397
|
+
|
|
398
|
+
const firstRectTop = firstRect.top + iframeOffset.y;
|
|
399
|
+
const firstRectLeft = firstRect.left + iframeOffset.x;
|
|
400
|
+
|
|
401
|
+
let labelTop = firstRectTop - labelHeight - 4 * 12 / fontSize;
|
|
402
|
+
let labelLeft = firstRectLeft + firstRect.width - labelWidth - 2;
|
|
403
|
+
|
|
404
|
+
// Adjust label position if first rect is too small
|
|
405
|
+
if (firstRect.width < labelWidth + 4 || firstRect.height < labelHeight + 4) {
|
|
406
|
+
labelTop = firstRectTop - labelHeight;
|
|
407
|
+
labelLeft = firstRectLeft + firstRect.width - labelWidth; // Align with right edge
|
|
408
|
+
if (labelLeft < iframeOffset.x) labelLeft = firstRectLeft; // Prevent going off-left
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// // Check if the label is too close to any existing label
|
|
412
|
+
// const minDistance = 2; // Minimum distance between labels
|
|
413
|
+
// let hasOverlap = true;
|
|
414
|
+
// let attempts = 0;
|
|
415
|
+
// const maxAttempts = 5;
|
|
416
|
+
|
|
417
|
+
// const alignmentPositions = [
|
|
418
|
+
// // align with right edge (current default)
|
|
419
|
+
// firstRectLeft + firstRect.width - labelWidth - 2,
|
|
420
|
+
// // 1/4 of the way from the right edge
|
|
421
|
+
// firstRectLeft + (firstRect.width - labelWidth) / 2 + firstRect.width / 4,
|
|
422
|
+
// // in middle
|
|
423
|
+
// firstRectLeft + (firstRect.width - labelWidth) / 2,
|
|
424
|
+
// firstRectLeft + (firstRect.width - labelWidth) / 2 - firstRect.width / 4,
|
|
425
|
+
// // align with left edge
|
|
426
|
+
// firstRectLeft,
|
|
427
|
+
// ];
|
|
428
|
+
|
|
429
|
+
// while (hasOverlap && attempts < maxAttempts) {
|
|
430
|
+
// hasOverlap = false;
|
|
431
|
+
|
|
432
|
+
// // Use the current alignment position
|
|
433
|
+
// labelLeft = alignmentPositions[attempts];
|
|
434
|
+
|
|
435
|
+
// // Current label bounding box
|
|
436
|
+
// const currentLabel = {
|
|
437
|
+
// xmin: labelLeft,
|
|
438
|
+
// ymin: labelTop,
|
|
439
|
+
// xmax: labelLeft + labelWidth,
|
|
440
|
+
// ymax: labelTop + labelHeight,
|
|
441
|
+
// };
|
|
442
|
+
|
|
443
|
+
// for (const existingLabel of existingLabelBoundingBoxes) {
|
|
444
|
+
// // Check if labels overlap or are too close (using minDistance buffer)
|
|
445
|
+
// if (!(currentLabel.xmax + minDistance < existingLabel.xmin ||
|
|
446
|
+
// currentLabel.xmin > existingLabel.xmax + minDistance ||
|
|
447
|
+
// currentLabel.ymax + minDistance < existingLabel.ymin ||
|
|
448
|
+
// currentLabel.ymin > existingLabel.ymax + minDistance)) {
|
|
449
|
+
// hasOverlap = true;
|
|
450
|
+
// break;
|
|
451
|
+
// }
|
|
452
|
+
// }
|
|
453
|
+
|
|
454
|
+
// if (hasOverlap) {
|
|
455
|
+
// attempts++;
|
|
456
|
+
// }
|
|
457
|
+
// }
|
|
458
|
+
|
|
459
|
+
// Ensure label stays within viewport bounds slightly better
|
|
460
|
+
labelTop = Math.max(0, Math.min(labelTop, window.innerHeight - labelHeight));
|
|
461
|
+
labelLeft = Math.max(0, Math.min(labelLeft, window.innerWidth - labelWidth - 2));
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
label.style.top = \`\${labelTop}px\`;
|
|
465
|
+
label.style.left = \`\${labelLeft}px\`;
|
|
466
|
+
|
|
467
|
+
// Add the label to the existing label bounding boxes
|
|
468
|
+
existingLabelBoundingBoxes.push({
|
|
469
|
+
xmin: labelLeft,
|
|
470
|
+
ymin: labelTop,
|
|
471
|
+
xmax: labelLeft + labelWidth,
|
|
472
|
+
ymax: labelTop + labelHeight,
|
|
473
|
+
});
|
|
474
|
+
|
|
475
|
+
fragment.appendChild(label);
|
|
476
|
+
|
|
477
|
+
// Update positions on scroll/resize
|
|
478
|
+
const updatePositions = () => {
|
|
479
|
+
let newRects = element.getClientRects(); // Get fresh rects
|
|
480
|
+
|
|
481
|
+
// Transform rects if element is inside an iframe (same logic as initial highlighting)
|
|
482
|
+
if (parentIframe) {
|
|
483
|
+
const transformedRects = [];
|
|
484
|
+
const iframeRect = parentIframe.getBoundingClientRect();
|
|
485
|
+
|
|
486
|
+
// Get iframe's content area offset and CSS transforms
|
|
487
|
+
const iframeStyle = window.getComputedStyle(parentIframe);
|
|
488
|
+
const borderLeft = parseFloat(iframeStyle.borderLeftWidth) || 0;
|
|
489
|
+
const borderTop = parseFloat(iframeStyle.borderTopWidth) || 0;
|
|
490
|
+
const paddingLeft = parseFloat(iframeStyle.paddingLeft) || 0;
|
|
491
|
+
const paddingTop = parseFloat(iframeStyle.paddingTop) || 0;
|
|
492
|
+
|
|
493
|
+
const contentOffsetX = borderLeft + paddingLeft;
|
|
494
|
+
const contentOffsetY = borderTop + paddingTop;
|
|
495
|
+
|
|
496
|
+
// Extract scale factor from CSS transform
|
|
497
|
+
let scaleX = 1, scaleY = 1;
|
|
498
|
+
const transform = iframeStyle.transform;
|
|
499
|
+
if (transform && transform !== 'none') {
|
|
500
|
+
// Parse scale from transform matrix or scale() function
|
|
501
|
+
const scaleMatch = transform.match(/scale\\(([^)]+)\\)/);
|
|
502
|
+
if (scaleMatch) {
|
|
503
|
+
const scaleValues = scaleMatch[1].split(',').map(v => parseFloat(v.trim()));
|
|
504
|
+
scaleX = scaleValues[0] || 1;
|
|
505
|
+
scaleY = scaleValues[1] || scaleX; // If only one value, use it for both X and Y
|
|
506
|
+
} else {
|
|
507
|
+
// Try to parse matrix() transform
|
|
508
|
+
const matrixMatch = transform.match(/matrix\\(([^)]+)\\)/);
|
|
509
|
+
if (matrixMatch) {
|
|
510
|
+
const values = matrixMatch[1].split(',').map(v => parseFloat(v.trim()));
|
|
511
|
+
if (values.length >= 6) {
|
|
512
|
+
scaleX = values[0]; // a value in matrix(a, b, c, d, e, f)
|
|
513
|
+
scaleY = values[3]; // d value in matrix(a, b, c, d, e, f)
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
// Get iframe scroll position
|
|
520
|
+
let scrollLeft = 0, scrollTop = 0;
|
|
521
|
+
try {
|
|
522
|
+
const iframeDoc = parentIframe.contentDocument || parentIframe.contentWindow?.document;
|
|
523
|
+
if (iframeDoc) {
|
|
524
|
+
scrollLeft = iframeDoc.documentElement?.scrollLeft || iframeDoc.body?.scrollLeft || 0;
|
|
525
|
+
scrollTop = iframeDoc.documentElement?.scrollTop || iframeDoc.body?.scrollTop || 0;
|
|
526
|
+
}
|
|
527
|
+
} catch (e) {
|
|
528
|
+
console.warn("Cannot access iframe scroll position (cross-origin):", e);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// Transform each rect accounting for iframe scaling
|
|
532
|
+
for (const rect of newRects) {
|
|
533
|
+
// Apply scale factor to coordinates and dimensions
|
|
534
|
+
const scaledWidth = rect.width * scaleX;
|
|
535
|
+
const scaledHeight = rect.height * scaleY;
|
|
536
|
+
const scaledTop = rect.top * scaleY;
|
|
537
|
+
const scaledLeft = rect.left * scaleX;
|
|
538
|
+
const scaledScrollTop = scrollTop * scaleY;
|
|
539
|
+
const scaledScrollLeft = scrollLeft * scaleX;
|
|
540
|
+
|
|
541
|
+
const transformedRect = {
|
|
542
|
+
top: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop,
|
|
543
|
+
left: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
544
|
+
bottom: scaledTop + scaledHeight + iframeRect.top + contentOffsetY - scaledScrollTop,
|
|
545
|
+
right: scaledLeft + scaledWidth + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
546
|
+
width: scaledWidth,
|
|
547
|
+
height: scaledHeight,
|
|
548
|
+
x: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
549
|
+
y: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop
|
|
550
|
+
};
|
|
551
|
+
transformedRects.push(transformedRect);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
newRects = transformedRects;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
const newIframeOffset = { x: 0, y: 0 }; // No offset needed since rects are already transformed
|
|
558
|
+
|
|
559
|
+
// Update each overlay
|
|
560
|
+
overlays.forEach((overlayData, i) => {
|
|
561
|
+
if (i < newRects.length) { // Check if rect still exists
|
|
562
|
+
const newRect = newRects[i];
|
|
563
|
+
const newTop = newRect.top + newIframeOffset.y;
|
|
564
|
+
const newLeft = newRect.left + newIframeOffset.x;
|
|
565
|
+
|
|
566
|
+
overlayData.element.style.top = \`\${newTop}px\`;
|
|
567
|
+
overlayData.element.style.left = \`\${newLeft}px\`;
|
|
568
|
+
overlayData.element.style.width = \`\${newRect.width}px\`;
|
|
569
|
+
overlayData.element.style.height = \`\${newRect.height}px\`;
|
|
570
|
+
overlayData.element.style.display = (newRect.width === 0 || newRect.height === 0) ? 'none' : 'block';
|
|
571
|
+
} else {
|
|
572
|
+
// If fewer rects now, hide extra overlays
|
|
573
|
+
overlayData.element.style.display = 'none';
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
|
|
577
|
+
// If there are fewer new rects than overlays, hide the extras
|
|
578
|
+
if (newRects.length < overlays.length) {
|
|
579
|
+
for (let i = newRects.length; i < overlays.length; i++) {
|
|
580
|
+
overlays[i].element.style.display = 'none';
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// Update label position based on the first new rect
|
|
585
|
+
if (label && newRects.length > 0) {
|
|
586
|
+
const firstNewRect = newRects[0];
|
|
587
|
+
const firstNewRectTop = firstNewRect.top + newIframeOffset.y;
|
|
588
|
+
const firstNewRectLeft = firstNewRect.left + newIframeOffset.x;
|
|
589
|
+
|
|
590
|
+
let newLabelTop = firstNewRectTop + 2;
|
|
591
|
+
let newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth - 2;
|
|
592
|
+
|
|
593
|
+
if (firstNewRect.width < labelWidth + 4 || firstNewRect.height < labelHeight + 4) {
|
|
594
|
+
newLabelTop = firstNewRectTop - labelHeight - 2;
|
|
595
|
+
newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth;
|
|
596
|
+
if (newLabelLeft < newIframeOffset.x) newLabelLeft = firstNewRectLeft;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// Ensure label stays within viewport bounds
|
|
600
|
+
newLabelTop = Math.max(0, Math.min(newLabelTop, window.innerHeight - labelHeight));
|
|
601
|
+
newLabelLeft = Math.max(0, Math.min(newLabelLeft, window.innerWidth - labelWidth));
|
|
602
|
+
|
|
603
|
+
label.style.top = \`\${newLabelTop}px\`;
|
|
604
|
+
label.style.left = \`\${newLabelLeft}px\`;
|
|
605
|
+
label.style.display = 'block';
|
|
606
|
+
} else if (label) {
|
|
607
|
+
// Hide label if element has no rects anymore
|
|
608
|
+
label.style.display = 'none';
|
|
609
|
+
}
|
|
610
|
+
};
|
|
611
|
+
|
|
612
|
+
const throttleFunction = (func, delay) => {
|
|
613
|
+
let lastCall = 0;
|
|
614
|
+
return (...args) => {
|
|
615
|
+
const now = performance.now();
|
|
616
|
+
if (now - lastCall < delay) return;
|
|
617
|
+
lastCall = now;
|
|
618
|
+
return func(...args);
|
|
619
|
+
};
|
|
620
|
+
};
|
|
621
|
+
|
|
622
|
+
// const throttledUpdatePositions = throttleFunction(updatePositions, 16); // ~60fps
|
|
623
|
+
// window.addEventListener('scroll', throttledUpdatePositions, true);
|
|
624
|
+
// window.addEventListener('resize', throttledUpdatePositions);
|
|
625
|
+
|
|
626
|
+
// Add cleanup function
|
|
627
|
+
cleanupFn = () => {
|
|
628
|
+
// window.removeEventListener('scroll', throttledUpdatePositions, true);
|
|
629
|
+
// window.removeEventListener('resize', throttledUpdatePositions);
|
|
630
|
+
// Remove overlay elements if needed
|
|
631
|
+
overlays.forEach(overlay => overlay.element.remove());
|
|
632
|
+
if (label) label.remove();
|
|
633
|
+
};
|
|
634
|
+
|
|
635
|
+
// Then add fragment to container in one operation
|
|
636
|
+
container.appendChild(fragment);
|
|
637
|
+
|
|
638
|
+
return index + 1;
|
|
639
|
+
} finally {
|
|
640
|
+
// Store cleanup function for later use
|
|
641
|
+
if (cleanupFn) {
|
|
642
|
+
// Keep a reference to cleanup functions in a global array
|
|
643
|
+
(window._highlightCleanupFunctions = window._highlightCleanupFunctions || []).push(cleanupFn);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Gets the position of an element in its parent.
|
|
651
|
+
*
|
|
652
|
+
* @param {HTMLElement} currentElement - The element to get the position for.
|
|
653
|
+
* @returns {number} The position of the element in its parent.
|
|
654
|
+
*/
|
|
655
|
+
function getElementPosition(currentElement) {
|
|
656
|
+
if (!currentElement.parentElement) {
|
|
657
|
+
return 0; // No parent means no siblings
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
const tagName = currentElement.nodeName.toLowerCase();
|
|
661
|
+
|
|
662
|
+
const siblings = Array.from(currentElement.parentElement.children)
|
|
663
|
+
.filter((sib) => sib.nodeName.toLowerCase() === tagName);
|
|
664
|
+
|
|
665
|
+
if (siblings.length === 1) {
|
|
666
|
+
return 0; // Only element of its type
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
const index = siblings.indexOf(currentElement) + 1; // 1-based index
|
|
670
|
+
return index;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
function getXPathTree(element, stopAtBoundary = true) {
|
|
675
|
+
if (xpathCache.has(element)) return xpathCache.get(element);
|
|
676
|
+
|
|
677
|
+
const segments = [];
|
|
678
|
+
let currentElement = element;
|
|
679
|
+
|
|
680
|
+
while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) {
|
|
681
|
+
// Stop if we hit a shadow root or iframe
|
|
682
|
+
if (
|
|
683
|
+
stopAtBoundary &&
|
|
684
|
+
(currentElement.parentNode instanceof ShadowRoot ||
|
|
685
|
+
currentElement.parentNode instanceof HTMLIFrameElement)
|
|
686
|
+
) {
|
|
687
|
+
break;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
const position = getElementPosition(currentElement);
|
|
691
|
+
const tagName = currentElement.nodeName.toLowerCase();
|
|
692
|
+
const xpathIndex = position > 0 ? \`[\${position}]\` : "";
|
|
693
|
+
segments.unshift(\`\${tagName}\${xpathIndex}\`);
|
|
694
|
+
|
|
695
|
+
currentElement = currentElement.parentNode;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
const result = segments.join("/");
|
|
699
|
+
xpathCache.set(element, result);
|
|
700
|
+
return result;
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Checks if a text node is visible.
|
|
705
|
+
*
|
|
706
|
+
* @param {Text} textNode - The text node to check.
|
|
707
|
+
* @returns {boolean} Whether the text node is visible.
|
|
708
|
+
*/
|
|
709
|
+
function isTextNodeVisible(textNode) {
|
|
710
|
+
try {
|
|
711
|
+
// Special case: when viewportExpansion is -1, consider all text nodes as visible
|
|
712
|
+
if (viewportExpansion === -1) {
|
|
713
|
+
// Still check parent visibility for basic filtering
|
|
714
|
+
const parentElement = textNode.parentElement;
|
|
715
|
+
if (!parentElement) return false;
|
|
716
|
+
|
|
717
|
+
try {
|
|
718
|
+
return parentElement.checkVisibility({
|
|
719
|
+
checkOpacity: true,
|
|
720
|
+
checkVisibilityCSS: true,
|
|
721
|
+
});
|
|
722
|
+
} catch (e) {
|
|
723
|
+
// Fallback if checkVisibility is not supported
|
|
724
|
+
const style = window.getComputedStyle(parentElement);
|
|
725
|
+
return style.display !== 'none' &&
|
|
726
|
+
style.visibility !== 'hidden' &&
|
|
727
|
+
style.opacity !== '0';
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
const range = document.createRange();
|
|
732
|
+
range.selectNodeContents(textNode);
|
|
733
|
+
const rects = range.getClientRects(); // Use getClientRects for Range
|
|
734
|
+
|
|
735
|
+
if (!rects || rects.length === 0) {
|
|
736
|
+
return false;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
let isAnyRectVisible = false;
|
|
740
|
+
let isAnyRectInViewport = false;
|
|
741
|
+
|
|
742
|
+
for (const rect of rects) {
|
|
743
|
+
// Check size
|
|
744
|
+
if (rect.width > 0 && rect.height > 0) {
|
|
745
|
+
isAnyRectVisible = true;
|
|
746
|
+
|
|
747
|
+
// Viewport check for this rect
|
|
748
|
+
if (!(
|
|
749
|
+
rect.bottom < -viewportExpansion ||
|
|
750
|
+
rect.top > window.innerHeight + viewportExpansion ||
|
|
751
|
+
rect.right < -viewportExpansion ||
|
|
752
|
+
rect.left > window.innerWidth + viewportExpansion
|
|
753
|
+
)) {
|
|
754
|
+
isAnyRectInViewport = true;
|
|
755
|
+
break; // Found a visible rect in viewport, no need to check others
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
if (!isAnyRectVisible || !isAnyRectInViewport) {
|
|
761
|
+
return false;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// Check parent visibility
|
|
765
|
+
const parentElement = textNode.parentElement;
|
|
766
|
+
if (!parentElement) return false;
|
|
767
|
+
|
|
768
|
+
try {
|
|
769
|
+
return parentElement.checkVisibility({
|
|
770
|
+
checkOpacity: true,
|
|
771
|
+
checkVisibilityCSS: true,
|
|
772
|
+
});
|
|
773
|
+
} catch (e) {
|
|
774
|
+
// Fallback if checkVisibility is not supported
|
|
775
|
+
const style = window.getComputedStyle(parentElement);
|
|
776
|
+
return style.display !== 'none' &&
|
|
777
|
+
style.visibility !== 'hidden' &&
|
|
778
|
+
style.opacity !== '0';
|
|
779
|
+
}
|
|
780
|
+
} catch (e) {
|
|
781
|
+
console.warn('Error checking text node visibility:', e);
|
|
782
|
+
return false;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Checks if an element is accepted.
|
|
788
|
+
*
|
|
789
|
+
* @param {HTMLElement} element - The element to check.
|
|
790
|
+
* @returns {boolean} Whether the element is accepted.
|
|
791
|
+
*/
|
|
792
|
+
function isElementAccepted(element) {
|
|
793
|
+
if (!element || !element.tagName) return false;
|
|
794
|
+
|
|
795
|
+
// Always accept body and common container elements
|
|
796
|
+
const alwaysAccept = new Set([
|
|
797
|
+
"body", "div", "main", "article", "section", "nav", "header", "footer"
|
|
798
|
+
]);
|
|
799
|
+
const tagName = element.tagName.toLowerCase();
|
|
800
|
+
|
|
801
|
+
if (alwaysAccept.has(tagName)) return true;
|
|
802
|
+
|
|
803
|
+
const leafElementDenyList = new Set([
|
|
804
|
+
"svg",
|
|
805
|
+
"script",
|
|
806
|
+
"style",
|
|
807
|
+
"link",
|
|
808
|
+
"meta",
|
|
809
|
+
"noscript",
|
|
810
|
+
"template",
|
|
811
|
+
]);
|
|
812
|
+
|
|
813
|
+
return !leafElementDenyList.has(tagName);
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/**
|
|
817
|
+
* Checks if an element is visible.
|
|
818
|
+
*
|
|
819
|
+
* @param {HTMLElement} element - The element to check.
|
|
820
|
+
* @returns {boolean} Whether the element is visible.
|
|
821
|
+
*/
|
|
822
|
+
function isElementVisible(element) {
|
|
823
|
+
if (element.tagName.toLowerCase() === "input" && element.type === "date") {
|
|
824
|
+
return true;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
if (alwaysHighlightFileInput && element.tagName.toLowerCase() === "input" && element.type === "file") return true;
|
|
828
|
+
|
|
829
|
+
// SVG elements need special handling for visibility
|
|
830
|
+
if (element.tagName.toLowerCase() === "svg") {
|
|
831
|
+
const rect = getCachedBoundingRect(element);
|
|
832
|
+
const style = getCachedComputedStyle(element);
|
|
833
|
+
return (
|
|
834
|
+
rect &&
|
|
835
|
+
rect.width > 0 &&
|
|
836
|
+
rect.height > 0 &&
|
|
837
|
+
style?.visibility !== "hidden" &&
|
|
838
|
+
style?.display !== "none"
|
|
839
|
+
);
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
const style = getCachedComputedStyle(element);
|
|
843
|
+
return (
|
|
844
|
+
element.offsetWidth > 0 &&
|
|
845
|
+
element.offsetHeight > 0 &&
|
|
846
|
+
style?.visibility !== "hidden" &&
|
|
847
|
+
style?.display !== "none"
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* Checks if an element is clickable (responds to click events).
|
|
853
|
+
*
|
|
854
|
+
* @param {HTMLElement} element - The element to check.
|
|
855
|
+
* @returns {boolean} Whether the element is clickable.
|
|
856
|
+
*/
|
|
857
|
+
function shouldMarkAsClickable(element) {
|
|
858
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) {
|
|
859
|
+
return false;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
const tagName = element.tagName.toLowerCase();
|
|
863
|
+
|
|
864
|
+
// Primarily clickable elements
|
|
865
|
+
const primaryClickableElements = new Set([
|
|
866
|
+
"a", // Links
|
|
867
|
+
"button", // Buttons
|
|
868
|
+
"details", // Expandable details
|
|
869
|
+
"summary", // Summary element (clickable part of details)
|
|
870
|
+
"label", // Form labels (often clickable)
|
|
871
|
+
"option", // Select options
|
|
872
|
+
"optgroup", // Option groups
|
|
873
|
+
]);
|
|
874
|
+
|
|
875
|
+
if (primaryClickableElements.has(tagName)) {
|
|
876
|
+
return false;
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
const role = element.getAttribute("role");
|
|
880
|
+
|
|
881
|
+
// Clickable roles
|
|
882
|
+
const clickableRoles = new Set([
|
|
883
|
+
'button', // Directly clickable element
|
|
884
|
+
'link', // Clickable link
|
|
885
|
+
'menuitem', // Clickable menu item
|
|
886
|
+
'menuitemradio', // Radio-style menu item (selectable)
|
|
887
|
+
'menuitemcheckbox', // Checkbox-style menu item (toggleable)
|
|
888
|
+
'radio', // Radio button (selectable)
|
|
889
|
+
'checkbox', // Checkbox (toggleable)
|
|
890
|
+
'tab', // Tab (clickable to switch content)
|
|
891
|
+
'switch', // Toggle switch (clickable to change state)
|
|
892
|
+
'option', // Selectable option in a list
|
|
893
|
+
]);
|
|
894
|
+
|
|
895
|
+
if (role && clickableRoles.has(role)) {
|
|
896
|
+
return true;
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
// Check for dropdown indicators
|
|
900
|
+
if (hasAnyClassName(element, buttonClassNames)) {
|
|
901
|
+
return true; // Return true for dropdown elements
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
if (
|
|
905
|
+
element.getAttribute('data-toggle') === 'dropdown' ||
|
|
906
|
+
element.getAttribute('aria-haspopup')
|
|
907
|
+
) {
|
|
908
|
+
return true;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
const clickEvents = ['click', 'mousedown', 'mouseup', 'dblclick'];
|
|
912
|
+
const listenedEvents = getCachedNodeEventListeners(element);
|
|
913
|
+
if (listenedEvents && listenedEvents.length > 0) {
|
|
914
|
+
for (const eventType of clickEvents) {
|
|
915
|
+
if (listenedEvents.includes(eventType)) {
|
|
916
|
+
return true;
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
return false;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
/**
|
|
925
|
+
* Checks if an element is interactive.
|
|
926
|
+
*
|
|
927
|
+
* lots of comments, and uncommented code - to show the logic of what we already tried
|
|
928
|
+
*
|
|
929
|
+
* One of the things we tried at the beginning was also to use event listeners, and other fancy class, style stuff -> what actually worked best was just combining most things with computed cursor style :)
|
|
930
|
+
*
|
|
931
|
+
* @param {HTMLElement} element - The element to check.
|
|
932
|
+
*/
|
|
933
|
+
function isInteractiveElement(element) {
|
|
934
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) {
|
|
935
|
+
return false;
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// Cache the tagName and style lookups
|
|
939
|
+
const tagName = element.tagName.toLowerCase();
|
|
940
|
+
const style = getCachedComputedStyle(element);
|
|
941
|
+
|
|
942
|
+
// Define interactive cursors
|
|
943
|
+
const interactiveCursors = new Set([
|
|
944
|
+
'pointer', // Link/clickable elements
|
|
945
|
+
'move', // Movable elements
|
|
946
|
+
'text', // Text selection
|
|
947
|
+
'grab', // Grabbable elements
|
|
948
|
+
'grabbing', // Currently grabbing
|
|
949
|
+
'cell', // Table cell selection
|
|
950
|
+
'copy', // Copy operation
|
|
951
|
+
'alias', // Alias creation
|
|
952
|
+
'all-scroll', // Scrollable content
|
|
953
|
+
'col-resize', // Column resize
|
|
954
|
+
'context-menu', // Context menu available
|
|
955
|
+
'crosshair', // Precise selection
|
|
956
|
+
'e-resize', // East resize
|
|
957
|
+
'ew-resize', // East-west resize
|
|
958
|
+
'help', // Help available
|
|
959
|
+
'n-resize', // North resize
|
|
960
|
+
'ne-resize', // Northeast resize
|
|
961
|
+
'nesw-resize', // Northeast-southwest resize
|
|
962
|
+
'ns-resize', // North-south resize
|
|
963
|
+
'nw-resize', // Northwest resize
|
|
964
|
+
'nwse-resize', // Northwest-southeast resize
|
|
965
|
+
'row-resize', // Row resize
|
|
966
|
+
's-resize', // South resize
|
|
967
|
+
'se-resize', // Southeast resize
|
|
968
|
+
'sw-resize', // Southwest resize
|
|
969
|
+
'vertical-text', // Vertical text selection
|
|
970
|
+
'w-resize', // West resize
|
|
971
|
+
'zoom-in', // Zoom in
|
|
972
|
+
'zoom-out' // Zoom out
|
|
973
|
+
]);
|
|
974
|
+
|
|
975
|
+
// Define non-interactive cursors
|
|
976
|
+
const nonInteractiveCursors = new Set([
|
|
977
|
+
'not-allowed', // Action not allowed
|
|
978
|
+
'no-drop', // Drop not allowed
|
|
979
|
+
'wait', // Processing
|
|
980
|
+
'progress', // In progress
|
|
981
|
+
'initial', // Initial value
|
|
982
|
+
'inherit' // Inherited value
|
|
983
|
+
//? Let's just include all potentially clickable elements that are not specifically blocked
|
|
984
|
+
// 'none', // No cursor
|
|
985
|
+
// 'default', // Default cursor
|
|
986
|
+
// 'auto', // Browser default
|
|
987
|
+
]);
|
|
988
|
+
|
|
989
|
+
/**
|
|
990
|
+
* Checks if an element has an interactive pointer.
|
|
991
|
+
*
|
|
992
|
+
* @param {HTMLElement} element - The element to check.
|
|
993
|
+
* @returns {boolean} Whether the element has an interactive pointer.
|
|
994
|
+
*/
|
|
995
|
+
function doesElementHaveInteractivePointer(element) {
|
|
996
|
+
if (element.tagName.toLowerCase() === "html") return false;
|
|
997
|
+
|
|
998
|
+
if (style?.cursor && interactiveCursors.has(style.cursor)) return true;
|
|
999
|
+
|
|
1000
|
+
return false;
|
|
1001
|
+
}
|
|
1002
|
+
// Disabled for now, since it adds too many false positives
|
|
1003
|
+
// let isInteractiveCursor = doesElementHaveInteractivePointer(element);
|
|
1004
|
+
|
|
1005
|
+
// // Genius fix for almost all interactive elements
|
|
1006
|
+
// if (isInteractiveCursor) {
|
|
1007
|
+
// return true;
|
|
1008
|
+
// }
|
|
1009
|
+
|
|
1010
|
+
const interactiveElements = new Set([
|
|
1011
|
+
"a", // Links
|
|
1012
|
+
"button", // Buttons
|
|
1013
|
+
"input", // All input types (text, checkbox, radio, etc.)
|
|
1014
|
+
"select", // Dropdown menus
|
|
1015
|
+
"textarea", // Text areas
|
|
1016
|
+
"details", // Expandable details
|
|
1017
|
+
"summary", // Summary element (clickable part of details)
|
|
1018
|
+
"label", // Form labels (often clickable)
|
|
1019
|
+
"option", // Select options
|
|
1020
|
+
"optgroup", // Option groups
|
|
1021
|
+
"fieldset", // Form fieldsets (can be interactive with legend)
|
|
1022
|
+
"legend", // Fieldset legends
|
|
1023
|
+
]);
|
|
1024
|
+
|
|
1025
|
+
// Define explicit disable attributes and properties
|
|
1026
|
+
const explicitDisableTags = new Set([
|
|
1027
|
+
'disabled', // Standard disabled attribute
|
|
1028
|
+
// 'aria-disabled', // ARIA disabled state
|
|
1029
|
+
// 'readonly', // Read-only state
|
|
1030
|
+
// 'aria-readonly', // ARIA read-only state
|
|
1031
|
+
// 'aria-hidden', // Hidden from accessibility
|
|
1032
|
+
// 'hidden', // Hidden attribute
|
|
1033
|
+
// 'inert', // Inert attribute
|
|
1034
|
+
// 'aria-inert', // ARIA inert state
|
|
1035
|
+
// 'tabindex="-1"', // Removed from tab order
|
|
1036
|
+
// 'aria-hidden="true"' // Hidden from screen readers
|
|
1037
|
+
]);
|
|
1038
|
+
|
|
1039
|
+
// Check for non-interactive cursor
|
|
1040
|
+
if (style?.cursor && nonInteractiveCursors.has(style.cursor)) {
|
|
1041
|
+
return false;
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
// handle inputs, select, checkbox, radio, textarea, button and make sure they are not cursor style disabled/not-allowed
|
|
1045
|
+
if (interactiveElements.has(tagName)) {
|
|
1046
|
+
// Check for explicit disable attributes
|
|
1047
|
+
for (const disableTag of explicitDisableTags) {
|
|
1048
|
+
if (element.hasAttribute(disableTag) ||
|
|
1049
|
+
element.getAttribute(disableTag) === 'true' ||
|
|
1050
|
+
element.getAttribute(disableTag) === '') {
|
|
1051
|
+
return false;
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
// Check for disabled property on form elements
|
|
1056
|
+
if (element.disabled) {
|
|
1057
|
+
return false;
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
// Don't mark as non-interactive yet
|
|
1061
|
+
// Check for readonly property on form elements
|
|
1062
|
+
if (element.readOnly) {
|
|
1063
|
+
// return false;
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
// Check for inert property
|
|
1067
|
+
if (element.inert) {
|
|
1068
|
+
return false;
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
return true;
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
const role = element.getAttribute("role");
|
|
1075
|
+
const ariaRole = element.getAttribute("aria-role");
|
|
1076
|
+
|
|
1077
|
+
// Check for contenteditable attribute
|
|
1078
|
+
if (element.getAttribute("contenteditable") === "true" || element.isContentEditable) {
|
|
1079
|
+
return true;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
// Added enhancement to capture dropdown interactive elements
|
|
1083
|
+
if (hasAnyClassName(element, buttonClassNames) ||
|
|
1084
|
+
hasAnyClassName(element, interactiveClassNames) ||
|
|
1085
|
+
element.getAttribute('data-index') ||
|
|
1086
|
+
element.getAttribute('data-toggle') === 'dropdown' ||
|
|
1087
|
+
element.getAttribute('aria-haspopup')) {
|
|
1088
|
+
return true;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
const interactiveRoles = new Set([
|
|
1093
|
+
'button', // Directly clickable element
|
|
1094
|
+
'link', // Clickable link
|
|
1095
|
+
'menuitem', // Clickable menu item
|
|
1096
|
+
'menuitemradio', // Radio-style menu item (selectable)
|
|
1097
|
+
'menuitemcheckbox', // Checkbox-style menu item (toggleable)
|
|
1098
|
+
'radio', // Radio button (selectable)
|
|
1099
|
+
'checkbox', // Checkbox (toggleable)
|
|
1100
|
+
'tab', // Tab (clickable to switch content)
|
|
1101
|
+
'switch', // Toggle switch (clickable to change state)
|
|
1102
|
+
'slider', // Slider control (draggable)
|
|
1103
|
+
'spinbutton', // Number input with up/down controls
|
|
1104
|
+
'combobox', // Dropdown with text input
|
|
1105
|
+
'searchbox', // Search input field
|
|
1106
|
+
'textbox', // Text input field
|
|
1107
|
+
'listbox', // Selectable list
|
|
1108
|
+
'option', // Selectable option in a list
|
|
1109
|
+
'scrollbar' // Scrollable control
|
|
1110
|
+
]);
|
|
1111
|
+
|
|
1112
|
+
|
|
1113
|
+
// Basic role/attribute checks
|
|
1114
|
+
const hasInteractiveRole =
|
|
1115
|
+
(role && interactiveRoles.has(role)) ||
|
|
1116
|
+
(ariaRole && interactiveRoles.has(ariaRole));
|
|
1117
|
+
|
|
1118
|
+
if (hasInteractiveRole) return true;
|
|
1119
|
+
|
|
1120
|
+
const listenedEvents = getCachedNodeEventListeners(element);
|
|
1121
|
+
if (listenedEvents && listenedEvents.length > 0) {
|
|
1122
|
+
for (const eventType of INTERACTION_EVENTS) {
|
|
1123
|
+
if (listenedEvents.includes(eventType)) {
|
|
1124
|
+
return true;
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
return false
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
/**
|
|
1134
|
+
* Checks if an element is the topmost element at its position.
|
|
1135
|
+
*
|
|
1136
|
+
* @param {HTMLElement} element - The element to check.
|
|
1137
|
+
* @returns {boolean} Whether the element is the topmost element at its position.
|
|
1138
|
+
*/
|
|
1139
|
+
function isTopElement(element) {
|
|
1140
|
+
if (element.tagName.toLowerCase() === "input" && element.type === "date") {
|
|
1141
|
+
return true;
|
|
1142
|
+
}
|
|
1143
|
+
// Special case: when viewportExpansion is -1, consider all elements as "top" elements
|
|
1144
|
+
if (viewportExpansion === -1) {
|
|
1145
|
+
return true;
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
const rects = getCachedClientRects(element); // Replace element.getClientRects()
|
|
1149
|
+
|
|
1150
|
+
if (!rects || rects.length === 0) {
|
|
1151
|
+
return false; // No geometry, cannot be top
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
let isAnyRectInViewport = false;
|
|
1155
|
+
for (const rect of rects) {
|
|
1156
|
+
// Use the same logic as isInExpandedViewport check
|
|
1157
|
+
if (rect.width > 0 && rect.height > 0 && !( // Only check non-empty rects
|
|
1158
|
+
rect.bottom < -viewportExpansion ||
|
|
1159
|
+
rect.top > window.innerHeight + viewportExpansion ||
|
|
1160
|
+
rect.right < -viewportExpansion ||
|
|
1161
|
+
rect.left > window.innerWidth + viewportExpansion
|
|
1162
|
+
)) {
|
|
1163
|
+
isAnyRectInViewport = true;
|
|
1164
|
+
break;
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
if (!isAnyRectInViewport) {
|
|
1169
|
+
return false; // All rects are outside the viewport area
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
// Find the correct document context and root element
|
|
1174
|
+
let doc = element.ownerDocument;
|
|
1175
|
+
|
|
1176
|
+
// If we're in an iframe, elements are considered top by default
|
|
1177
|
+
if (doc !== window.document) {
|
|
1178
|
+
return true;
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
// For shadow DOM, we need to check within its own root context
|
|
1182
|
+
const shadowRoot = element.getRootNode();
|
|
1183
|
+
if (shadowRoot instanceof ShadowRoot) {
|
|
1184
|
+
const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
|
|
1185
|
+
const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
|
|
1186
|
+
|
|
1187
|
+
try {
|
|
1188
|
+
const topEl = shadowRoot.elementFromPoint(centerX, centerY);
|
|
1189
|
+
if (!topEl) return false;
|
|
1190
|
+
|
|
1191
|
+
let current = topEl;
|
|
1192
|
+
while (current && current !== shadowRoot) {
|
|
1193
|
+
if (current === element) return true;
|
|
1194
|
+
current = current.parentElement;
|
|
1195
|
+
}
|
|
1196
|
+
return false;
|
|
1197
|
+
} catch (e) {
|
|
1198
|
+
return true;
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
// For elements in viewport, check if they're topmost
|
|
1203
|
+
const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
|
|
1204
|
+
const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
|
|
1205
|
+
|
|
1206
|
+
try {
|
|
1207
|
+
const topEl = document.elementFromPoint(centerX, centerY);
|
|
1208
|
+
if (!topEl) return false;
|
|
1209
|
+
|
|
1210
|
+
let current = topEl;
|
|
1211
|
+
while (current && current !== document.documentElement) {
|
|
1212
|
+
if (current === element) return true;
|
|
1213
|
+
current = current.parentElement;
|
|
1214
|
+
}
|
|
1215
|
+
return false;
|
|
1216
|
+
} catch (e) {
|
|
1217
|
+
return true;
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
/**
|
|
1222
|
+
* Checks if an element is within the expanded viewport.
|
|
1223
|
+
*
|
|
1224
|
+
* @param {HTMLElement} element - The element to check.
|
|
1225
|
+
* @param {number} viewportExpansion - The viewport expansion.
|
|
1226
|
+
* @returns {boolean} Whether the element is within the expanded viewport.
|
|
1227
|
+
*/
|
|
1228
|
+
function isInExpandedViewport(element, viewportExpansion) {
|
|
1229
|
+
if (viewportExpansion === -1) {
|
|
1230
|
+
return true;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
const rects = element.getClientRects(); // Use getClientRects
|
|
1234
|
+
|
|
1235
|
+
if (!rects || rects.length === 0) {
|
|
1236
|
+
// Fallback to getBoundingClientRect if getClientRects is empty,
|
|
1237
|
+
// useful for elements like <svg> that might not have client rects but have a bounding box.
|
|
1238
|
+
const boundingRect = getCachedBoundingRect(element);
|
|
1239
|
+
if (!boundingRect || boundingRect.width === 0 || boundingRect.height === 0) {
|
|
1240
|
+
return false;
|
|
1241
|
+
}
|
|
1242
|
+
return !(
|
|
1243
|
+
boundingRect.bottom < -viewportExpansion ||
|
|
1244
|
+
boundingRect.top > window.innerHeight + viewportExpansion ||
|
|
1245
|
+
boundingRect.right < -viewportExpansion ||
|
|
1246
|
+
boundingRect.left > window.innerWidth + viewportExpansion
|
|
1247
|
+
);
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
// Check if *any* client rect is within the viewport
|
|
1251
|
+
for (const rect of rects) {
|
|
1252
|
+
if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects
|
|
1253
|
+
|
|
1254
|
+
if (!(
|
|
1255
|
+
rect.bottom < -viewportExpansion ||
|
|
1256
|
+
rect.top > window.innerHeight + viewportExpansion ||
|
|
1257
|
+
rect.right < -viewportExpansion ||
|
|
1258
|
+
rect.left > window.innerWidth + viewportExpansion
|
|
1259
|
+
)) {
|
|
1260
|
+
return true; // Found at least one rect in the viewport
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
return false; // No rects were found in the viewport
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
// /**
|
|
1268
|
+
// * Gets the effective scroll of an element.
|
|
1269
|
+
// *
|
|
1270
|
+
// * @param {HTMLElement} element - The element to get the effective scroll for.
|
|
1271
|
+
// * @returns {Object} The effective scroll of the element.
|
|
1272
|
+
// */
|
|
1273
|
+
// function getEffectiveScroll(element) {
|
|
1274
|
+
// let currentEl = element;
|
|
1275
|
+
// let scrollX = 0;
|
|
1276
|
+
// let scrollY = 0;
|
|
1277
|
+
|
|
1278
|
+
// while (currentEl && currentEl !== document.documentElement) {
|
|
1279
|
+
// if (currentEl.scrollLeft || currentEl.scrollTop) {
|
|
1280
|
+
// scrollX += currentEl.scrollLeft;
|
|
1281
|
+
// scrollY += currentEl.scrollTop;
|
|
1282
|
+
// }
|
|
1283
|
+
// currentEl = currentEl.parentElement;
|
|
1284
|
+
// }
|
|
1285
|
+
|
|
1286
|
+
// scrollX += window.scrollX;
|
|
1287
|
+
// scrollY += window.scrollY;
|
|
1288
|
+
|
|
1289
|
+
// return { scrollX, scrollY };
|
|
1290
|
+
// }
|
|
1291
|
+
|
|
1292
|
+
/**
|
|
1293
|
+
* Checks if an element is an interactive candidate.
|
|
1294
|
+
*
|
|
1295
|
+
* @param {HTMLElement} element - The element to check.
|
|
1296
|
+
* @returns {boolean} Whether the element is an interactive candidate.
|
|
1297
|
+
*/
|
|
1298
|
+
function isInteractiveCandidate(element) {
|
|
1299
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
|
|
1300
|
+
|
|
1301
|
+
const tagName = element.tagName.toLowerCase();
|
|
1302
|
+
|
|
1303
|
+
// Fast-path for common interactive elements
|
|
1304
|
+
const interactiveElements = new Set([
|
|
1305
|
+
"a", "button", "input", "select", "textarea", "details", "summary", "label"
|
|
1306
|
+
]);
|
|
1307
|
+
|
|
1308
|
+
if (interactiveElements.has(tagName)) return true;
|
|
1309
|
+
|
|
1310
|
+
// Quick attribute checks without getting full lists
|
|
1311
|
+
const hasQuickInteractiveAttr = element.hasAttribute("onclick") ||
|
|
1312
|
+
element.hasAttribute("role") ||
|
|
1313
|
+
element.hasAttribute("tabindex") ||
|
|
1314
|
+
element.hasAttribute("aria-") ||
|
|
1315
|
+
element.hasAttribute("data-action") ||
|
|
1316
|
+
element.getAttribute("contenteditable") === "true";
|
|
1317
|
+
|
|
1318
|
+
return hasQuickInteractiveAttr;
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
// --- Define constants for distinct interaction check ---
|
|
1322
|
+
const DISTINCT_INTERACTIVE_TAGS = new Set([
|
|
1323
|
+
'a', 'button', 'input', 'select', 'textarea', 'summary', 'details', 'label', 'option'
|
|
1324
|
+
]);
|
|
1325
|
+
const INTERACTIVE_ROLES = new Set([
|
|
1326
|
+
'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox',
|
|
1327
|
+
'radio', 'checkbox', 'tab', 'switch', 'slider', 'spinbutton',
|
|
1328
|
+
'combobox', 'searchbox', 'textbox', 'listbox', 'option', 'scrollbar'
|
|
1329
|
+
]);
|
|
1330
|
+
|
|
1331
|
+
|
|
1332
|
+
/**
|
|
1333
|
+
* Heuristically determines if an element should be considered as independently interactive,
|
|
1334
|
+
* even if it's nested inside another interactive container.
|
|
1335
|
+
*
|
|
1336
|
+
* This function helps detect deeply nested actionable elements (e.g., menu items within a button)
|
|
1337
|
+
* that may not be picked up by strict interactivity checks.
|
|
1338
|
+
*
|
|
1339
|
+
* @param {HTMLElement} element - The element to check.
|
|
1340
|
+
* @returns {boolean} Whether the element is heuristically interactive.
|
|
1341
|
+
*/
|
|
1342
|
+
function isHeuristicallyInteractive(element) {
|
|
1343
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
|
|
1344
|
+
|
|
1345
|
+
// Skip non-visible elements early for performance
|
|
1346
|
+
if (!isElementVisible(element)) return false;
|
|
1347
|
+
|
|
1348
|
+
// Check for common attributes that often indicate interactivity
|
|
1349
|
+
const hasInteractiveAttributes =
|
|
1350
|
+
element.hasAttribute('role') ||
|
|
1351
|
+
element.hasAttribute('tabindex') ||
|
|
1352
|
+
element.hasAttribute('onclick') ||
|
|
1353
|
+
typeof element.onclick === 'function';
|
|
1354
|
+
|
|
1355
|
+
// Check for semantic class names suggesting interactivity
|
|
1356
|
+
const hasInteractiveClass = heuristicClassPattern.test(element.className || '');
|
|
1357
|
+
|
|
1358
|
+
// Determine whether the element is inside a known interactive container
|
|
1359
|
+
const isInKnownContainer = Boolean(
|
|
1360
|
+
element.closest(containerSelectors)
|
|
1361
|
+
);
|
|
1362
|
+
|
|
1363
|
+
// Ensure the element has at least one visible child (to avoid marking empty wrappers)
|
|
1364
|
+
const hasVisibleChildren = [...element.children].some(isElementVisible);
|
|
1365
|
+
|
|
1366
|
+
// Avoid highlighting elements whose parent is <body> (top-level wrappers)
|
|
1367
|
+
const isParentBody = element.parentElement && element.parentElement.isSameNode(document.body);
|
|
1368
|
+
|
|
1369
|
+
return (
|
|
1370
|
+
(isInteractiveElement(element) || hasInteractiveAttributes || hasInteractiveClass) &&
|
|
1371
|
+
hasVisibleChildren &&
|
|
1372
|
+
isInKnownContainer &&
|
|
1373
|
+
!isParentBody
|
|
1374
|
+
);
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
/**
|
|
1379
|
+
* Checks if an element likely represents a distinct interaction
|
|
1380
|
+
* separate from its parent (if the parent is also interactive).
|
|
1381
|
+
*
|
|
1382
|
+
* @param {HTMLElement} element - The element to check.
|
|
1383
|
+
* @param {Object} nodeData - The node data object.
|
|
1384
|
+
* @returns {boolean} Whether the element is a distinct interaction.
|
|
1385
|
+
*/
|
|
1386
|
+
function isElementDistinctInteraction(element, nodeData) {
|
|
1387
|
+
if (nodeData.isScrollable) {
|
|
1388
|
+
return true;
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) {
|
|
1392
|
+
return false;
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
const tagName = element.tagName.toLowerCase();
|
|
1396
|
+
const role = element.getAttribute('role');
|
|
1397
|
+
|
|
1398
|
+
// Check if it's an iframe - always distinct boundary
|
|
1399
|
+
if (tagName === 'iframe') {
|
|
1400
|
+
return true;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
// Check tag name
|
|
1404
|
+
if (DISTINCT_INTERACTIVE_TAGS.has(tagName)) {
|
|
1405
|
+
return true;
|
|
1406
|
+
}
|
|
1407
|
+
// Check interactive roles
|
|
1408
|
+
if (role && INTERACTIVE_ROLES.has(role)) {
|
|
1409
|
+
return true;
|
|
1410
|
+
}
|
|
1411
|
+
// Check contenteditable
|
|
1412
|
+
if (element.isContentEditable || element.getAttribute('contenteditable') === 'true') {
|
|
1413
|
+
return true;
|
|
1414
|
+
}
|
|
1415
|
+
// Check for common testing/automation attributes
|
|
1416
|
+
if (element.hasAttribute('data-testid') || element.hasAttribute('data-cy') || element.hasAttribute('data-test')) {
|
|
1417
|
+
return true;
|
|
1418
|
+
}
|
|
1419
|
+
// Check for explicit onclick handler (attribute or property)
|
|
1420
|
+
if (element.hasAttribute('onclick') || typeof element.onclick === 'function') {
|
|
1421
|
+
return true;
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
if (element.hasAttribute('aria-haspopup')) {
|
|
1425
|
+
return true;
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
if (hasAnyClassName(element, interactiveClassNames)) {
|
|
1429
|
+
return true;
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
// return false
|
|
1433
|
+
|
|
1434
|
+
// Check for other common interaction event listeners
|
|
1435
|
+
try {
|
|
1436
|
+
const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
|
|
1437
|
+
if (typeof getEventListenersForNode === 'function') {
|
|
1438
|
+
const listeners = getEventListenersForNode(element);
|
|
1439
|
+
const interactionEvents = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave', 'keydown', 'keyup', 'submit', 'change', 'focus', 'blur'];
|
|
1440
|
+
for (const eventType of interactionEvents) {
|
|
1441
|
+
for (const listener of listeners) {
|
|
1442
|
+
if (listener.type === eventType) {
|
|
1443
|
+
return true; // Found a common interaction listener
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
// Fallback: Check common event attributes if getEventListeners is not available (getEventListenersForNode doesn't work in page.evaluate context)
|
|
1449
|
+
const commonEventAttrs = ['onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', 'onsubmit', 'onmouseenter', 'onmouseleave', 'onchange', 'oninput', 'onfocus', 'onblur'];
|
|
1450
|
+
if (commonEventAttrs.some(attr => element.hasAttribute(attr))) {
|
|
1451
|
+
return true;
|
|
1452
|
+
}
|
|
1453
|
+
} catch (e) {
|
|
1454
|
+
// console.warn(\`Could not check event listeners for \${element.tagName}:\`, e);
|
|
1455
|
+
// If checking listeners fails, rely on other checks
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
|
|
1459
|
+
|
|
1460
|
+
// if the element is not strictly interactive but appears clickable based on heuristic signals
|
|
1461
|
+
if (isHeuristicallyInteractive(element)) {
|
|
1462
|
+
return true;
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
// Default to false: if it's interactive but doesn't match above,
|
|
1466
|
+
// assume it triggers the same action as the parent.
|
|
1467
|
+
return false;
|
|
1468
|
+
}
|
|
1469
|
+
// --- End distinct interaction check ---
|
|
1470
|
+
|
|
1471
|
+
/**
|
|
1472
|
+
* Handles the logic for deciding whether to highlight an element and performing the highlight.
|
|
1473
|
+
* @param {
|
|
1474
|
+
{
|
|
1475
|
+
tagName: string;
|
|
1476
|
+
attributes: Record<string, string>;
|
|
1477
|
+
xpath: any;
|
|
1478
|
+
children: never[];
|
|
1479
|
+
isVisible?: boolean;
|
|
1480
|
+
isTopElement?: boolean;
|
|
1481
|
+
isInteractive?: boolean;
|
|
1482
|
+
isInViewport?: boolean;
|
|
1483
|
+
highlightIndex?: number;
|
|
1484
|
+
shadowRoot?: boolean;
|
|
1485
|
+
}} nodeData - The node data object.
|
|
1486
|
+
* @param {HTMLElement} node - The node to highlight.
|
|
1487
|
+
* @param {HTMLElement | null} parentIframe - The parent iframe node.
|
|
1488
|
+
* @param {boolean} isParentHighlighted - Whether the parent node is highlighted.
|
|
1489
|
+
* @returns {boolean} Whether the element was highlighted.
|
|
1490
|
+
*/
|
|
1491
|
+
function handleHighlighting(nodeData, node, parentIframe, isParentHighlighted) {
|
|
1492
|
+
if (!nodeData.isInteractive) return false; // Not interactive, definitely don't highlight
|
|
1493
|
+
|
|
1494
|
+
let shouldHighlight = false;
|
|
1495
|
+
if (!isParentHighlighted) {
|
|
1496
|
+
// Parent wasn't highlighted, this interactive node can be highlighted.
|
|
1497
|
+
shouldHighlight = true;
|
|
1498
|
+
} else {
|
|
1499
|
+
// Parent *was* highlighted. Only highlight this node if it represents a distinct interaction.
|
|
1500
|
+
if (isElementDistinctInteraction(node, nodeData)) {
|
|
1501
|
+
shouldHighlight = true;
|
|
1502
|
+
} else {
|
|
1503
|
+
// console.log(\`Skipping highlight for \${nodeData.tagName} (parent highlighted)\`);
|
|
1504
|
+
shouldHighlight = false;
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
if (shouldHighlight) {
|
|
1509
|
+
const attributeNames = node.getAttributeNames?.() || [];
|
|
1510
|
+
for (const name of attributeNames) {
|
|
1511
|
+
const value = node.getAttribute(name);
|
|
1512
|
+
nodeData.attributes[name] = value;
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
// Check viewport status before assigning index and highlighting
|
|
1516
|
+
if (nodeData.isInViewport === undefined) {
|
|
1517
|
+
nodeData.isInViewport = isInExpandedViewport(node, viewportExpansion);
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
// When viewportExpansion is -1, all interactive elements should get a highlight index
|
|
1521
|
+
// regardless of viewport status
|
|
1522
|
+
if (nodeData.isInViewport || viewportExpansion === -1) {
|
|
1523
|
+
nodeData.highlightIndex = highlightIndex++;
|
|
1524
|
+
|
|
1525
|
+
if (doHighlightElements) {
|
|
1526
|
+
if (focusHighlightIndex >= 0) {
|
|
1527
|
+
if (focusHighlightIndex === nodeData.highlightIndex) {
|
|
1528
|
+
highlightElement(node, nodeData.highlightIndex, parentIframe);
|
|
1529
|
+
}
|
|
1530
|
+
} else {
|
|
1531
|
+
highlightElement(node, nodeData.highlightIndex, parentIframe);
|
|
1532
|
+
}
|
|
1533
|
+
return true; // Successfully highlighted
|
|
1534
|
+
}
|
|
1535
|
+
} else {
|
|
1536
|
+
// console.log(\`Skipping highlight for \${nodeData.tagName} (outside viewport)\`);
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
return false; // Did not highlight
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
function isElementScrollable(element) {
|
|
1544
|
+
const listenedEvents = getCachedNodeEventListeners(element);
|
|
1545
|
+
if (listenedEvents && listenedEvents.includes('scroll')) {
|
|
1546
|
+
const hasScrollableX = element.scrollWidth > element.clientWidth;
|
|
1547
|
+
const hasScrollableY = element.scrollHeight > element.clientHeight;
|
|
1548
|
+
return hasScrollableX || hasScrollableY;
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
const style = getCachedComputedStyle(element);
|
|
1552
|
+
const hasScrollableX = ['auto', 'scroll'].includes(style.overflowX) &&
|
|
1553
|
+
element.scrollWidth > element.clientWidth;
|
|
1554
|
+
const hasScrollableY = ['auto', 'scroll'].includes(style.overflowY) &&
|
|
1555
|
+
element.scrollHeight > element.clientHeight;
|
|
1556
|
+
return hasScrollableX || hasScrollableY;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
/**
|
|
1560
|
+
* Creates a node data object for a given node and its descendants.
|
|
1561
|
+
*
|
|
1562
|
+
* @param {HTMLElement} node - The node to process.
|
|
1563
|
+
* @param {HTMLElement | null} parentIframe - The parent iframe node.
|
|
1564
|
+
* @param {boolean} isParentHighlighted - Whether the parent node is highlighted.
|
|
1565
|
+
* @returns {string | null} The ID of the node data object, or null if the node is not processed.
|
|
1566
|
+
*/
|
|
1567
|
+
function buildDomTree(node, parentIframe = null, isParentHighlighted = false) {
|
|
1568
|
+
|
|
1569
|
+
// Fast rejection checks first
|
|
1570
|
+
if (!node || node.id === HIGHLIGHT_CONTAINER_ID) {
|
|
1571
|
+
return null;
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
if (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE) {
|
|
1575
|
+
return null;
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
// Special handling for root node (body)
|
|
1579
|
+
if (node === document.body) {
|
|
1580
|
+
const nodeData = {
|
|
1581
|
+
tagName: 'body',
|
|
1582
|
+
attributes: {},
|
|
1583
|
+
xpath: '/body',
|
|
1584
|
+
children: [],
|
|
1585
|
+
};
|
|
1586
|
+
|
|
1587
|
+
// Process children of body
|
|
1588
|
+
for (const child of node.childNodes) {
|
|
1589
|
+
const domElement = buildDomTree(child, parentIframe, false); // Body's children have no highlighted parent initially
|
|
1590
|
+
if (domElement) nodeData.children.push(domElement);
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
const id = \`\${ID.current++}\`;
|
|
1594
|
+
DOM_HASH_MAP[id] = nodeData;
|
|
1595
|
+
return id;
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
// Process text nodes
|
|
1599
|
+
if (node.nodeType === Node.TEXT_NODE) {
|
|
1600
|
+
const textContent = node.textContent?.trim();
|
|
1601
|
+
if (!textContent) {
|
|
1602
|
+
return null;
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
// Only check visibility for text nodes that might be visible
|
|
1606
|
+
const parentElement = node.parentElement;
|
|
1607
|
+
if (!parentElement || parentElement.tagName.toLowerCase() === 'script') {
|
|
1608
|
+
return null;
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
const id = \`\${ID.current++}\`;
|
|
1612
|
+
DOM_HASH_MAP[id] = {
|
|
1613
|
+
type: "TEXT_NODE",
|
|
1614
|
+
text: textContent,
|
|
1615
|
+
isVisible: isTextNodeVisible(node),
|
|
1616
|
+
};
|
|
1617
|
+
return id;
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
// Quick checks for element nodes
|
|
1621
|
+
if (node.nodeType === Node.ELEMENT_NODE && !isElementAccepted(node)) {
|
|
1622
|
+
return null;
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
/**
|
|
1626
|
+
* @type {
|
|
1627
|
+
{
|
|
1628
|
+
tagName: string;
|
|
1629
|
+
attributes: Record<string, string | null>;
|
|
1630
|
+
xpath: any;
|
|
1631
|
+
children: never[];
|
|
1632
|
+
isVisible?: boolean;
|
|
1633
|
+
isTopElement?: boolean;
|
|
1634
|
+
isInteractive?: boolean;
|
|
1635
|
+
isInViewport?: boolean;
|
|
1636
|
+
highlightIndex?: number;
|
|
1637
|
+
shadowRoot?: boolean;
|
|
1638
|
+
}
|
|
1639
|
+
} nodeData - The node data object.
|
|
1640
|
+
*/
|
|
1641
|
+
const nodeData = {
|
|
1642
|
+
tagName: node.tagName.toLowerCase(),
|
|
1643
|
+
attributes: {},
|
|
1644
|
+
xpath: getXPathTree(node, true),
|
|
1645
|
+
children: [],
|
|
1646
|
+
};
|
|
1647
|
+
|
|
1648
|
+
// Get attributes for interactive elements or potential text containers
|
|
1649
|
+
if (node.tagName.toLowerCase() === 'iframe' || node.tagName.toLowerCase() === 'body') {
|
|
1650
|
+
const attributeNames = node.getAttributeNames?.() || [];
|
|
1651
|
+
for (const name of attributeNames) {
|
|
1652
|
+
const value = node.getAttribute(name);
|
|
1653
|
+
nodeData.attributes[name] = value;
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
let nodeWasHighlighted = false;
|
|
1658
|
+
// Perform visibility, interactivity, and highlighting checks
|
|
1659
|
+
if (node.nodeType === Node.ELEMENT_NODE) {
|
|
1660
|
+
if (alwaysHighlightFileInput && node.tagName.toLowerCase() === 'input' && node.type === 'file') {
|
|
1661
|
+
nodeData.isTopElement = true;
|
|
1662
|
+
if (nodeData.isTopElement) {
|
|
1663
|
+
nodeData.isInteractive = true;
|
|
1664
|
+
nodeData.isInViewport = true; // File inputs should always be considered in viewport
|
|
1665
|
+
// Call the dedicated highlighting function
|
|
1666
|
+
nodeWasHighlighted = handleHighlighting(nodeData, node, parentIframe, isParentHighlighted);
|
|
1667
|
+
}
|
|
1668
|
+
} else {
|
|
1669
|
+
nodeData.isVisible = isElementVisible(node); // isElementVisible uses offsetWidth/Height, which is fine
|
|
1670
|
+
|
|
1671
|
+
if (nodeData.isVisible) {
|
|
1672
|
+
nodeData.isTopElement = isTopElement(node);
|
|
1673
|
+
if (nodeData.isTopElement) {
|
|
1674
|
+
let isScrollable = isElementScrollable(node);
|
|
1675
|
+
nodeData.isInteractive = isInteractiveElement(node) || isScrollable;
|
|
1676
|
+
nodeData.isScrollable = isScrollable;
|
|
1677
|
+
nodeData.markAsClickable = shouldMarkAsClickable(node);
|
|
1678
|
+
// Call the dedicated highlighting function
|
|
1679
|
+
nodeWasHighlighted = handleHighlighting(nodeData, node, parentIframe, isParentHighlighted);
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
// Process children, with special handling for iframes and rich text editors
|
|
1686
|
+
if (node.tagName) {
|
|
1687
|
+
const tagName = node.tagName.toLowerCase();
|
|
1688
|
+
|
|
1689
|
+
// Handle iframes
|
|
1690
|
+
if (tagName === "iframe") {
|
|
1691
|
+
try {
|
|
1692
|
+
const iframeDoc = node.contentDocument || node.contentWindow?.document;
|
|
1693
|
+
if (iframeDoc) {
|
|
1694
|
+
for (const child of iframeDoc.childNodes) {
|
|
1695
|
+
const domElement = buildDomTree(child, node, false);
|
|
1696
|
+
if (domElement) nodeData.children.push(domElement);
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
} catch (e) {
|
|
1700
|
+
console.warn("Unable to access iframe:", e);
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
// Handle rich text editors and contenteditable elements
|
|
1704
|
+
else if (
|
|
1705
|
+
node.isContentEditable ||
|
|
1706
|
+
node.getAttribute("contenteditable") === "true" ||
|
|
1707
|
+
node.id === "tinymce" ||
|
|
1708
|
+
node.classList.contains("mce-content-body") ||
|
|
1709
|
+
(tagName === "body" && node.getAttribute("data-id")?.startsWith("mce_"))
|
|
1710
|
+
) {
|
|
1711
|
+
// Process all child nodes to capture formatted text
|
|
1712
|
+
for (const child of node.childNodes) {
|
|
1713
|
+
const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted);
|
|
1714
|
+
if (domElement) nodeData.children.push(domElement);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
else {
|
|
1718
|
+
// Handle shadow DOM
|
|
1719
|
+
if (node.shadowRoot) {
|
|
1720
|
+
nodeData.shadowRoot = true;
|
|
1721
|
+
for (const child of node.shadowRoot.childNodes) {
|
|
1722
|
+
const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted);
|
|
1723
|
+
if (domElement) nodeData.children.push(domElement);
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
// Handle regular elements
|
|
1727
|
+
for (const child of node.childNodes) {
|
|
1728
|
+
// Pass the highlighted status of the *current* node to its children
|
|
1729
|
+
const passHighlightStatusToChild = nodeWasHighlighted || isParentHighlighted;
|
|
1730
|
+
const domElement = buildDomTree(child, parentIframe, passHighlightStatusToChild);
|
|
1731
|
+
if (domElement) nodeData.children.push(domElement);
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
const id = \`\${ID.current++}\`;
|
|
1737
|
+
DOM_HASH_MAP[id] = nodeData;
|
|
1738
|
+
return id;
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
const rootId = buildDomTree(document.body);
|
|
1742
|
+
|
|
1743
|
+
// Clear the cache before starting
|
|
1744
|
+
DOM_CACHE.clearCache();
|
|
1745
|
+
|
|
1746
|
+
return { rootId, map: DOM_HASH_MAP };
|
|
1747
|
+
}
|
|
1748
|
+
`,ee=`((args = {
|
|
1749
|
+
doHighlightElements: true,
|
|
1750
|
+
focusHighlightIndex: -1,
|
|
1751
|
+
viewportExpansion: 0,
|
|
1752
|
+
debugMode: false,
|
|
1753
|
+
interactiveClassNames: [],
|
|
1754
|
+
alwaysHighlightFileInput: false,
|
|
1755
|
+
sameRectIoUThreshold: 0.85,
|
|
1756
|
+
}) => {
|
|
1757
|
+
// Default threshold if not provided
|
|
1758
|
+
const sameRectIoUThreshold = args.sameRectIoUThreshold ?? 0.85;
|
|
1759
|
+
const EVENT_LISTENER_MAPPING = {
|
|
1760
|
+
'onclick': 'click',
|
|
1761
|
+
'onmousedown': 'mousedown',
|
|
1762
|
+
'onmouseup': 'mouseup',
|
|
1763
|
+
'ondblclick': 'dblclick',
|
|
1764
|
+
'onmouseenter': 'mouseenter',
|
|
1765
|
+
'onmouseleave': 'mouseleave',
|
|
1766
|
+
'onmousemove': 'mousemove',
|
|
1767
|
+
'onmouseout': 'mouseout',
|
|
1768
|
+
'onmouseover': 'mouseover',
|
|
1769
|
+
'onmousewheel': 'mousewheel',
|
|
1770
|
+
'onscroll': 'scroll',
|
|
1771
|
+
'onselect': 'select',
|
|
1772
|
+
'onchange': 'change',
|
|
1773
|
+
'onfocus': 'focus',
|
|
1774
|
+
'onblur': 'blur',
|
|
1775
|
+
'onkeydown': 'keydown',
|
|
1776
|
+
'onkeyup': 'keyup',
|
|
1777
|
+
'onkeypress': 'keypress',
|
|
1778
|
+
'oninput': 'input',
|
|
1779
|
+
};
|
|
1780
|
+
const INTERACTION_EVENTS = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave'];
|
|
1781
|
+
const { doHighlightElements, focusHighlightIndex, viewportExpansion, debugMode, interactiveClassNames, alwaysHighlightFileInput, grayscaleImage, uniformityTolerance = 32, captureDebugSnapshots = false, onSnapshot, onLog, phase, elementData: inputElementData, } = args;
|
|
1782
|
+
// Helper to stream logs if callback provided
|
|
1783
|
+
const streamLog = (msg) => {
|
|
1784
|
+
if (onLog)
|
|
1785
|
+
onLog(msg);
|
|
1786
|
+
};
|
|
1787
|
+
streamLog(\`[dom-tree] Starting phase=\${phase || 'legacy'}, grayscaleImage=\${!!grayscaleImage}, captureDebugSnapshots=\${captureDebugSnapshots}\`);
|
|
1788
|
+
const buttonClassNames = ['button', 'dropdown-toggle'];
|
|
1789
|
+
const cursorPointerClassNames = ['cursor-pointer', 'tw-cursor-pointer', 'clickable'];
|
|
1790
|
+
const heuristicClassPattern = /\\b(btn|const clickable|menu|item|entry|link)\\b/i;
|
|
1791
|
+
const containerSelectors = 'button,a,[role="button"],.menu,.dropdown,.list,.toolbar';
|
|
1792
|
+
let highlightIndex = 0; // Reset highlight index
|
|
1793
|
+
/**
|
|
1794
|
+
* Helper function to check if element has any of the specified class names.
|
|
1795
|
+
*/
|
|
1796
|
+
function hasAnyClassName(element, classNames) {
|
|
1797
|
+
if (!element.classList || !classNames || classNames.length === 0)
|
|
1798
|
+
return false;
|
|
1799
|
+
return classNames.some(className => element.classList.contains(className));
|
|
1800
|
+
}
|
|
1801
|
+
// Add caching mechanisms at the top level
|
|
1802
|
+
const DOM_CACHE = {
|
|
1803
|
+
boundingRects: new WeakMap(),
|
|
1804
|
+
clientRects: new WeakMap(),
|
|
1805
|
+
computedStyles: new WeakMap(),
|
|
1806
|
+
nodeEventListeners: new WeakMap(),
|
|
1807
|
+
clearCache: () => {
|
|
1808
|
+
DOM_CACHE.boundingRects = new WeakMap();
|
|
1809
|
+
DOM_CACHE.clientRects = new WeakMap();
|
|
1810
|
+
DOM_CACHE.computedStyles = new WeakMap();
|
|
1811
|
+
DOM_CACHE.nodeEventListeners = new WeakMap();
|
|
1812
|
+
}
|
|
1813
|
+
};
|
|
1814
|
+
/**
|
|
1815
|
+
* Gets the cached bounding rect for an element.
|
|
1816
|
+
*/
|
|
1817
|
+
function getCachedBoundingRect(element) {
|
|
1818
|
+
if (!element)
|
|
1819
|
+
return null;
|
|
1820
|
+
if (DOM_CACHE.boundingRects.has(element)) {
|
|
1821
|
+
return DOM_CACHE.boundingRects.get(element);
|
|
1822
|
+
}
|
|
1823
|
+
const rect = element.getBoundingClientRect();
|
|
1824
|
+
if (rect) {
|
|
1825
|
+
DOM_CACHE.boundingRects.set(element, rect);
|
|
1826
|
+
}
|
|
1827
|
+
return rect;
|
|
1828
|
+
}
|
|
1829
|
+
/**
|
|
1830
|
+
* Gets the cached computed style for an element.
|
|
1831
|
+
*/
|
|
1832
|
+
function getCachedComputedStyle(element) {
|
|
1833
|
+
if (!element)
|
|
1834
|
+
return null;
|
|
1835
|
+
if (DOM_CACHE.computedStyles.has(element)) {
|
|
1836
|
+
return DOM_CACHE.computedStyles.get(element);
|
|
1837
|
+
}
|
|
1838
|
+
const style = window.getComputedStyle(element);
|
|
1839
|
+
if (style) {
|
|
1840
|
+
DOM_CACHE.computedStyles.set(element, style);
|
|
1841
|
+
}
|
|
1842
|
+
return style;
|
|
1843
|
+
}
|
|
1844
|
+
/**
|
|
1845
|
+
* Gets the cached client rects for an element.
|
|
1846
|
+
*/
|
|
1847
|
+
function getCachedClientRects(element) {
|
|
1848
|
+
if (!element)
|
|
1849
|
+
return null;
|
|
1850
|
+
if (DOM_CACHE.clientRects.has(element)) {
|
|
1851
|
+
return DOM_CACHE.clientRects.get(element);
|
|
1852
|
+
}
|
|
1853
|
+
const rects = element.getClientRects();
|
|
1854
|
+
if (rects) {
|
|
1855
|
+
DOM_CACHE.clientRects.set(element, rects);
|
|
1856
|
+
}
|
|
1857
|
+
return rects;
|
|
1858
|
+
}
|
|
1859
|
+
/**
|
|
1860
|
+
* Gets the event listeners for a node.
|
|
1861
|
+
*/
|
|
1862
|
+
function getNodeEventListeners(element) {
|
|
1863
|
+
const set = new Set();
|
|
1864
|
+
try {
|
|
1865
|
+
if (typeof getEventListeners === 'function') {
|
|
1866
|
+
const listeners = getEventListeners(element);
|
|
1867
|
+
for (const eventType in listeners) {
|
|
1868
|
+
if (listeners[eventType] && listeners[eventType].length > 0) {
|
|
1869
|
+
set.add(eventType);
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
|
|
1874
|
+
if (typeof getEventListenersForNode === 'function') {
|
|
1875
|
+
const listeners = getEventListenersForNode(element);
|
|
1876
|
+
for (const listener of listeners) {
|
|
1877
|
+
if (listener.type) {
|
|
1878
|
+
set.add(listener.type);
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
for (const attr in EVENT_LISTENER_MAPPING) {
|
|
1883
|
+
if (element.hasAttribute(attr) || typeof element[attr] === 'function') {
|
|
1884
|
+
set.add(EVENT_LISTENER_MAPPING[attr]);
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
catch (error) {
|
|
1889
|
+
// Silently ignore errors
|
|
1890
|
+
}
|
|
1891
|
+
const listenedEvents = Array.from(set);
|
|
1892
|
+
return listenedEvents;
|
|
1893
|
+
}
|
|
1894
|
+
function getCachedNodeEventListeners(element) {
|
|
1895
|
+
if (!element)
|
|
1896
|
+
return null;
|
|
1897
|
+
if (DOM_CACHE.nodeEventListeners.has(element)) {
|
|
1898
|
+
return DOM_CACHE.nodeEventListeners.get(element);
|
|
1899
|
+
}
|
|
1900
|
+
const listenedEvents = getNodeEventListeners(element);
|
|
1901
|
+
if (listenedEvents) {
|
|
1902
|
+
DOM_CACHE.nodeEventListeners.set(element, listenedEvents);
|
|
1903
|
+
}
|
|
1904
|
+
return listenedEvents;
|
|
1905
|
+
}
|
|
1906
|
+
// ============================================================================
|
|
1907
|
+
// Action Intent Predicates
|
|
1908
|
+
// ============================================================================
|
|
1909
|
+
const CLICKABLE_TAGS = new Set(['a', 'button', 'summary', 'label', 'option', 'optgroup']);
|
|
1910
|
+
const CLICKABLE_ROLES = new Set([
|
|
1911
|
+
'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox',
|
|
1912
|
+
'radio', 'checkbox', 'tab', 'switch', 'option', 'treeitem'
|
|
1913
|
+
]);
|
|
1914
|
+
const CLICK_EVENTS = ['click', 'mousedown', 'mouseup', 'dblclick'];
|
|
1915
|
+
const TEXT_INPUT_TYPES = new Set([
|
|
1916
|
+
'text', 'email', 'password', 'search', 'tel', 'url', 'number',
|
|
1917
|
+
'date', 'datetime-local', 'month', 'week', 'time'
|
|
1918
|
+
]);
|
|
1919
|
+
const INPUT_ROLES = new Set(['textbox', 'searchbox', 'spinbutton', 'combobox']);
|
|
1920
|
+
/**
|
|
1921
|
+
* Checks if an element matches the 'click' intent.
|
|
1922
|
+
* Includes: buttons, links, elements with click handlers, clickable roles
|
|
1923
|
+
*/
|
|
1924
|
+
function isClickIntentElement(element) {
|
|
1925
|
+
const tagName = element.tagName.toLowerCase();
|
|
1926
|
+
if (CLICKABLE_TAGS.has(tagName))
|
|
1927
|
+
return true;
|
|
1928
|
+
const role = element.getAttribute('role');
|
|
1929
|
+
if (role && CLICKABLE_ROLES.has(role))
|
|
1930
|
+
return true;
|
|
1931
|
+
// Check for click event listeners
|
|
1932
|
+
const listeners = getCachedNodeEventListeners(element);
|
|
1933
|
+
if (listeners?.some(e => CLICK_EVENTS.includes(e)))
|
|
1934
|
+
return true;
|
|
1935
|
+
// Dropdown/popup triggers
|
|
1936
|
+
if (element.getAttribute('aria-haspopup') ||
|
|
1937
|
+
element.getAttribute('data-toggle') === 'dropdown')
|
|
1938
|
+
return true;
|
|
1939
|
+
return false;
|
|
1940
|
+
}
|
|
1941
|
+
/**
|
|
1942
|
+
* Checks if an element matches the 'input' intent.
|
|
1943
|
+
* Includes: text inputs, textareas, contenteditable elements
|
|
1944
|
+
*/
|
|
1945
|
+
function isInputIntentElement(element) {
|
|
1946
|
+
const tagName = element.tagName.toLowerCase();
|
|
1947
|
+
if (tagName === 'textarea')
|
|
1948
|
+
return true;
|
|
1949
|
+
if (tagName === 'input') {
|
|
1950
|
+
const type = element.type?.toLowerCase() || 'text';
|
|
1951
|
+
return TEXT_INPUT_TYPES.has(type);
|
|
1952
|
+
}
|
|
1953
|
+
if (element.isContentEditable ||
|
|
1954
|
+
element.getAttribute('contenteditable') === 'true')
|
|
1955
|
+
return true;
|
|
1956
|
+
const role = element.getAttribute('role');
|
|
1957
|
+
return role ? INPUT_ROLES.has(role) : false;
|
|
1958
|
+
}
|
|
1959
|
+
/**
|
|
1960
|
+
* Checks if an element matches the 'scroll' intent.
|
|
1961
|
+
* Delegates to existing isElementScrollable function.
|
|
1962
|
+
*/
|
|
1963
|
+
function isScrollIntentElement(element) {
|
|
1964
|
+
// Note: isElementScrollable is defined later but hoisted due to function declaration
|
|
1965
|
+
return isElementScrollable(element);
|
|
1966
|
+
}
|
|
1967
|
+
/**
|
|
1968
|
+
* Checks if an element matches the specified action intent.
|
|
1969
|
+
*/
|
|
1970
|
+
function matchesActionIntent(element, intent) {
|
|
1971
|
+
if (intent === 'all')
|
|
1972
|
+
return true;
|
|
1973
|
+
if (intent === 'click')
|
|
1974
|
+
return isClickIntentElement(element);
|
|
1975
|
+
if (intent === 'input')
|
|
1976
|
+
return isInputIntentElement(element);
|
|
1977
|
+
if (intent === 'scroll')
|
|
1978
|
+
return isScrollIntentElement(element);
|
|
1979
|
+
return true;
|
|
1980
|
+
}
|
|
1981
|
+
/**
|
|
1982
|
+
* Hash map of DOM nodes indexed by their highlight index.
|
|
1983
|
+
*/
|
|
1984
|
+
const DOM_HASH_MAP = {};
|
|
1985
|
+
const ID = { current: 0 };
|
|
1986
|
+
const HIGHLIGHT_CONTAINER_ID = "playwright-highlight-container";
|
|
1987
|
+
// Add a WeakMap cache for XPath strings
|
|
1988
|
+
const xpathCache = new WeakMap();
|
|
1989
|
+
const debugLogs = [];
|
|
1990
|
+
const debugLog = (msg) => {
|
|
1991
|
+
debugLogs.push(msg);
|
|
1992
|
+
};
|
|
1993
|
+
// ============================================================================
|
|
1994
|
+
// Grayscale Image Label Placement (Dynamic Convolution)
|
|
1995
|
+
// ============================================================================
|
|
1996
|
+
/**
|
|
1997
|
+
* 1D sliding window min/max using monotonic deque (Lemire algorithm).
|
|
1998
|
+
* O(n) time complexity - each element is pushed and popped at most once.
|
|
1999
|
+
*
|
|
2000
|
+
* @param arr - Array of values
|
|
2001
|
+
* @param k - Window size
|
|
2002
|
+
* @returns Object with maxResults and minResults arrays
|
|
2003
|
+
*/
|
|
2004
|
+
function slidingWindowMinMax(arr, k) {
|
|
2005
|
+
if (k <= 0 || arr.length === 0 || k > arr.length) {
|
|
2006
|
+
return { maxResults: [], minResults: [] };
|
|
2007
|
+
}
|
|
2008
|
+
const maxDeque = []; // Indices, values in descending order
|
|
2009
|
+
const minDeque = []; // Indices, values in ascending order
|
|
2010
|
+
const maxResults = [];
|
|
2011
|
+
const minResults = [];
|
|
2012
|
+
for (let i = 0; i < arr.length; i++) {
|
|
2013
|
+
const val = arr[i];
|
|
2014
|
+
// Update max deque - remove smaller elements from back
|
|
2015
|
+
while (maxDeque.length > 0 && arr[maxDeque[maxDeque.length - 1]] <= val) {
|
|
2016
|
+
maxDeque.pop();
|
|
2017
|
+
}
|
|
2018
|
+
maxDeque.push(i);
|
|
2019
|
+
// Remove front if outside window
|
|
2020
|
+
if (maxDeque[0] <= i - k)
|
|
2021
|
+
maxDeque.shift();
|
|
2022
|
+
// Update min deque - remove larger elements from back
|
|
2023
|
+
while (minDeque.length > 0 && arr[minDeque[minDeque.length - 1]] >= val) {
|
|
2024
|
+
minDeque.pop();
|
|
2025
|
+
}
|
|
2026
|
+
minDeque.push(i);
|
|
2027
|
+
// Remove front if outside window
|
|
2028
|
+
if (minDeque[0] <= i - k)
|
|
2029
|
+
minDeque.shift();
|
|
2030
|
+
// Record result once window is full
|
|
2031
|
+
if (i >= k - 1) {
|
|
2032
|
+
maxResults.push(arr[maxDeque[0]]);
|
|
2033
|
+
minResults.push(arr[minDeque[0]]);
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
return { maxResults, minResults };
|
|
2037
|
+
}
|
|
2038
|
+
/**
|
|
2039
|
+
* Compute 2D min/max over sliding windows within a bounded region.
|
|
2040
|
+
* Uses two-pass decomposition: horizontal pass then vertical pass.
|
|
2041
|
+
*
|
|
2042
|
+
* @param image - Full grayscale image (2D array where image[y][x] is intensity 0-255)
|
|
2043
|
+
* @param regionX - Starting X of the region to scan
|
|
2044
|
+
* @param regionY - Starting Y of the region to scan
|
|
2045
|
+
* @param regionWidth - Width of the region to scan
|
|
2046
|
+
* @param regionHeight - Height of the region to scan
|
|
2047
|
+
* @param labelWidth - Width of the label (kernel width)
|
|
2048
|
+
* @param labelHeight - Height of the label (kernel height)
|
|
2049
|
+
* @returns Object with windowMax and windowMin 2D arrays
|
|
2050
|
+
*/
|
|
2051
|
+
function compute2DMinMaxInRegion(image, regionX, regionY, regionWidth, regionHeight, labelWidth, labelHeight) {
|
|
2052
|
+
// Handle edge cases
|
|
2053
|
+
if (regionWidth < labelWidth || regionHeight < labelHeight) {
|
|
2054
|
+
return { windowMax: [], windowMin: [] };
|
|
2055
|
+
}
|
|
2056
|
+
const imageHeight = image.length;
|
|
2057
|
+
const imageWidth = image[0]?.length || 0;
|
|
2058
|
+
// Step 1: Horizontal pass - compute row-wise min/max for each row in region
|
|
2059
|
+
const rowMax = [];
|
|
2060
|
+
const rowMin = [];
|
|
2061
|
+
for (let dy = 0; dy < regionHeight; dy++) {
|
|
2062
|
+
const y = regionY + dy;
|
|
2063
|
+
if (y < 0 || y >= imageHeight) {
|
|
2064
|
+
// Out of bounds - use empty arrays
|
|
2065
|
+
rowMax[dy] = [];
|
|
2066
|
+
rowMin[dy] = [];
|
|
2067
|
+
continue;
|
|
2068
|
+
}
|
|
2069
|
+
// Extract the row slice from the region
|
|
2070
|
+
const rowSlice = [];
|
|
2071
|
+
for (let dx = 0; dx < regionWidth; dx++) {
|
|
2072
|
+
const x = regionX + dx;
|
|
2073
|
+
// Use 0 for out of bounds pixels (they'll fail uniformity check)
|
|
2074
|
+
rowSlice.push(x >= 0 && x < imageWidth ? (image[y][x] ?? 0) : 0);
|
|
2075
|
+
}
|
|
2076
|
+
const { maxResults, minResults } = slidingWindowMinMax(rowSlice, labelWidth);
|
|
2077
|
+
rowMax[dy] = maxResults;
|
|
2078
|
+
rowMin[dy] = minResults;
|
|
2079
|
+
}
|
|
2080
|
+
// Step 2: Vertical pass - compute column-wise min/max on intermediate buffers
|
|
2081
|
+
const resultWidth = regionWidth - labelWidth + 1;
|
|
2082
|
+
const resultHeight = regionHeight - labelHeight + 1;
|
|
2083
|
+
if (resultWidth <= 0 || resultHeight <= 0) {
|
|
2084
|
+
return { windowMax: [], windowMin: [] };
|
|
2085
|
+
}
|
|
2086
|
+
const windowMax = [];
|
|
2087
|
+
const windowMin = [];
|
|
2088
|
+
for (let x = 0; x < resultWidth; x++) {
|
|
2089
|
+
// Extract column from intermediate buffers
|
|
2090
|
+
const colMax = [];
|
|
2091
|
+
const colMin = [];
|
|
2092
|
+
for (let y = 0; y < regionHeight; y++) {
|
|
2093
|
+
colMax.push(rowMax[y]?.[x] ?? 0);
|
|
2094
|
+
colMin.push(rowMin[y]?.[x] ?? 255);
|
|
2095
|
+
}
|
|
2096
|
+
const { maxResults: colMaxResults } = slidingWindowMinMax(colMax, labelHeight);
|
|
2097
|
+
const { minResults: colMinResults } = slidingWindowMinMax(colMin, labelHeight);
|
|
2098
|
+
for (let y = 0; y < resultHeight; y++) {
|
|
2099
|
+
if (!windowMax[y])
|
|
2100
|
+
windowMax[y] = [];
|
|
2101
|
+
if (!windowMin[y])
|
|
2102
|
+
windowMin[y] = [];
|
|
2103
|
+
windowMax[y][x] = colMaxResults[y] ?? 0;
|
|
2104
|
+
windowMin[y][x] = colMinResults[y] ?? 255;
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
return { windowMax, windowMin };
|
|
2108
|
+
}
|
|
2109
|
+
/**
|
|
2110
|
+
* Draw a border on the grayscale image.
|
|
2111
|
+
* Used to mark element bounding boxes after they've been processed.
|
|
2112
|
+
*
|
|
2113
|
+
* @param image - Grayscale image (modified in place)
|
|
2114
|
+
* @param x - Left edge of the border
|
|
2115
|
+
* @param y - Top edge of the border
|
|
2116
|
+
* @param width - Width of the bordered region
|
|
2117
|
+
* @param height - Height of the bordered region
|
|
2118
|
+
* @param borderWidth - Width of the border in pixels (default 2)
|
|
2119
|
+
* @param borderColor - Gray value for the border (default 128)
|
|
2120
|
+
*/
|
|
2121
|
+
function drawBorderOnImage(image, x, y, width, height, borderWidth = 2, borderColor = 128) {
|
|
2122
|
+
const imageHeight = image.length;
|
|
2123
|
+
const imageWidth = image[0]?.length || 0;
|
|
2124
|
+
const x1 = Math.floor(x);
|
|
2125
|
+
const y1 = Math.floor(y);
|
|
2126
|
+
const x2 = Math.floor(x + width);
|
|
2127
|
+
const y2 = Math.floor(y + height);
|
|
2128
|
+
// Top edge
|
|
2129
|
+
for (let dy = 0; dy < borderWidth; dy++) {
|
|
2130
|
+
const py = y1 + dy;
|
|
2131
|
+
if (py >= 0 && py < imageHeight) {
|
|
2132
|
+
for (let px = x1; px < x2; px++) {
|
|
2133
|
+
if (px >= 0 && px < imageWidth) {
|
|
2134
|
+
image[py][px] = borderColor;
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
// Bottom edge
|
|
2140
|
+
for (let dy = 0; dy < borderWidth; dy++) {
|
|
2141
|
+
const py = y2 - 1 - dy;
|
|
2142
|
+
if (py >= 0 && py < imageHeight) {
|
|
2143
|
+
for (let px = x1; px < x2; px++) {
|
|
2144
|
+
if (px >= 0 && px < imageWidth) {
|
|
2145
|
+
image[py][px] = borderColor;
|
|
2146
|
+
}
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
}
|
|
2150
|
+
// Left edge (excluding corners)
|
|
2151
|
+
for (let py = y1 + borderWidth; py < y2 - borderWidth; py++) {
|
|
2152
|
+
if (py >= 0 && py < imageHeight) {
|
|
2153
|
+
for (let dx = 0; dx < borderWidth; dx++) {
|
|
2154
|
+
const px = x1 + dx;
|
|
2155
|
+
if (px >= 0 && px < imageWidth) {
|
|
2156
|
+
image[py][px] = borderColor;
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
2161
|
+
// Right edge (excluding corners)
|
|
2162
|
+
for (let py = y1 + borderWidth; py < y2 - borderWidth; py++) {
|
|
2163
|
+
if (py >= 0 && py < imageHeight) {
|
|
2164
|
+
for (let dx = 0; dx < borderWidth; dx++) {
|
|
2165
|
+
const px = x2 - 1 - dx;
|
|
2166
|
+
if (px >= 0 && px < imageWidth) {
|
|
2167
|
+
image[py][px] = borderColor;
|
|
2168
|
+
}
|
|
2169
|
+
}
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
/**
|
|
2174
|
+
* Mark a region as occupied with a striped pattern (0, 255, 0, 255...).
|
|
2175
|
+
* This guarantees max - min = 255 > tolerance, blocking any future label placement.
|
|
2176
|
+
*
|
|
2177
|
+
* Called after an element is done processing (bbox + label placed).
|
|
2178
|
+
*
|
|
2179
|
+
* @param image - Grayscale image (modified in place)
|
|
2180
|
+
* @param x1 - Left edge of element bbox
|
|
2181
|
+
* @param y1 - Top edge of element bbox
|
|
2182
|
+
* @param x2 - Right edge of element bbox
|
|
2183
|
+
* @param y2 - Bottom edge of element bbox
|
|
2184
|
+
* @param labelX - Label top-left X
|
|
2185
|
+
* @param labelY - Label top-left Y
|
|
2186
|
+
* @param labelW - Label width
|
|
2187
|
+
* @param labelH - Label height
|
|
2188
|
+
*/
|
|
2189
|
+
function markRegionAsOccupied(image, x1, y1, x2, y2, labelX, labelY, labelW, labelH) {
|
|
2190
|
+
const imageHeight = image.length;
|
|
2191
|
+
const imageWidth = image[0]?.length || 0;
|
|
2192
|
+
// Compute combined region (element + label)
|
|
2193
|
+
const minX = Math.floor(Math.min(x1, labelX));
|
|
2194
|
+
const minY = Math.floor(Math.min(y1, labelY));
|
|
2195
|
+
const maxX = Math.floor(Math.max(x2, labelX + labelW));
|
|
2196
|
+
const maxY = Math.floor(Math.max(y2, labelY + labelH));
|
|
2197
|
+
// Fill with interleaving 0, 255 pattern to guarantee non-uniformity
|
|
2198
|
+
for (let py = minY; py < maxY; py++) {
|
|
2199
|
+
if (py >= 0 && py < imageHeight) {
|
|
2200
|
+
for (let px = minX; px < maxX; px++) {
|
|
2201
|
+
if (px >= 0 && px < imageWidth) {
|
|
2202
|
+
image[py][px] = ((px + py) % 2 === 0) ? 0 : 255;
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
}
|
|
2208
|
+
/**
|
|
2209
|
+
* Find a valid label position using dynamic convolution on grayscale image.
|
|
2210
|
+
*
|
|
2211
|
+
* Algorithm:
|
|
2212
|
+
* 1. Define candidate region based on element bbox and label dimensions
|
|
2213
|
+
* 2. Compute 2D min/max over the region using sliding window
|
|
2214
|
+
* 3. Find first position where max - min <= tolerance (uniform region)
|
|
2215
|
+
* 4. Return that position or fallback
|
|
2216
|
+
*
|
|
2217
|
+
* @param elementRect - The element's bounding rect
|
|
2218
|
+
* @param labelW - Label width in pixels
|
|
2219
|
+
* @param labelH - Label height in pixels
|
|
2220
|
+
* @returns Position { x, y } for label top-left corner and whether grayscale was used
|
|
2221
|
+
*/
|
|
2222
|
+
function findLabelPosition(elementRect, labelW, labelH) {
|
|
2223
|
+
// If no grayscale image available, return fallback
|
|
2224
|
+
if (!grayscaleImage || grayscaleImage.length === 0) {
|
|
2225
|
+
return {
|
|
2226
|
+
x: Math.max(0, elementRect.x),
|
|
2227
|
+
y: Math.max(0, elementRect.y),
|
|
2228
|
+
usedGrayscale: false,
|
|
2229
|
+
};
|
|
2230
|
+
}
|
|
2231
|
+
const imageHeight = grayscaleImage.length;
|
|
2232
|
+
const imageWidth = grayscaleImage[0]?.length || 0;
|
|
2233
|
+
if (imageWidth === 0) {
|
|
2234
|
+
return {
|
|
2235
|
+
x: Math.max(0, elementRect.x),
|
|
2236
|
+
y: Math.max(0, elementRect.y),
|
|
2237
|
+
usedGrayscale: false,
|
|
2238
|
+
};
|
|
2239
|
+
}
|
|
2240
|
+
const { x: X, y: Y, width: N, height: M } = elementRect;
|
|
2241
|
+
const x1 = Math.floor(X);
|
|
2242
|
+
const y1 = Math.floor(Y);
|
|
2243
|
+
const x2 = Math.floor(X + N);
|
|
2244
|
+
const y2 = Math.floor(Y + M);
|
|
2245
|
+
// Define candidate region:
|
|
2246
|
+
// Label can be placed from (x1 - labelW, y1 - labelH) to (x2, y2)
|
|
2247
|
+
// This ensures label can touch any edge of the element
|
|
2248
|
+
// +1 on right/bottom to allow labels adjacent to (not overlapping) the border
|
|
2249
|
+
const regionX = Math.max(0, x1 - labelW);
|
|
2250
|
+
const regionY = Math.max(0, y1 - labelH);
|
|
2251
|
+
const regionX2 = Math.min(imageWidth, x2 + labelW);
|
|
2252
|
+
const regionY2 = Math.min(imageHeight, y2 + labelH);
|
|
2253
|
+
const regionWidth = regionX2 - regionX;
|
|
2254
|
+
const regionHeight = regionY2 - regionY;
|
|
2255
|
+
// Compute 2D min/max for the candidate region
|
|
2256
|
+
const { windowMax, windowMin } = compute2DMinMaxInRegion(grayscaleImage, regionX, regionY, regionWidth, regionHeight, labelW, labelH);
|
|
2257
|
+
if (windowMax.length === 0 || windowMin.length === 0) {
|
|
2258
|
+
// Region too small for label, use fallback
|
|
2259
|
+
const fallbackX = Math.max(0, Math.min(imageWidth - labelW, x2 - labelW));
|
|
2260
|
+
const fallbackY = Math.max(0, y1 - labelH);
|
|
2261
|
+
return { x: fallbackX, y: fallbackY, usedGrayscale: true };
|
|
2262
|
+
}
|
|
2263
|
+
// Search for a uniform position (max - min <= tolerance)
|
|
2264
|
+
// Prefer positions near element center
|
|
2265
|
+
const resultHeight = windowMax.length;
|
|
2266
|
+
const resultWidth = windowMax[0]?.length || 0;
|
|
2267
|
+
// Calculate center of candidate region (in result coordinates)
|
|
2268
|
+
const centerResultX = Math.floor(resultWidth / 2);
|
|
2269
|
+
const centerResultY = Math.floor(resultHeight / 2);
|
|
2270
|
+
// BFS from center to find nearest uniform position
|
|
2271
|
+
const visited = new Set();
|
|
2272
|
+
const queue = [{ rx: centerResultX, ry: centerResultY }];
|
|
2273
|
+
while (queue.length > 0) {
|
|
2274
|
+
const pos = queue.shift();
|
|
2275
|
+
const key = \`\${pos.rx},\${pos.ry}\`;
|
|
2276
|
+
if (visited.has(key))
|
|
2277
|
+
continue;
|
|
2278
|
+
visited.add(key);
|
|
2279
|
+
// Skip if already visited too many positions (performance limit)
|
|
2280
|
+
if (visited.size > 5000)
|
|
2281
|
+
break;
|
|
2282
|
+
// Check bounds
|
|
2283
|
+
if (pos.rx < 0 || pos.rx >= resultWidth || pos.ry < 0 || pos.ry >= resultHeight)
|
|
2284
|
+
continue;
|
|
2285
|
+
// Convert result coordinates to absolute image coordinates
|
|
2286
|
+
const absX = regionX + pos.rx;
|
|
2287
|
+
const absY = regionY + pos.ry;
|
|
2288
|
+
// Check uniformity: max - min <= tolerance
|
|
2289
|
+
const maxVal = windowMax[pos.ry]?.[pos.rx] ?? 255;
|
|
2290
|
+
const minVal = windowMin[pos.ry]?.[pos.rx] ?? 0;
|
|
2291
|
+
const diff = maxVal - minVal;
|
|
2292
|
+
if (diff <= uniformityTolerance) {
|
|
2293
|
+
// Uniform position found!
|
|
2294
|
+
return { x: absX, y: absY, usedGrayscale: true };
|
|
2295
|
+
}
|
|
2296
|
+
// Position not uniform, keep searching
|
|
2297
|
+
queue.push({ rx: pos.rx - 1, ry: pos.ry });
|
|
2298
|
+
queue.push({ rx: pos.rx + 1, ry: pos.ry });
|
|
2299
|
+
queue.push({ rx: pos.rx, ry: pos.ry - 1 });
|
|
2300
|
+
queue.push({ rx: pos.rx, ry: pos.ry + 1 });
|
|
2301
|
+
}
|
|
2302
|
+
// No uniform position found - fallback to outside element (top-right)
|
|
2303
|
+
const fallbackX = Math.max(0, Math.min(imageWidth - labelW, x2 - labelW));
|
|
2304
|
+
const fallbackY = Math.max(0, y1 - labelH);
|
|
2305
|
+
return { x: fallbackX, y: fallbackY, usedGrayscale: true };
|
|
2306
|
+
}
|
|
2307
|
+
/**
|
|
2308
|
+
* Mark both the element and label region as occupied after processing.
|
|
2309
|
+
* This prevents future labels from overlapping with already-labeled elements.
|
|
2310
|
+
*
|
|
2311
|
+
* In post-order traversal, children are labeled first. By marking the combined
|
|
2312
|
+
* region as occupied, we ensure parent labels don't cross child element areas.
|
|
2313
|
+
*/
|
|
2314
|
+
function markElementAndLabelAsOccupied(elementRect, labelX, labelY, labelW, labelH) {
|
|
2315
|
+
if (!grayscaleImage)
|
|
2316
|
+
return;
|
|
2317
|
+
const { x, y, width, height } = elementRect;
|
|
2318
|
+
markRegionAsOccupied(grayscaleImage, Math.floor(x), Math.floor(y), Math.floor(x + width), Math.floor(y + height), Math.floor(labelX), Math.floor(labelY), labelW, labelH);
|
|
2319
|
+
}
|
|
2320
|
+
const elementsToHighlight = [];
|
|
2321
|
+
/**
|
|
2322
|
+
* Reorder elements for post-order traversal (children before parents).
|
|
2323
|
+
* This ensures child elements get their labels placed before their parent containers.
|
|
2324
|
+
*
|
|
2325
|
+
* Builds a tree based on actual DOM ancestry between highlighted elements,
|
|
2326
|
+
* not the layout parent grouping used for flow detection.
|
|
2327
|
+
*/
|
|
2328
|
+
function getPostOrderElements(elements) {
|
|
2329
|
+
if (elements.length === 0)
|
|
2330
|
+
return [];
|
|
2331
|
+
// Build set of highlighted elements for quick lookup
|
|
2332
|
+
const highlightedSet = new Set();
|
|
2333
|
+
for (const info of elements) {
|
|
2334
|
+
highlightedSet.add(info.element);
|
|
2335
|
+
}
|
|
2336
|
+
// For each element, find its nearest highlighted ancestor (if any)
|
|
2337
|
+
// This builds a tree of highlighted elements based on DOM ancestry
|
|
2338
|
+
const childrenOfHighlighted = new Map();
|
|
2339
|
+
for (const info of elements) {
|
|
2340
|
+
let highlightedParent = null;
|
|
2341
|
+
let current = info.element.parentElement;
|
|
2342
|
+
// Walk up the DOM to find nearest highlighted ancestor
|
|
2343
|
+
while (current && current !== document.body) {
|
|
2344
|
+
if (highlightedSet.has(current)) {
|
|
2345
|
+
highlightedParent = current;
|
|
2346
|
+
break;
|
|
2347
|
+
}
|
|
2348
|
+
current = current.parentElement;
|
|
2349
|
+
}
|
|
2350
|
+
// Group elements by their highlighted parent
|
|
2351
|
+
if (!childrenOfHighlighted.has(highlightedParent)) {
|
|
2352
|
+
childrenOfHighlighted.set(highlightedParent, []);
|
|
2353
|
+
}
|
|
2354
|
+
childrenOfHighlighted.get(highlightedParent).push(info.element);
|
|
2355
|
+
}
|
|
2356
|
+
// Build element -> ElementToHighlight lookup
|
|
2357
|
+
const elementToInfo = new Map();
|
|
2358
|
+
for (const info of elements) {
|
|
2359
|
+
elementToInfo.set(info.element, info);
|
|
2360
|
+
}
|
|
2361
|
+
// Recursive post-order collection
|
|
2362
|
+
const result = [];
|
|
2363
|
+
const visited = new Set();
|
|
2364
|
+
function visit(element) {
|
|
2365
|
+
if (visited.has(element))
|
|
2366
|
+
return;
|
|
2367
|
+
visited.add(element);
|
|
2368
|
+
// First, visit all highlighted children (post-order: children before parent)
|
|
2369
|
+
const children = childrenOfHighlighted.get(element) || [];
|
|
2370
|
+
for (const child of children) {
|
|
2371
|
+
visit(child);
|
|
2372
|
+
}
|
|
2373
|
+
// Then add this element
|
|
2374
|
+
const info = elementToInfo.get(element);
|
|
2375
|
+
if (info)
|
|
2376
|
+
result.push(info);
|
|
2377
|
+
}
|
|
2378
|
+
// Start from roots (elements with no highlighted parent)
|
|
2379
|
+
const roots = childrenOfHighlighted.get(null) || [];
|
|
2380
|
+
for (const root of roots) {
|
|
2381
|
+
visit(root);
|
|
2382
|
+
}
|
|
2383
|
+
return result;
|
|
2384
|
+
}
|
|
2385
|
+
/**
|
|
2386
|
+
* Process elements in recursive tree order:
|
|
2387
|
+
* - Pre-order: Create bounding boxes and mark in hot map
|
|
2388
|
+
* - Post-order: Create labels and mark in hot map
|
|
2389
|
+
*
|
|
2390
|
+
* This ensures children's boxes are visible before parent places its label,
|
|
2391
|
+
* and children's labels are placed before parent's label.
|
|
2392
|
+
*/
|
|
2393
|
+
// Collected element data during boxes phase, for return
|
|
2394
|
+
const collectedElementData = [];
|
|
2395
|
+
function processElementTreeRecursively(elements) {
|
|
2396
|
+
if (elements.length === 0)
|
|
2397
|
+
return;
|
|
2398
|
+
// Build tree structure based on DOM ancestry (same as getPostOrderElements)
|
|
2399
|
+
const highlightedSet = new Set();
|
|
2400
|
+
for (const info of elements) {
|
|
2401
|
+
highlightedSet.add(info.element);
|
|
2402
|
+
}
|
|
2403
|
+
const childrenOfHighlighted = new Map();
|
|
2404
|
+
for (const info of elements) {
|
|
2405
|
+
let highlightedParent = null;
|
|
2406
|
+
let current = info.element.parentElement;
|
|
2407
|
+
while (current && current !== document.body) {
|
|
2408
|
+
if (highlightedSet.has(current)) {
|
|
2409
|
+
highlightedParent = current;
|
|
2410
|
+
break;
|
|
2411
|
+
}
|
|
2412
|
+
current = current.parentElement;
|
|
2413
|
+
}
|
|
2414
|
+
if (!childrenOfHighlighted.has(highlightedParent)) {
|
|
2415
|
+
childrenOfHighlighted.set(highlightedParent, []);
|
|
2416
|
+
}
|
|
2417
|
+
childrenOfHighlighted.get(highlightedParent).push(info.element);
|
|
2418
|
+
}
|
|
2419
|
+
// Build element -> ElementToHighlight lookup
|
|
2420
|
+
const elementToInfo = new Map();
|
|
2421
|
+
for (const info of elements) {
|
|
2422
|
+
elementToInfo.set(info.element, info);
|
|
2423
|
+
}
|
|
2424
|
+
// Get or create highlight container
|
|
2425
|
+
let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
|
|
2426
|
+
if (!container) {
|
|
2427
|
+
container = document.createElement("div");
|
|
2428
|
+
container.id = HIGHLIGHT_CONTAINER_ID;
|
|
2429
|
+
container.style.position = "fixed";
|
|
2430
|
+
container.style.pointerEvents = "none";
|
|
2431
|
+
container.style.top = "0";
|
|
2432
|
+
container.style.left = "0";
|
|
2433
|
+
container.style.width = "100%";
|
|
2434
|
+
container.style.height = "100%";
|
|
2435
|
+
container.style.zIndex = "2147483647";
|
|
2436
|
+
container.style.backgroundColor = 'transparent';
|
|
2437
|
+
document.body.appendChild(container);
|
|
2438
|
+
}
|
|
2439
|
+
// Store element info after box creation for later label creation
|
|
2440
|
+
const renderInfoMap = new Map();
|
|
2441
|
+
// Recursive function that processes elements in correct order
|
|
2442
|
+
function processElement(element) {
|
|
2443
|
+
const info = elementToInfo.get(element);
|
|
2444
|
+
if (!info)
|
|
2445
|
+
return;
|
|
2446
|
+
// === PRE-ORDER: Create bounding box ===
|
|
2447
|
+
const renderInfo = createBoundingBoxForElement(info.element, info.index, info.parentIframe, container);
|
|
2448
|
+
if (renderInfo) {
|
|
2449
|
+
renderInfoMap.set(element, renderInfo);
|
|
2450
|
+
// In boxes phase, collect element data for return (no grayscale drawing needed)
|
|
2451
|
+
if (phase === 'boxes') {
|
|
2452
|
+
collectedElementData.push({
|
|
2453
|
+
index: renderInfo.index,
|
|
2454
|
+
xpath: getXPathTree(info.element),
|
|
2455
|
+
rect: { ...renderInfo.elementRect },
|
|
2456
|
+
color: renderInfo.color,
|
|
2457
|
+
});
|
|
2458
|
+
}
|
|
2459
|
+
// In legacy mode (no phase), draw border on grayscale image
|
|
2460
|
+
// This ensures child labels don't get placed crossing this element's border
|
|
2461
|
+
if (!phase && grayscaleImage) {
|
|
2462
|
+
drawBorderOnImage(grayscaleImage, renderInfo.elementRect.x, renderInfo.elementRect.y, renderInfo.elementRect.width, renderInfo.elementRect.height, 2, // borderWidth
|
|
2463
|
+
128 // borderColor (mid-gray to break uniformity)
|
|
2464
|
+
);
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2467
|
+
// === RECURSE: Process children ===
|
|
2468
|
+
const children = childrenOfHighlighted.get(element) || [];
|
|
2469
|
+
for (const child of children) {
|
|
2470
|
+
processElement(child);
|
|
2471
|
+
}
|
|
2472
|
+
// === POST-ORDER: Create label (skip in boxes phase) ===
|
|
2473
|
+
if (renderInfo && phase !== 'boxes') {
|
|
2474
|
+
createLabelForElement(renderInfo);
|
|
2475
|
+
}
|
|
2476
|
+
}
|
|
2477
|
+
// Process roots (elements with no highlighted parent)
|
|
2478
|
+
const roots = childrenOfHighlighted.get(null) || [];
|
|
2479
|
+
for (const root of roots) {
|
|
2480
|
+
processElement(root);
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
/**
|
|
2484
|
+
* Process labels phase using pre-collected element data.
|
|
2485
|
+
* Called when phase === 'labels' with inputElementData.
|
|
2486
|
+
*/
|
|
2487
|
+
function processLabelsPhase() {
|
|
2488
|
+
if (!inputElementData || inputElementData.length === 0)
|
|
2489
|
+
return;
|
|
2490
|
+
streamLog(\`[dom-tree] Labels phase: processing \${inputElementData.length} elements\`);
|
|
2491
|
+
// Get or create highlight container (should already exist from boxes phase)
|
|
2492
|
+
let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
|
|
2493
|
+
if (!container) {
|
|
2494
|
+
container = document.createElement("div");
|
|
2495
|
+
container.id = HIGHLIGHT_CONTAINER_ID;
|
|
2496
|
+
container.style.position = "fixed";
|
|
2497
|
+
container.style.pointerEvents = "none";
|
|
2498
|
+
container.style.top = "0";
|
|
2499
|
+
container.style.left = "0";
|
|
2500
|
+
container.style.width = "100%";
|
|
2501
|
+
container.style.height = "100%";
|
|
2502
|
+
container.style.zIndex = "2147483647";
|
|
2503
|
+
container.style.backgroundColor = 'transparent';
|
|
2504
|
+
document.body.appendChild(container);
|
|
2505
|
+
}
|
|
2506
|
+
// Sort by index to maintain correct order
|
|
2507
|
+
const sortedData = [...inputElementData].sort((a, b) => a.index - b.index);
|
|
2508
|
+
// Create labels for each element
|
|
2509
|
+
for (const data of sortedData) {
|
|
2510
|
+
const renderInfo = {
|
|
2511
|
+
element: null, // Not needed for label creation
|
|
2512
|
+
index: data.index,
|
|
2513
|
+
parentIframe: null,
|
|
2514
|
+
elementRect: data.rect,
|
|
2515
|
+
color: data.color,
|
|
2516
|
+
container,
|
|
2517
|
+
};
|
|
2518
|
+
createLabelForElement(renderInfo);
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
/**
|
|
2522
|
+
* Create bounding box overlays for an element and append to container.
|
|
2523
|
+
* Returns element render info for later label creation.
|
|
2524
|
+
*/
|
|
2525
|
+
function createBoundingBoxForElement(element, index, parentIframe, container) {
|
|
2526
|
+
if (!element)
|
|
2527
|
+
return null;
|
|
2528
|
+
// Get element client rects
|
|
2529
|
+
let rects = element.getClientRects();
|
|
2530
|
+
if (!rects || rects.length === 0)
|
|
2531
|
+
return null;
|
|
2532
|
+
// Transform rects if inside an iframe
|
|
2533
|
+
if (parentIframe) {
|
|
2534
|
+
const transformedRects = [];
|
|
2535
|
+
const iframeRect = parentIframe.getBoundingClientRect();
|
|
2536
|
+
const iframeStyle = window.getComputedStyle(parentIframe);
|
|
2537
|
+
const borderLeft = parseFloat(iframeStyle.borderLeftWidth) || 0;
|
|
2538
|
+
const borderTop = parseFloat(iframeStyle.borderTopWidth) || 0;
|
|
2539
|
+
const paddingLeft = parseFloat(iframeStyle.paddingLeft) || 0;
|
|
2540
|
+
const paddingTop = parseFloat(iframeStyle.paddingTop) || 0;
|
|
2541
|
+
const contentOffsetX = borderLeft + paddingLeft;
|
|
2542
|
+
const contentOffsetY = borderTop + paddingTop;
|
|
2543
|
+
let scaleX = 1, scaleY = 1;
|
|
2544
|
+
const transform = iframeStyle.transform;
|
|
2545
|
+
if (transform && transform !== 'none') {
|
|
2546
|
+
const scaleMatch = transform.match(/scale\\(([^)]+)\\)/);
|
|
2547
|
+
if (scaleMatch) {
|
|
2548
|
+
const scaleValues = scaleMatch[1].split(',').map(v => parseFloat(v.trim()));
|
|
2549
|
+
scaleX = scaleValues[0] || 1;
|
|
2550
|
+
scaleY = scaleValues[1] || scaleX;
|
|
2551
|
+
}
|
|
2552
|
+
else {
|
|
2553
|
+
const matrixMatch = transform.match(/matrix\\(([^)]+)\\)/);
|
|
2554
|
+
if (matrixMatch) {
|
|
2555
|
+
const values = matrixMatch[1].split(',').map(v => parseFloat(v.trim()));
|
|
2556
|
+
if (values.length >= 6) {
|
|
2557
|
+
scaleX = values[0];
|
|
2558
|
+
scaleY = values[3];
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
let scrollLeft = 0, scrollTop = 0;
|
|
2564
|
+
try {
|
|
2565
|
+
const iframeDoc = parentIframe.contentDocument || parentIframe.contentWindow?.document;
|
|
2566
|
+
if (iframeDoc) {
|
|
2567
|
+
scrollLeft = iframeDoc.documentElement?.scrollLeft || iframeDoc.body?.scrollLeft || 0;
|
|
2568
|
+
scrollTop = iframeDoc.documentElement?.scrollTop || iframeDoc.body?.scrollTop || 0;
|
|
2569
|
+
}
|
|
2570
|
+
}
|
|
2571
|
+
catch (e) {
|
|
2572
|
+
console.warn("Cannot access iframe scroll position:", e);
|
|
2573
|
+
}
|
|
2574
|
+
for (const rect of rects) {
|
|
2575
|
+
const scaledWidth = rect.width * scaleX;
|
|
2576
|
+
const scaledHeight = rect.height * scaleY;
|
|
2577
|
+
const scaledTop = rect.top * scaleY;
|
|
2578
|
+
const scaledLeft = rect.left * scaleX;
|
|
2579
|
+
const scaledScrollTop = scrollTop * scaleY;
|
|
2580
|
+
const scaledScrollLeft = scrollLeft * scaleX;
|
|
2581
|
+
transformedRects.push({
|
|
2582
|
+
top: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop,
|
|
2583
|
+
left: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
2584
|
+
bottom: scaledTop + scaledHeight + iframeRect.top + contentOffsetY - scaledScrollTop,
|
|
2585
|
+
right: scaledLeft + scaledWidth + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
2586
|
+
width: scaledWidth,
|
|
2587
|
+
height: scaledHeight,
|
|
2588
|
+
x: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
|
|
2589
|
+
y: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop
|
|
2590
|
+
});
|
|
2591
|
+
}
|
|
2592
|
+
rects = transformedRects;
|
|
2593
|
+
}
|
|
2594
|
+
// Generate color based on index
|
|
2595
|
+
const colors = [
|
|
2596
|
+
"#E53935", "#1E88E5", "#7B1FA2", "#00897B", "#F4511E",
|
|
2597
|
+
"#3949AB", "#C2185B", "#00796B", "#5E35B1", "#D81B60",
|
|
2598
|
+
"#039BE5", "#388E3C"
|
|
2599
|
+
];
|
|
2600
|
+
const color = colors[index % colors.length];
|
|
2601
|
+
// Create bounding box overlays
|
|
2602
|
+
for (const rect of rects) {
|
|
2603
|
+
if (rect.width === 0 || rect.height === 0)
|
|
2604
|
+
continue;
|
|
2605
|
+
const overlay = document.createElement("div");
|
|
2606
|
+
overlay.style.position = "fixed";
|
|
2607
|
+
overlay.style.border = \`1px solid \${color}\`;
|
|
2608
|
+
overlay.style.backgroundColor = "none";
|
|
2609
|
+
overlay.style.pointerEvents = "none";
|
|
2610
|
+
overlay.style.boxSizing = "border-box";
|
|
2611
|
+
overlay.setAttribute("data-element-index", String(index));
|
|
2612
|
+
overlay.style.top = \`\${rect.top}px\`;
|
|
2613
|
+
overlay.style.left = \`\${rect.left}px\`;
|
|
2614
|
+
overlay.style.width = \`\${rect.width}px\`;
|
|
2615
|
+
overlay.style.height = \`\${rect.height}px\`;
|
|
2616
|
+
container.appendChild(overlay);
|
|
2617
|
+
}
|
|
2618
|
+
// Return element info for label creation
|
|
2619
|
+
const firstRect = rects[0];
|
|
2620
|
+
return {
|
|
2621
|
+
element,
|
|
2622
|
+
index,
|
|
2623
|
+
parentIframe,
|
|
2624
|
+
elementRect: {
|
|
2625
|
+
x: firstRect.left,
|
|
2626
|
+
y: firstRect.top,
|
|
2627
|
+
width: firstRect.width,
|
|
2628
|
+
height: firstRect.height
|
|
2629
|
+
},
|
|
2630
|
+
color,
|
|
2631
|
+
container
|
|
2632
|
+
};
|
|
2633
|
+
}
|
|
2634
|
+
/**
|
|
2635
|
+
* Create and position label for an element and append to container.
|
|
2636
|
+
*/
|
|
2637
|
+
function createLabelForElement(renderInfo) {
|
|
2638
|
+
const { index, elementRect, color, container } = renderInfo;
|
|
2639
|
+
// Capture BEFORE snapshot if debug mode enabled (stream via callback)
|
|
2640
|
+
streamLog(\`[dom-tree] Element \${index}: captureDebugSnapshots=\${captureDebugSnapshots}, hasGrayscale=\${!!grayscaleImage}, hasOnSnapshot=\${!!onSnapshot}\`);
|
|
2641
|
+
if (captureDebugSnapshots && grayscaleImage && onSnapshot) {
|
|
2642
|
+
streamLog(\`[dom-tree] Calling onSnapshot for element \${index} (before)\`);
|
|
2643
|
+
onSnapshot({ elementIndex: index, type: 'before', data: grayscaleImage.map(row => [...row]) });
|
|
2644
|
+
}
|
|
2645
|
+
// Calculate label dimensions based on digit count
|
|
2646
|
+
const digits = index.toString().length;
|
|
2647
|
+
const labelWidth = (digits * 8) + 8;
|
|
2648
|
+
const labelHeight = 16;
|
|
2649
|
+
// Find label position using grayscale image or fallback to heuristics
|
|
2650
|
+
let labelTop = 0;
|
|
2651
|
+
let labelLeft = 0;
|
|
2652
|
+
const grayscaleResult = findLabelPosition(elementRect, labelWidth, labelHeight);
|
|
2653
|
+
if (grayscaleResult.usedGrayscale) {
|
|
2654
|
+
labelTop = grayscaleResult.y;
|
|
2655
|
+
labelLeft = grayscaleResult.x;
|
|
2656
|
+
}
|
|
2657
|
+
else {
|
|
2658
|
+
// Fallback to heuristic positioning (assume vertical flow)
|
|
2659
|
+
const isNarrow = elementRect.width <= 32;
|
|
2660
|
+
const centerX = elementRect.x + elementRect.width / 2;
|
|
2661
|
+
let positions;
|
|
2662
|
+
if (isNarrow) {
|
|
2663
|
+
positions = [
|
|
2664
|
+
{ top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x, hAlign: 'right', name: 'left' },
|
|
2665
|
+
{ top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x + elementRect.width, hAlign: 'left', name: 'right' },
|
|
2666
|
+
{ top: elementRect.y - labelHeight, anchorX: centerX, hAlign: 'center', name: 'top' },
|
|
2667
|
+
{ top: elementRect.y + elementRect.height, anchorX: centerX, hAlign: 'center', name: 'bottom' },
|
|
2668
|
+
];
|
|
2669
|
+
}
|
|
2670
|
+
else {
|
|
2671
|
+
positions = [
|
|
2672
|
+
{ top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x, hAlign: 'right', name: 'left' },
|
|
2673
|
+
{ top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x + elementRect.width, hAlign: 'left', name: 'right' },
|
|
2674
|
+
{ top: elementRect.y - labelHeight, anchorX: elementRect.x + elementRect.width, hAlign: 'right', name: 'top-right' },
|
|
2675
|
+
{ top: elementRect.y - labelHeight, anchorX: elementRect.x, hAlign: 'left', name: 'top-left' },
|
|
2676
|
+
];
|
|
2677
|
+
}
|
|
2678
|
+
// Use first position as default
|
|
2679
|
+
const pos = positions[0];
|
|
2680
|
+
labelTop = Math.max(0, Math.min(pos.top, window.innerHeight - labelHeight));
|
|
2681
|
+
if (pos.hAlign === 'right') {
|
|
2682
|
+
labelLeft = pos.anchorX - labelWidth;
|
|
2683
|
+
}
|
|
2684
|
+
else if (pos.hAlign === 'center') {
|
|
2685
|
+
labelLeft = pos.anchorX - labelWidth / 2;
|
|
2686
|
+
}
|
|
2687
|
+
else {
|
|
2688
|
+
labelLeft = pos.anchorX;
|
|
2689
|
+
}
|
|
2690
|
+
labelLeft = Math.max(0, Math.min(labelLeft, window.innerWidth - labelWidth));
|
|
2691
|
+
}
|
|
2692
|
+
// Always mark the label position on grayscale image to prevent future overlaps
|
|
2693
|
+
markElementAndLabelAsOccupied(elementRect, labelLeft, labelTop, labelWidth, labelHeight);
|
|
2694
|
+
// Capture AFTER snapshot if debug mode enabled (stream via callback)
|
|
2695
|
+
if (captureDebugSnapshots && grayscaleImage && onSnapshot) {
|
|
2696
|
+
streamLog(\`[dom-tree] Calling onSnapshot for element \${index} (after)\`);
|
|
2697
|
+
onSnapshot({ elementIndex: index, type: 'after', data: grayscaleImage.map(row => [...row]) });
|
|
2698
|
+
}
|
|
2699
|
+
// Create label element with explicit dimensions matching theoretical calculation
|
|
2700
|
+
const label = document.createElement("div");
|
|
2701
|
+
label.className = "playwright-highlight-label";
|
|
2702
|
+
label.style.position = "fixed";
|
|
2703
|
+
label.style.background = color;
|
|
2704
|
+
label.style.color = "white";
|
|
2705
|
+
label.style.padding = "1px 4px";
|
|
2706
|
+
label.style.fontSize = "12px";
|
|
2707
|
+
label.style.width = \`\${labelWidth}px\`;
|
|
2708
|
+
label.style.height = \`\${labelHeight}px\`;
|
|
2709
|
+
label.style.boxSizing = "border-box";
|
|
2710
|
+
label.style.textAlign = "center";
|
|
2711
|
+
label.style.lineHeight = \`\${labelHeight - 2}px\`;
|
|
2712
|
+
label.textContent = String(index);
|
|
2713
|
+
label.setAttribute("data-element-index", String(index));
|
|
2714
|
+
label.style.top = \`\${labelTop}px\`;
|
|
2715
|
+
label.style.left = \`\${labelLeft}px\`;
|
|
2716
|
+
container.appendChild(label);
|
|
2717
|
+
}
|
|
2718
|
+
/**
|
|
2719
|
+
* Build tree entries for debug output showing element hierarchy.
|
|
2720
|
+
*
|
|
2721
|
+
* The tree collapses single-child chains: if A contains only B, which contains only C (highlighted),
|
|
2722
|
+
* then A, B, C collapse into just C.
|
|
2723
|
+
*
|
|
2724
|
+
* Parents are only included if they have more than one highlighted descendant.
|
|
2725
|
+
*/
|
|
2726
|
+
function buildTreeEntries() {
|
|
2727
|
+
const treeEntries = [];
|
|
2728
|
+
if (elementsToHighlight.length === 0)
|
|
2729
|
+
return treeEntries;
|
|
2730
|
+
// For each highlighted element, find all ancestors and count highlighted descendants
|
|
2731
|
+
const ancestorCounts = new Map();
|
|
2732
|
+
for (const { element } of elementsToHighlight) {
|
|
2733
|
+
let current = element.parentElement;
|
|
2734
|
+
while (current && current !== document.body) {
|
|
2735
|
+
ancestorCounts.set(current, (ancestorCounts.get(current) || 0) + 1);
|
|
2736
|
+
current = current.parentElement;
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
// Build tree structure: map each element to its layout parent
|
|
2740
|
+
const childrenMap = new Map(); // parent -> children
|
|
2741
|
+
for (const { element } of elementsToHighlight) {
|
|
2742
|
+
let layoutParent = null;
|
|
2743
|
+
let current = element.parentElement;
|
|
2744
|
+
while (current && current !== document.body) {
|
|
2745
|
+
const count = ancestorCounts.get(current) || 0;
|
|
2746
|
+
if (count > 1) {
|
|
2747
|
+
layoutParent = current;
|
|
2748
|
+
break;
|
|
2749
|
+
}
|
|
2750
|
+
current = current.parentElement;
|
|
2751
|
+
}
|
|
2752
|
+
// Group children by parent
|
|
2753
|
+
if (!childrenMap.has(layoutParent)) {
|
|
2754
|
+
childrenMap.set(layoutParent, []);
|
|
2755
|
+
}
|
|
2756
|
+
childrenMap.get(layoutParent).push(element);
|
|
2757
|
+
}
|
|
2758
|
+
// Output tree structure to debug logs and build tree entries
|
|
2759
|
+
const elementToIndex = new Map();
|
|
2760
|
+
for (const { element, index } of elementsToHighlight) {
|
|
2761
|
+
elementToIndex.set(element, index);
|
|
2762
|
+
}
|
|
2763
|
+
const describeElement = (el) => {
|
|
2764
|
+
const tag = el.tagName.toLowerCase();
|
|
2765
|
+
const id = el.id ? \`#\${el.id}\` : '';
|
|
2766
|
+
const cls = el.className && typeof el.className === 'string'
|
|
2767
|
+
? '.' + el.className.split(' ').slice(0, 2).join('.')
|
|
2768
|
+
: '';
|
|
2769
|
+
return \`<\${tag}\${id}\${cls.slice(0, 30)}>\`;
|
|
2770
|
+
};
|
|
2771
|
+
debugLog(\`[Tree] Detected elements tree:\`);
|
|
2772
|
+
// Get unique layout parents and sort by DOM order
|
|
2773
|
+
const parents = Array.from(childrenMap.keys());
|
|
2774
|
+
for (const parent of parents) {
|
|
2775
|
+
const children = childrenMap.get(parent) || [];
|
|
2776
|
+
const parentDesc = parent ? describeElement(parent) : '(root)';
|
|
2777
|
+
debugLog(\`[Tree] \${parentDesc}\`);
|
|
2778
|
+
// Add parent entry to tree
|
|
2779
|
+
treeEntries.push({
|
|
2780
|
+
type: 'parent',
|
|
2781
|
+
label: parentDesc,
|
|
2782
|
+
});
|
|
2783
|
+
for (const child of children) {
|
|
2784
|
+
const idx = elementToIndex.get(child) ?? -1;
|
|
2785
|
+
debugLog(\`[Tree] [\${idx}] \${describeElement(child)}\`);
|
|
2786
|
+
// Add element entry to tree
|
|
2787
|
+
treeEntries.push({
|
|
2788
|
+
type: 'element',
|
|
2789
|
+
highlightIndex: idx,
|
|
2790
|
+
label: describeElement(child),
|
|
2791
|
+
});
|
|
2792
|
+
}
|
|
2793
|
+
}
|
|
2794
|
+
return treeEntries;
|
|
2795
|
+
}
|
|
2796
|
+
/**
|
|
2797
|
+
* Gets the position of an element in its parent.
|
|
2798
|
+
*/
|
|
2799
|
+
function getElementPosition(currentElement) {
|
|
2800
|
+
if (!currentElement.parentElement) {
|
|
2801
|
+
return 0; // No parent means no siblings
|
|
2802
|
+
}
|
|
2803
|
+
const tagName = currentElement.nodeName.toLowerCase();
|
|
2804
|
+
const siblings = Array.from(currentElement.parentElement.children)
|
|
2805
|
+
.filter((sib) => sib.nodeName.toLowerCase() === tagName);
|
|
2806
|
+
if (siblings.length === 1) {
|
|
2807
|
+
return 0; // Only element of its type
|
|
2808
|
+
}
|
|
2809
|
+
const index = siblings.indexOf(currentElement) + 1; // 1-based index
|
|
2810
|
+
return index;
|
|
2811
|
+
}
|
|
2812
|
+
function getXPathTree(element, stopAtBoundary = true) {
|
|
2813
|
+
if (xpathCache.has(element))
|
|
2814
|
+
return xpathCache.get(element);
|
|
2815
|
+
const segments = [];
|
|
2816
|
+
let currentElement = element;
|
|
2817
|
+
while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) {
|
|
2818
|
+
// Stop if we hit a shadow root or iframe
|
|
2819
|
+
if (stopAtBoundary &&
|
|
2820
|
+
(currentElement.parentNode instanceof ShadowRoot ||
|
|
2821
|
+
currentElement.parentNode instanceof HTMLIFrameElement)) {
|
|
2822
|
+
break;
|
|
2823
|
+
}
|
|
2824
|
+
const position = getElementPosition(currentElement);
|
|
2825
|
+
const tagName = currentElement.nodeName.toLowerCase();
|
|
2826
|
+
const xpathIndex = position > 0 ? \`[\${position}]\` : "";
|
|
2827
|
+
segments.unshift(\`\${tagName}\${xpathIndex}\`);
|
|
2828
|
+
currentElement = currentElement.parentNode;
|
|
2829
|
+
}
|
|
2830
|
+
const result = segments.join("/");
|
|
2831
|
+
xpathCache.set(element, result);
|
|
2832
|
+
return result;
|
|
2833
|
+
}
|
|
2834
|
+
/**
|
|
2835
|
+
* Checks if a text node is visible.
|
|
2836
|
+
*/
|
|
2837
|
+
function isTextNodeVisible(textNode) {
|
|
2838
|
+
try {
|
|
2839
|
+
// Special case: when viewportExpansion is -1, consider all text nodes as visible
|
|
2840
|
+
if (viewportExpansion === -1) {
|
|
2841
|
+
// Still check parent visibility for basic filtering
|
|
2842
|
+
const parentElement = textNode.parentElement;
|
|
2843
|
+
if (!parentElement)
|
|
2844
|
+
return false;
|
|
2845
|
+
try {
|
|
2846
|
+
return parentElement.checkVisibility({
|
|
2847
|
+
checkOpacity: true,
|
|
2848
|
+
checkVisibilityCSS: true,
|
|
2849
|
+
});
|
|
2850
|
+
}
|
|
2851
|
+
catch (e) {
|
|
2852
|
+
// Fallback if checkVisibility is not supported
|
|
2853
|
+
const style = window.getComputedStyle(parentElement);
|
|
2854
|
+
return style.display !== 'none' &&
|
|
2855
|
+
style.visibility !== 'hidden' &&
|
|
2856
|
+
style.opacity !== '0';
|
|
2857
|
+
}
|
|
2858
|
+
}
|
|
2859
|
+
const range = document.createRange();
|
|
2860
|
+
range.selectNodeContents(textNode);
|
|
2861
|
+
const rects = range.getClientRects(); // Use getClientRects for Range
|
|
2862
|
+
if (!rects || rects.length === 0) {
|
|
2863
|
+
return false;
|
|
2864
|
+
}
|
|
2865
|
+
let isAnyRectVisible = false;
|
|
2866
|
+
let isAnyRectInViewport = false;
|
|
2867
|
+
for (const rect of rects) {
|
|
2868
|
+
// Check size
|
|
2869
|
+
if (rect.width > 0 && rect.height > 0) {
|
|
2870
|
+
isAnyRectVisible = true;
|
|
2871
|
+
// Viewport check for this rect
|
|
2872
|
+
if (!(rect.bottom < -viewportExpansion ||
|
|
2873
|
+
rect.top > window.innerHeight + viewportExpansion ||
|
|
2874
|
+
rect.right < -viewportExpansion ||
|
|
2875
|
+
rect.left > window.innerWidth + viewportExpansion)) {
|
|
2876
|
+
isAnyRectInViewport = true;
|
|
2877
|
+
break; // Found a visible rect in viewport, no need to check others
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
if (!isAnyRectVisible || !isAnyRectInViewport) {
|
|
2882
|
+
return false;
|
|
2883
|
+
}
|
|
2884
|
+
// Check parent visibility
|
|
2885
|
+
const parentElement = textNode.parentElement;
|
|
2886
|
+
if (!parentElement)
|
|
2887
|
+
return false;
|
|
2888
|
+
try {
|
|
2889
|
+
return parentElement.checkVisibility({
|
|
2890
|
+
checkOpacity: true,
|
|
2891
|
+
checkVisibilityCSS: true,
|
|
2892
|
+
});
|
|
2893
|
+
}
|
|
2894
|
+
catch (e) {
|
|
2895
|
+
// Fallback if checkVisibility is not supported
|
|
2896
|
+
const style = window.getComputedStyle(parentElement);
|
|
2897
|
+
return style.display !== 'none' &&
|
|
2898
|
+
style.visibility !== 'hidden' &&
|
|
2899
|
+
style.opacity !== '0';
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
catch (e) {
|
|
2903
|
+
console.warn('Error checking text node visibility:', e);
|
|
2904
|
+
return false;
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
/**
|
|
2908
|
+
* Checks if an element is accepted.
|
|
2909
|
+
*/
|
|
2910
|
+
function isElementAccepted(element) {
|
|
2911
|
+
if (!element || !element.tagName)
|
|
2912
|
+
return false;
|
|
2913
|
+
// Always accept body and common container elements
|
|
2914
|
+
const alwaysAccept = new Set([
|
|
2915
|
+
"body", "div", "main", "article", "section", "nav", "header", "footer"
|
|
2916
|
+
]);
|
|
2917
|
+
const tagName = element.tagName.toLowerCase();
|
|
2918
|
+
if (alwaysAccept.has(tagName))
|
|
2919
|
+
return true;
|
|
2920
|
+
const leafElementDenyList = new Set([
|
|
2921
|
+
"svg",
|
|
2922
|
+
"script",
|
|
2923
|
+
"style",
|
|
2924
|
+
"link",
|
|
2925
|
+
"meta",
|
|
2926
|
+
"noscript",
|
|
2927
|
+
"template",
|
|
2928
|
+
]);
|
|
2929
|
+
return !leafElementDenyList.has(tagName);
|
|
2930
|
+
}
|
|
2931
|
+
/**
|
|
2932
|
+
* Checks if an element is visible.
|
|
2933
|
+
*/
|
|
2934
|
+
function isElementVisible(element) {
|
|
2935
|
+
if (alwaysHighlightFileInput && element.tagName.toLowerCase() === "input" && element.type === "file")
|
|
2936
|
+
return true;
|
|
2937
|
+
// SVG elements need special handling for visibility
|
|
2938
|
+
if (element.tagName.toLowerCase() === "svg") {
|
|
2939
|
+
const rect = getCachedBoundingRect(element);
|
|
2940
|
+
const style = getCachedComputedStyle(element);
|
|
2941
|
+
return (rect !== null &&
|
|
2942
|
+
rect.width > 0 &&
|
|
2943
|
+
rect.height > 0 &&
|
|
2944
|
+
style?.visibility !== "hidden" &&
|
|
2945
|
+
style?.display !== "none");
|
|
2946
|
+
}
|
|
2947
|
+
const style = getCachedComputedStyle(element);
|
|
2948
|
+
return (element.offsetWidth > 0 &&
|
|
2949
|
+
element.offsetHeight > 0 &&
|
|
2950
|
+
style?.visibility !== "hidden" &&
|
|
2951
|
+
style?.display !== "none");
|
|
2952
|
+
}
|
|
2953
|
+
/**
|
|
2954
|
+
* Checks if an element is clickable (responds to click events).
|
|
2955
|
+
*/
|
|
2956
|
+
function shouldMarkAsClickable(element) {
|
|
2957
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) {
|
|
2958
|
+
return false;
|
|
2959
|
+
}
|
|
2960
|
+
const tagName = element.tagName.toLowerCase();
|
|
2961
|
+
// Primarily clickable elements
|
|
2962
|
+
const primaryClickableElements = new Set([
|
|
2963
|
+
"a", // Links
|
|
2964
|
+
"button", // Buttons
|
|
2965
|
+
"summary", // Summary element (clickable part of details)
|
|
2966
|
+
"label", // Form labels (often clickable)
|
|
2967
|
+
"option", // Select options
|
|
2968
|
+
"optgroup", // Option groups
|
|
2969
|
+
]);
|
|
2970
|
+
if (primaryClickableElements.has(tagName)) {
|
|
2971
|
+
return false;
|
|
2972
|
+
}
|
|
2973
|
+
const role = element.getAttribute("role");
|
|
2974
|
+
// Clickable roles
|
|
2975
|
+
const clickableRoles = new Set([
|
|
2976
|
+
'button', // Directly clickable element
|
|
2977
|
+
'link', // Clickable link
|
|
2978
|
+
'menuitem', // Clickable menu item
|
|
2979
|
+
'menuitemradio', // Radio-style menu item (selectable)
|
|
2980
|
+
'menuitemcheckbox', // Checkbox-style menu item (toggleable)
|
|
2981
|
+
'radio', // Radio button (selectable)
|
|
2982
|
+
'checkbox', // Checkbox (toggleable)
|
|
2983
|
+
'tab', // Tab (clickable to switch content)
|
|
2984
|
+
'switch', // Toggle switch (clickable to change state)
|
|
2985
|
+
'option', // Selectable option in a list
|
|
2986
|
+
]);
|
|
2987
|
+
if (role && clickableRoles.has(role)) {
|
|
2988
|
+
return true;
|
|
2989
|
+
}
|
|
2990
|
+
// Check for dropdown indicators
|
|
2991
|
+
if (hasAnyClassName(element, buttonClassNames)) {
|
|
2992
|
+
return true; // Return true for dropdown elements
|
|
2993
|
+
}
|
|
2994
|
+
if (element.getAttribute('data-toggle') === 'dropdown' ||
|
|
2995
|
+
element.getAttribute('aria-haspopup')) {
|
|
2996
|
+
return true;
|
|
2997
|
+
}
|
|
2998
|
+
const clickEvents = ['click', 'mousedown', 'mouseup', 'dblclick'];
|
|
2999
|
+
const listenedEvents = getCachedNodeEventListeners(element);
|
|
3000
|
+
if (listenedEvents && listenedEvents.length > 0) {
|
|
3001
|
+
for (const eventType of clickEvents) {
|
|
3002
|
+
if (listenedEvents.includes(eventType)) {
|
|
3003
|
+
return true;
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
}
|
|
3007
|
+
return false;
|
|
3008
|
+
}
|
|
3009
|
+
/**
|
|
3010
|
+
* Checks if an element is interactive.
|
|
3011
|
+
*
|
|
3012
|
+
* lots of comments, and uncommented code - to show the logic of what we already tried
|
|
3013
|
+
*
|
|
3014
|
+
* One of the things we tried at the beginning was also to use event listeners, and other fancy class, style stuff -> what actually worked best was just combining most things with computed cursor style :)
|
|
3015
|
+
*/
|
|
3016
|
+
function isInteractiveElement(element) {
|
|
3017
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) {
|
|
3018
|
+
return false;
|
|
3019
|
+
}
|
|
3020
|
+
// Cache the tagName and style lookups
|
|
3021
|
+
const tagName = element.tagName.toLowerCase();
|
|
3022
|
+
const style = getCachedComputedStyle(element);
|
|
3023
|
+
// Define interactive cursors
|
|
3024
|
+
const interactiveCursors = new Set([
|
|
3025
|
+
'pointer', // Link/clickable elements
|
|
3026
|
+
'move', // Movable elements
|
|
3027
|
+
'text', // Text selection
|
|
3028
|
+
'grab', // Grabbable elements
|
|
3029
|
+
'grabbing', // Currently grabbing
|
|
3030
|
+
'cell', // Table cell selection
|
|
3031
|
+
'copy', // Copy operation
|
|
3032
|
+
'alias', // Alias creation
|
|
3033
|
+
'all-scroll', // Scrollable content
|
|
3034
|
+
'col-resize', // Column resize
|
|
3035
|
+
'context-menu', // Context menu available
|
|
3036
|
+
'crosshair', // Precise selection
|
|
3037
|
+
'e-resize', // East resize
|
|
3038
|
+
'ew-resize', // East-west resize
|
|
3039
|
+
'help', // Help available
|
|
3040
|
+
'n-resize', // North resize
|
|
3041
|
+
'ne-resize', // Northeast resize
|
|
3042
|
+
'nesw-resize', // Northeast-southwest resize
|
|
3043
|
+
'ns-resize', // North-south resize
|
|
3044
|
+
'nw-resize', // Northwest resize
|
|
3045
|
+
'nwse-resize', // Northwest-southeast resize
|
|
3046
|
+
'row-resize', // Row resize
|
|
3047
|
+
's-resize', // South resize
|
|
3048
|
+
'se-resize', // Southeast resize
|
|
3049
|
+
'sw-resize', // Southwest resize
|
|
3050
|
+
'vertical-text', // Vertical text selection
|
|
3051
|
+
'w-resize', // West resize
|
|
3052
|
+
'zoom-in', // Zoom in
|
|
3053
|
+
'zoom-out' // Zoom out
|
|
3054
|
+
]);
|
|
3055
|
+
// Define non-interactive cursors
|
|
3056
|
+
const nonInteractiveCursors = new Set([
|
|
3057
|
+
'not-allowed', // Action not allowed
|
|
3058
|
+
'no-drop', // Drop not allowed
|
|
3059
|
+
'wait', // Processing
|
|
3060
|
+
'progress', // In progress
|
|
3061
|
+
'initial', // Initial value
|
|
3062
|
+
'inherit' // Inherited value
|
|
3063
|
+
//? Let's just include all potentially clickable elements that are not specifically blocked
|
|
3064
|
+
// 'none', // No cursor
|
|
3065
|
+
// 'default', // Default cursor
|
|
3066
|
+
// 'auto', // Browser default
|
|
3067
|
+
]);
|
|
3068
|
+
/**
|
|
3069
|
+
* Checks if an element has an interactive pointer.
|
|
3070
|
+
*/
|
|
3071
|
+
function doesElementHaveInteractivePointer(element) {
|
|
3072
|
+
if (element.tagName.toLowerCase() === "html")
|
|
3073
|
+
return false;
|
|
3074
|
+
if (style?.cursor && interactiveCursors.has(style.cursor))
|
|
3075
|
+
return true;
|
|
3076
|
+
return false;
|
|
3077
|
+
}
|
|
3078
|
+
// Disabled for now, since it adds too many false positives
|
|
3079
|
+
// let isInteractiveCursor = doesElementHaveInteractivePointer(element);
|
|
3080
|
+
// // Genius fix for almost all interactive elements
|
|
3081
|
+
// if (isInteractiveCursor) {
|
|
3082
|
+
// return true;
|
|
3083
|
+
// }
|
|
3084
|
+
const interactiveElements = new Set([
|
|
3085
|
+
"a", // Links
|
|
3086
|
+
"button", // Buttons
|
|
3087
|
+
"input", // All input types (text, checkbox, radio, etc.)
|
|
3088
|
+
"select", // Dropdown menus
|
|
3089
|
+
"textarea", // Text areas
|
|
3090
|
+
"summary", // Summary element (clickable part of details)
|
|
3091
|
+
"label", // Form labels (often clickable)
|
|
3092
|
+
"option", // Select options
|
|
3093
|
+
"optgroup", // Option groups
|
|
3094
|
+
"fieldset", // Form fieldsets (can be interactive with legend)
|
|
3095
|
+
"legend", // Fieldset legends
|
|
3096
|
+
]);
|
|
3097
|
+
// Define explicit disable attributes and properties
|
|
3098
|
+
const explicitDisableTags = new Set([
|
|
3099
|
+
'disabled', // Standard disabled attribute
|
|
3100
|
+
// 'aria-disabled', // ARIA disabled state
|
|
3101
|
+
// 'readonly', // Read-only state
|
|
3102
|
+
// 'aria-readonly', // ARIA read-only state
|
|
3103
|
+
// 'aria-hidden', // Hidden from accessibility
|
|
3104
|
+
// 'hidden', // Hidden attribute
|
|
3105
|
+
// 'inert', // Inert attribute
|
|
3106
|
+
// 'aria-inert', // ARIA inert state
|
|
3107
|
+
// 'tabindex="-1"', // Removed from tab order
|
|
3108
|
+
// 'aria-hidden="true"' // Hidden from screen readers
|
|
3109
|
+
]);
|
|
3110
|
+
// Check for non-interactive cursor
|
|
3111
|
+
if (style?.cursor && nonInteractiveCursors.has(style.cursor)) {
|
|
3112
|
+
return false;
|
|
3113
|
+
}
|
|
3114
|
+
// handle inputs, select, checkbox, radio, textarea, button and make sure they are not cursor style disabled/not-allowed
|
|
3115
|
+
if (interactiveElements.has(tagName)) {
|
|
3116
|
+
// Check for explicit disable attributes
|
|
3117
|
+
for (const disableTag of explicitDisableTags) {
|
|
3118
|
+
if (element.hasAttribute(disableTag) ||
|
|
3119
|
+
element.getAttribute(disableTag) === 'true' ||
|
|
3120
|
+
element.getAttribute(disableTag) === '') {
|
|
3121
|
+
return false;
|
|
3122
|
+
}
|
|
3123
|
+
}
|
|
3124
|
+
// Check for disabled property on form elements
|
|
3125
|
+
if (element.disabled) {
|
|
3126
|
+
return false;
|
|
3127
|
+
}
|
|
3128
|
+
// Don't mark as non-interactive yet
|
|
3129
|
+
// Check for readonly property on form elements
|
|
3130
|
+
if (element.readOnly) {
|
|
3131
|
+
// return false;
|
|
3132
|
+
}
|
|
3133
|
+
// Check for inert property
|
|
3134
|
+
if (element.inert) {
|
|
3135
|
+
return false;
|
|
3136
|
+
}
|
|
3137
|
+
return true;
|
|
3138
|
+
}
|
|
3139
|
+
const role = element.getAttribute("role");
|
|
3140
|
+
const ariaRole = element.getAttribute("aria-role");
|
|
3141
|
+
// Check for contenteditable attribute
|
|
3142
|
+
if (element.getAttribute("contenteditable") === "true" || element.isContentEditable) {
|
|
3143
|
+
return true;
|
|
3144
|
+
}
|
|
3145
|
+
// Added enhancement to capture dropdown interactive elements
|
|
3146
|
+
if (hasAnyClassName(element, buttonClassNames) ||
|
|
3147
|
+
hasAnyClassName(element, interactiveClassNames) ||
|
|
3148
|
+
hasAnyClassName(element, cursorPointerClassNames) ||
|
|
3149
|
+
element.getAttribute('data-index') ||
|
|
3150
|
+
element.getAttribute('data-toggle') === 'dropdown' ||
|
|
3151
|
+
element.getAttribute('aria-haspopup')) {
|
|
3152
|
+
return true;
|
|
3153
|
+
}
|
|
3154
|
+
const interactiveRoles = new Set([
|
|
3155
|
+
'button', // Directly clickable element
|
|
3156
|
+
'link', // Clickable link
|
|
3157
|
+
'menuitem', // Clickable menu item
|
|
3158
|
+
'menuitemradio', // Radio-style menu item (selectable)
|
|
3159
|
+
'menuitemcheckbox', // Checkbox-style menu item (toggleable)
|
|
3160
|
+
'radio', // Radio button (selectable)
|
|
3161
|
+
'checkbox', // Checkbox (toggleable)
|
|
3162
|
+
'tab', // Tab (clickable to switch content)
|
|
3163
|
+
'switch', // Toggle switch (clickable to change state)
|
|
3164
|
+
'slider', // Slider control (draggable)
|
|
3165
|
+
'spinbutton', // Number input with up/down controls
|
|
3166
|
+
'combobox', // Dropdown with text input
|
|
3167
|
+
'searchbox', // Search input field
|
|
3168
|
+
'textbox', // Text input field
|
|
3169
|
+
'listbox', // Selectable list
|
|
3170
|
+
'option', // Selectable option in a list
|
|
3171
|
+
'scrollbar' // Scrollable control
|
|
3172
|
+
]);
|
|
3173
|
+
// Basic role/attribute checks
|
|
3174
|
+
const hasInteractiveRole = (role && interactiveRoles.has(role)) ||
|
|
3175
|
+
(ariaRole && interactiveRoles.has(ariaRole));
|
|
3176
|
+
if (hasInteractiveRole)
|
|
3177
|
+
return true;
|
|
3178
|
+
const listenedEvents = getCachedNodeEventListeners(element);
|
|
3179
|
+
if (listenedEvents && listenedEvents.length > 0) {
|
|
3180
|
+
for (const eventType of INTERACTION_EVENTS) {
|
|
3181
|
+
if (listenedEvents.includes(eventType)) {
|
|
3182
|
+
return true;
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
}
|
|
3186
|
+
return false;
|
|
3187
|
+
}
|
|
3188
|
+
/**
|
|
3189
|
+
* Checks if an element is the topmost element at its position.
|
|
3190
|
+
*/
|
|
3191
|
+
function isTopElement(element) {
|
|
3192
|
+
// Special case: when viewportExpansion is -1, consider all elements as "top" elements
|
|
3193
|
+
if (viewportExpansion === -1) {
|
|
3194
|
+
return true;
|
|
3195
|
+
}
|
|
3196
|
+
const rects = getCachedClientRects(element);
|
|
3197
|
+
if (!rects || rects.length === 0) {
|
|
3198
|
+
return false; // No geometry, cannot be top
|
|
3199
|
+
}
|
|
3200
|
+
let isAnyRectInViewport = false;
|
|
3201
|
+
for (const rect of rects) {
|
|
3202
|
+
// Use the same logic as isInExpandedViewport check
|
|
3203
|
+
if (rect.width > 0 && rect.height > 0 && !( // Only check non-empty rects
|
|
3204
|
+
rect.bottom < -viewportExpansion ||
|
|
3205
|
+
rect.top > window.innerHeight + viewportExpansion ||
|
|
3206
|
+
rect.right < -viewportExpansion ||
|
|
3207
|
+
rect.left > window.innerWidth + viewportExpansion)) {
|
|
3208
|
+
isAnyRectInViewport = true;
|
|
3209
|
+
break;
|
|
3210
|
+
}
|
|
3211
|
+
}
|
|
3212
|
+
if (!isAnyRectInViewport) {
|
|
3213
|
+
return false; // All rects are outside the viewport area
|
|
3214
|
+
}
|
|
3215
|
+
// Find the correct document context and root element
|
|
3216
|
+
let doc = element.ownerDocument;
|
|
3217
|
+
// If we're in an iframe, elements are considered top by default
|
|
3218
|
+
if (doc !== window.document) {
|
|
3219
|
+
return true;
|
|
3220
|
+
}
|
|
3221
|
+
// For shadow DOM, we need to check within its own root context
|
|
3222
|
+
const shadowRoot = element.getRootNode();
|
|
3223
|
+
if (shadowRoot instanceof ShadowRoot) {
|
|
3224
|
+
const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
|
|
3225
|
+
const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
|
|
3226
|
+
try {
|
|
3227
|
+
const topEl = shadowRoot.elementFromPoint(centerX, centerY);
|
|
3228
|
+
if (!topEl)
|
|
3229
|
+
return false;
|
|
3230
|
+
let current = topEl;
|
|
3231
|
+
while (current && current !== shadowRoot) {
|
|
3232
|
+
if (current === element)
|
|
3233
|
+
return true;
|
|
3234
|
+
current = current.parentElement;
|
|
3235
|
+
}
|
|
3236
|
+
return false;
|
|
3237
|
+
}
|
|
3238
|
+
catch (e) {
|
|
3239
|
+
return true;
|
|
3240
|
+
}
|
|
3241
|
+
}
|
|
3242
|
+
// For elements in viewport, check if they're topmost
|
|
3243
|
+
const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
|
|
3244
|
+
const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
|
|
3245
|
+
try {
|
|
3246
|
+
const topEl = document.elementFromPoint(centerX, centerY);
|
|
3247
|
+
if (!topEl)
|
|
3248
|
+
return false;
|
|
3249
|
+
let current = topEl;
|
|
3250
|
+
while (current && current !== document.documentElement) {
|
|
3251
|
+
if (current === element)
|
|
3252
|
+
return true;
|
|
3253
|
+
current = current.parentElement;
|
|
3254
|
+
}
|
|
3255
|
+
return false;
|
|
3256
|
+
}
|
|
3257
|
+
catch (e) {
|
|
3258
|
+
return true;
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
3261
|
+
/**
|
|
3262
|
+
* Checks if an element is within the expanded viewport.
|
|
3263
|
+
*/
|
|
3264
|
+
function isInExpandedViewport(element, viewportExpansion) {
|
|
3265
|
+
if (viewportExpansion === -1) {
|
|
3266
|
+
return true;
|
|
3267
|
+
}
|
|
3268
|
+
const rects = element.getClientRects();
|
|
3269
|
+
if (!rects || rects.length === 0) {
|
|
3270
|
+
// Fallback to getBoundingClientRect if getClientRects is empty,
|
|
3271
|
+
// useful for elements like <svg> that might not have client rects but have a bounding box.
|
|
3272
|
+
const boundingRect = getCachedBoundingRect(element);
|
|
3273
|
+
if (!boundingRect || boundingRect.width === 0 || boundingRect.height === 0) {
|
|
3274
|
+
return false;
|
|
3275
|
+
}
|
|
3276
|
+
return !(boundingRect.bottom < -viewportExpansion ||
|
|
3277
|
+
boundingRect.top > window.innerHeight + viewportExpansion ||
|
|
3278
|
+
boundingRect.right < -viewportExpansion ||
|
|
3279
|
+
boundingRect.left > window.innerWidth + viewportExpansion);
|
|
3280
|
+
}
|
|
3281
|
+
// Check if *any* client rect is within the viewport
|
|
3282
|
+
for (const rect of rects) {
|
|
3283
|
+
if (rect.width === 0 || rect.height === 0)
|
|
3284
|
+
continue; // Skip empty rects
|
|
3285
|
+
if (!(rect.bottom < -viewportExpansion ||
|
|
3286
|
+
rect.top > window.innerHeight + viewportExpansion ||
|
|
3287
|
+
rect.right < -viewportExpansion ||
|
|
3288
|
+
rect.left > window.innerWidth + viewportExpansion)) {
|
|
3289
|
+
return true; // Found at least one rect in the viewport
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
return false; // No rects were found in the viewport
|
|
3293
|
+
}
|
|
3294
|
+
// /**
|
|
3295
|
+
// * Gets the effective scroll of an element.
|
|
3296
|
+
// *
|
|
3297
|
+
// * @param {HTMLElement} element - The element to get the effective scroll for.
|
|
3298
|
+
// * @returns {Object} The effective scroll of the element.
|
|
3299
|
+
// */
|
|
3300
|
+
// function getEffectiveScroll(element) {
|
|
3301
|
+
// let currentEl = element;
|
|
3302
|
+
// let scrollX = 0;
|
|
3303
|
+
// let scrollY = 0;
|
|
3304
|
+
// while (currentEl && currentEl !== document.documentElement) {
|
|
3305
|
+
// if (currentEl.scrollLeft || currentEl.scrollTop) {
|
|
3306
|
+
// scrollX += currentEl.scrollLeft;
|
|
3307
|
+
// scrollY += currentEl.scrollTop;
|
|
3308
|
+
// }
|
|
3309
|
+
// currentEl = currentEl.parentElement;
|
|
3310
|
+
// }
|
|
3311
|
+
// scrollX += window.scrollX;
|
|
3312
|
+
// scrollY += window.scrollY;
|
|
3313
|
+
// return { scrollX, scrollY };
|
|
3314
|
+
// }
|
|
3315
|
+
/**
|
|
3316
|
+
* Checks if an element is an interactive candidate.
|
|
3317
|
+
*/
|
|
3318
|
+
function isInteractiveCandidate(element) {
|
|
3319
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE)
|
|
3320
|
+
return false;
|
|
3321
|
+
const tagName = element.tagName.toLowerCase();
|
|
3322
|
+
// Fast-path for common interactive elements
|
|
3323
|
+
const interactiveElements = new Set([
|
|
3324
|
+
"a", "button", "input", "select", "textarea", "summary", "label"
|
|
3325
|
+
]);
|
|
3326
|
+
if (interactiveElements.has(tagName))
|
|
3327
|
+
return true;
|
|
3328
|
+
// Quick attribute checks without getting full lists
|
|
3329
|
+
const hasQuickInteractiveAttr = element.hasAttribute("onclick") ||
|
|
3330
|
+
element.hasAttribute("role") ||
|
|
3331
|
+
element.hasAttribute("tabindex") ||
|
|
3332
|
+
element.hasAttribute("aria-") ||
|
|
3333
|
+
element.hasAttribute("data-action") ||
|
|
3334
|
+
element.getAttribute("contenteditable") === "true";
|
|
3335
|
+
return hasQuickInteractiveAttr;
|
|
3336
|
+
}
|
|
3337
|
+
// --- Define constants for distinct interaction check ---
|
|
3338
|
+
const DISTINCT_INTERACTIVE_TAGS = new Set([
|
|
3339
|
+
'a', 'button', 'input', 'select', 'textarea', 'summary', 'label', 'option'
|
|
3340
|
+
]);
|
|
3341
|
+
const INTERACTIVE_ROLES = new Set([
|
|
3342
|
+
'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox',
|
|
3343
|
+
'radio', 'checkbox', 'tab', 'switch', 'slider', 'spinbutton',
|
|
3344
|
+
'combobox', 'searchbox', 'textbox', 'listbox', 'option', 'scrollbar'
|
|
3345
|
+
]);
|
|
3346
|
+
/**
|
|
3347
|
+
* Heuristically determines if an element should be considered as independently interactive,
|
|
3348
|
+
* even if it's nested inside another interactive container.
|
|
3349
|
+
*
|
|
3350
|
+
* This function helps detect deeply nested actionable elements (e.g., menu items within a button)
|
|
3351
|
+
* that may not be picked up by strict interactivity checks.
|
|
3352
|
+
*/
|
|
3353
|
+
function isHeuristicallyInteractive(element) {
|
|
3354
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE)
|
|
3355
|
+
return false;
|
|
3356
|
+
// Skip non-visible elements early for performance
|
|
3357
|
+
if (!isElementVisible(element))
|
|
3358
|
+
return false;
|
|
3359
|
+
// Check for common attributes that often indicate interactivity
|
|
3360
|
+
const hasInteractiveAttributes = element.hasAttribute('role') ||
|
|
3361
|
+
element.hasAttribute('tabindex') ||
|
|
3362
|
+
element.hasAttribute('onclick') ||
|
|
3363
|
+
typeof element.onclick === 'function';
|
|
3364
|
+
// Check for semantic class names suggesting interactivity
|
|
3365
|
+
const hasInteractiveClass = heuristicClassPattern.test(element.className || '');
|
|
3366
|
+
// Determine whether the element is inside a known interactive container
|
|
3367
|
+
const isInKnownContainer = Boolean(element.closest(containerSelectors));
|
|
3368
|
+
// Ensure the element has at least one visible child (to avoid marking empty wrappers)
|
|
3369
|
+
const hasVisibleChildren = [...element.children].some(child => isElementVisible(child));
|
|
3370
|
+
// Avoid highlighting elements whose parent is <body> (top-level wrappers)
|
|
3371
|
+
const isParentBody = element.parentElement && element.parentElement.isSameNode(document.body);
|
|
3372
|
+
return ((isInteractiveElement(element) || hasInteractiveAttributes || hasInteractiveClass) &&
|
|
3373
|
+
hasVisibleChildren &&
|
|
3374
|
+
isInKnownContainer &&
|
|
3375
|
+
!isParentBody);
|
|
3376
|
+
}
|
|
3377
|
+
/**
|
|
3378
|
+
* Checks if an element likely represents a distinct interaction
|
|
3379
|
+
* separate from its parent (if the parent is also interactive).
|
|
3380
|
+
*/
|
|
3381
|
+
function isElementDistinctInteraction(element, nodeData) {
|
|
3382
|
+
if (nodeData.isScrollable) {
|
|
3383
|
+
return true;
|
|
3384
|
+
}
|
|
3385
|
+
if (!element || element.nodeType !== Node.ELEMENT_NODE) {
|
|
3386
|
+
return false;
|
|
3387
|
+
}
|
|
3388
|
+
const tagName = element.tagName.toLowerCase();
|
|
3389
|
+
const role = element.getAttribute('role');
|
|
3390
|
+
// Check if it's an iframe - always distinct boundary
|
|
3391
|
+
if (tagName === 'iframe') {
|
|
3392
|
+
return true;
|
|
3393
|
+
}
|
|
3394
|
+
// Check tag name
|
|
3395
|
+
if (DISTINCT_INTERACTIVE_TAGS.has(tagName)) {
|
|
3396
|
+
return true;
|
|
3397
|
+
}
|
|
3398
|
+
// Check interactive roles
|
|
3399
|
+
if (role && INTERACTIVE_ROLES.has(role)) {
|
|
3400
|
+
return true;
|
|
3401
|
+
}
|
|
3402
|
+
// Check contenteditable
|
|
3403
|
+
if (element.isContentEditable || element.getAttribute('contenteditable') === 'true') {
|
|
3404
|
+
return true;
|
|
3405
|
+
}
|
|
3406
|
+
// Check for common testing/automation attributes
|
|
3407
|
+
if (element.hasAttribute('data-testid') || element.hasAttribute('data-cy') || element.hasAttribute('data-test')) {
|
|
3408
|
+
return true;
|
|
3409
|
+
}
|
|
3410
|
+
// Check for explicit onclick handler (attribute or property)
|
|
3411
|
+
if (element.hasAttribute('onclick') || typeof element.onclick === 'function') {
|
|
3412
|
+
return true;
|
|
3413
|
+
}
|
|
3414
|
+
if (element.hasAttribute('aria-haspopup')) {
|
|
3415
|
+
return true;
|
|
3416
|
+
}
|
|
3417
|
+
if (hasAnyClassName(element, interactiveClassNames)) {
|
|
3418
|
+
return true;
|
|
3419
|
+
}
|
|
3420
|
+
// Check for cursor-pointer class names that indicate clickability
|
|
3421
|
+
if (hasAnyClassName(element, cursorPointerClassNames)) {
|
|
3422
|
+
return true;
|
|
3423
|
+
}
|
|
3424
|
+
// return false
|
|
3425
|
+
// Check for other common interaction event listeners
|
|
3426
|
+
try {
|
|
3427
|
+
const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
|
|
3428
|
+
if (typeof getEventListenersForNode === 'function') {
|
|
3429
|
+
const listeners = getEventListenersForNode(element);
|
|
3430
|
+
const interactionEvents = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave', 'keydown', 'keyup', 'submit', 'change', 'focus', 'blur'];
|
|
3431
|
+
for (const eventType of interactionEvents) {
|
|
3432
|
+
for (const listener of listeners) {
|
|
3433
|
+
if (listener.type === eventType) {
|
|
3434
|
+
return true; // Found a common interaction listener
|
|
3435
|
+
}
|
|
3436
|
+
}
|
|
3437
|
+
}
|
|
3438
|
+
}
|
|
3439
|
+
// Fallback: Check common event attributes if getEventListeners is not available (getEventListenersForNode doesn't work in page.evaluate context)
|
|
3440
|
+
const commonEventAttrs = ['onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', 'onsubmit', 'onmouseenter', 'onmouseleave', 'onchange', 'oninput', 'onfocus', 'onblur'];
|
|
3441
|
+
if (commonEventAttrs.some(attr => element.hasAttribute(attr))) {
|
|
3442
|
+
return true;
|
|
3443
|
+
}
|
|
3444
|
+
}
|
|
3445
|
+
catch (e) {
|
|
3446
|
+
// console.warn(\`Could not check event listeners for \${element.tagName}:\`, e);
|
|
3447
|
+
// If checking listeners fails, rely on other checks
|
|
3448
|
+
}
|
|
3449
|
+
// if the element is not strictly interactive but appears clickable based on heuristic signals
|
|
3450
|
+
if (isHeuristicallyInteractive(element)) {
|
|
3451
|
+
return true;
|
|
3452
|
+
}
|
|
3453
|
+
// Default to false: if it's interactive but doesn't match above,
|
|
3454
|
+
// assume it triggers the same action as the parent.
|
|
3455
|
+
return false;
|
|
3456
|
+
}
|
|
3457
|
+
// --- End distinct interaction check ---
|
|
3458
|
+
/**
|
|
3459
|
+
* Calculates Intersection over Union (IoU) for two rectangles.
|
|
3460
|
+
* Returns a value between 0 (no overlap) and 1 (identical).
|
|
3461
|
+
*/
|
|
3462
|
+
function calculateIoU(rect1, rect2) {
|
|
3463
|
+
// Calculate intersection
|
|
3464
|
+
const xOverlap = Math.max(0, Math.min(rect1.right, rect2.right) - Math.max(rect1.left, rect2.left));
|
|
3465
|
+
const yOverlap = Math.max(0, Math.min(rect1.bottom, rect2.bottom) - Math.max(rect1.top, rect2.top));
|
|
3466
|
+
const intersectionArea = xOverlap * yOverlap;
|
|
3467
|
+
// Calculate union
|
|
3468
|
+
const area1 = rect1.width * rect1.height;
|
|
3469
|
+
const area2 = rect2.width * rect2.height;
|
|
3470
|
+
const unionArea = area1 + area2 - intersectionArea;
|
|
3471
|
+
// Avoid division by zero
|
|
3472
|
+
if (unionArea === 0)
|
|
3473
|
+
return 0;
|
|
3474
|
+
return intersectionArea / unionArea;
|
|
3475
|
+
}
|
|
3476
|
+
/**
|
|
3477
|
+
* Checks if two rects are effectively the same using IoU threshold.
|
|
3478
|
+
*/
|
|
3479
|
+
function areRectsEqual(rect1, rect2) {
|
|
3480
|
+
return calculateIoU(rect1, rect2) >= sameRectIoUThreshold;
|
|
3481
|
+
}
|
|
3482
|
+
/**
|
|
3483
|
+
* Checks if an element can actually receive pointer events.
|
|
3484
|
+
* Returns false if the element is hidden, disabled, or has pointer-events: none.
|
|
3485
|
+
*/
|
|
3486
|
+
function canReceivePointerEvents(element) {
|
|
3487
|
+
const style = getCachedComputedStyle(element);
|
|
3488
|
+
// Check CSS properties that prevent events
|
|
3489
|
+
if (style?.pointerEvents === 'none')
|
|
3490
|
+
return false;
|
|
3491
|
+
if (style?.visibility === 'hidden')
|
|
3492
|
+
return false;
|
|
3493
|
+
if (style?.display === 'none')
|
|
3494
|
+
return false;
|
|
3495
|
+
// Check disabled attribute for form elements
|
|
3496
|
+
if (element.disabled)
|
|
3497
|
+
return false;
|
|
3498
|
+
return true;
|
|
3499
|
+
}
|
|
3500
|
+
/**
|
|
3501
|
+
* Checks if an element has an interactive descendant with the same bounding rect.
|
|
3502
|
+
* If so, the descendant should be highlighted instead of this element (innermost wins).
|
|
3503
|
+
*/
|
|
3504
|
+
function hasInteractiveDescendantWithSameRect(element, rect) {
|
|
3505
|
+
for (const child of element.children) {
|
|
3506
|
+
if (!(child instanceof HTMLElement))
|
|
3507
|
+
continue;
|
|
3508
|
+
if (!canReceivePointerEvents(child))
|
|
3509
|
+
continue;
|
|
3510
|
+
const childRect = child.getBoundingClientRect();
|
|
3511
|
+
if (!areRectsEqual(rect, childRect))
|
|
3512
|
+
continue;
|
|
3513
|
+
// Child has same rect - check if it's interactive
|
|
3514
|
+
if (isInteractiveElement(child) || isElementScrollable(child)) {
|
|
3515
|
+
return true;
|
|
3516
|
+
}
|
|
3517
|
+
// Child has same rect but isn't interactive - check its descendants
|
|
3518
|
+
if (hasInteractiveDescendantWithSameRect(child, rect)) {
|
|
3519
|
+
return true;
|
|
3520
|
+
}
|
|
3521
|
+
}
|
|
3522
|
+
return false;
|
|
3523
|
+
}
|
|
3524
|
+
/**
|
|
3525
|
+
* Handles the logic for deciding whether to highlight an element and performing the highlight.
|
|
3526
|
+
*/
|
|
3527
|
+
function handleHighlighting(nodeData, node, parentIframe, parentHighlightedRect) {
|
|
3528
|
+
if (!nodeData.isInteractive)
|
|
3529
|
+
return { highlighted: false, rect: null }; // Not interactive, definitely don't highlight
|
|
3530
|
+
// Apply action intent filter - skip elements that don't match the specified intent
|
|
3531
|
+
const actionIntent = args.actionIntent || 'all';
|
|
3532
|
+
if (actionIntent !== 'all' && !matchesActionIntent(node, actionIntent)) {
|
|
3533
|
+
return { highlighted: false, rect: null };
|
|
3534
|
+
}
|
|
3535
|
+
// Check if element can actually receive pointer events
|
|
3536
|
+
if (!canReceivePointerEvents(node)) {
|
|
3537
|
+
return { highlighted: false, rect: null };
|
|
3538
|
+
}
|
|
3539
|
+
const currentRect = node.getBoundingClientRect();
|
|
3540
|
+
// Check if there's an interactive descendant with the same rect
|
|
3541
|
+
// If so, skip this element - let the innermost element be highlighted
|
|
3542
|
+
if (hasInteractiveDescendantWithSameRect(node, currentRect)) {
|
|
3543
|
+
return { highlighted: false, rect: null };
|
|
3544
|
+
}
|
|
3545
|
+
let shouldHighlight = false;
|
|
3546
|
+
if (!parentHighlightedRect) {
|
|
3547
|
+
// Parent wasn't highlighted, this interactive node can be highlighted.
|
|
3548
|
+
shouldHighlight = true;
|
|
3549
|
+
}
|
|
3550
|
+
else {
|
|
3551
|
+
// Parent *was* highlighted. Only highlight this node if it represents a distinct interaction.
|
|
3552
|
+
if (areRectsEqual(currentRect, parentHighlightedRect)) {
|
|
3553
|
+
// Same rect as parent - this is the innermost element, should be highlighted
|
|
3554
|
+
// (parent should have been skipped by hasInteractiveDescendantWithSameRect)
|
|
3555
|
+
shouldHighlight = true;
|
|
3556
|
+
}
|
|
3557
|
+
else if (isElementDistinctInteraction(node, nodeData)) {
|
|
3558
|
+
shouldHighlight = true;
|
|
3559
|
+
}
|
|
3560
|
+
else {
|
|
3561
|
+
// console.log(\`Skipping highlight for \${nodeData.tagName} (parent highlighted)\`);
|
|
3562
|
+
shouldHighlight = false;
|
|
3563
|
+
}
|
|
3564
|
+
}
|
|
3565
|
+
if (shouldHighlight) {
|
|
3566
|
+
const attributeNames = node.getAttributeNames?.() || [];
|
|
3567
|
+
for (const name of attributeNames) {
|
|
3568
|
+
const value = node.getAttribute(name);
|
|
3569
|
+
nodeData.attributes[name] = value;
|
|
3570
|
+
}
|
|
3571
|
+
// Check viewport status before assigning index and highlighting
|
|
3572
|
+
if (nodeData.isInViewport === undefined) {
|
|
3573
|
+
nodeData.isInViewport = isInExpandedViewport(node, viewportExpansion);
|
|
3574
|
+
}
|
|
3575
|
+
// When viewportExpansion is -1, all interactive elements should get a highlight index
|
|
3576
|
+
// regardless of viewport status
|
|
3577
|
+
if (nodeData.isInViewport || viewportExpansion === -1) {
|
|
3578
|
+
nodeData.highlightIndex = highlightIndex++;
|
|
3579
|
+
if (doHighlightElements) {
|
|
3580
|
+
// Collect elements for deferred highlighting (after tree-based flow detection)
|
|
3581
|
+
if (focusHighlightIndex >= 0) {
|
|
3582
|
+
if (focusHighlightIndex === nodeData.highlightIndex) {
|
|
3583
|
+
elementsToHighlight.push({ element: node, index: nodeData.highlightIndex, parentIframe });
|
|
3584
|
+
}
|
|
3585
|
+
}
|
|
3586
|
+
else {
|
|
3587
|
+
elementsToHighlight.push({ element: node, index: nodeData.highlightIndex, parentIframe });
|
|
3588
|
+
}
|
|
3589
|
+
return { highlighted: true, rect: currentRect }; // Will be highlighted after traversal
|
|
3590
|
+
}
|
|
3591
|
+
// Even if not drawing highlights, we still "highlighted" for tracking purposes
|
|
3592
|
+
return { highlighted: true, rect: currentRect };
|
|
3593
|
+
}
|
|
3594
|
+
else {
|
|
3595
|
+
// console.log(\`Skipping highlight for \${nodeData.tagName} (outside viewport)\`);
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
return { highlighted: false, rect: null }; // Did not highlight
|
|
3599
|
+
}
|
|
3600
|
+
function isElementScrollable(element) {
|
|
3601
|
+
const listenedEvents = getCachedNodeEventListeners(element);
|
|
3602
|
+
if (listenedEvents && listenedEvents.includes('scroll')) {
|
|
3603
|
+
const hasScrollableX = element.scrollWidth > element.clientWidth;
|
|
3604
|
+
const hasScrollableY = element.scrollHeight > element.clientHeight;
|
|
3605
|
+
return hasScrollableX || hasScrollableY;
|
|
3606
|
+
}
|
|
3607
|
+
const style = getCachedComputedStyle(element);
|
|
3608
|
+
const hasScrollableX = ['auto', 'scroll'].includes(style?.overflowX || '') &&
|
|
3609
|
+
element.scrollWidth > element.clientWidth;
|
|
3610
|
+
const hasScrollableY = ['auto', 'scroll'].includes(style?.overflowY || '') &&
|
|
3611
|
+
element.scrollHeight > element.clientHeight;
|
|
3612
|
+
return hasScrollableX || hasScrollableY;
|
|
3613
|
+
}
|
|
3614
|
+
/**
|
|
3615
|
+
* Creates a node data object for a given node and its descendants.
|
|
3616
|
+
*/
|
|
3617
|
+
function buildDomTree(node, parentIframe = null, parentHighlightedRect = null) {
|
|
3618
|
+
// Fast rejection checks first
|
|
3619
|
+
if (!node || node.id === HIGHLIGHT_CONTAINER_ID) {
|
|
3620
|
+
return null;
|
|
3621
|
+
}
|
|
3622
|
+
if (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE) {
|
|
3623
|
+
return null;
|
|
3624
|
+
}
|
|
3625
|
+
// Special handling for root node (body)
|
|
3626
|
+
if (node === document.body) {
|
|
3627
|
+
const nodeData = {
|
|
3628
|
+
tagName: 'body',
|
|
3629
|
+
attributes: {},
|
|
3630
|
+
xpath: '/body',
|
|
3631
|
+
children: [],
|
|
3632
|
+
};
|
|
3633
|
+
// Process children of body
|
|
3634
|
+
for (const child of node.childNodes) {
|
|
3635
|
+
const domElement = buildDomTree(child, parentIframe, null); // Body's children have no highlighted parent initially
|
|
3636
|
+
if (domElement)
|
|
3637
|
+
nodeData.children.push(domElement);
|
|
3638
|
+
}
|
|
3639
|
+
const id = \`\${ID.current++}\`;
|
|
3640
|
+
DOM_HASH_MAP[id] = nodeData;
|
|
3641
|
+
return id;
|
|
3642
|
+
}
|
|
3643
|
+
// Process text nodes
|
|
3644
|
+
if (node.nodeType === Node.TEXT_NODE) {
|
|
3645
|
+
const textContent = node.textContent?.trim();
|
|
3646
|
+
if (!textContent) {
|
|
3647
|
+
return null;
|
|
3648
|
+
}
|
|
3649
|
+
// Only check visibility for text nodes that might be visible
|
|
3650
|
+
const parentElement = node.parentElement;
|
|
3651
|
+
if (!parentElement || parentElement.tagName.toLowerCase() === 'script') {
|
|
3652
|
+
return null;
|
|
3653
|
+
}
|
|
3654
|
+
const id = \`\${ID.current++}\`;
|
|
3655
|
+
DOM_HASH_MAP[id] = {
|
|
3656
|
+
type: "TEXT_NODE",
|
|
3657
|
+
text: textContent,
|
|
3658
|
+
isVisible: isTextNodeVisible(node),
|
|
3659
|
+
};
|
|
3660
|
+
return id;
|
|
3661
|
+
}
|
|
3662
|
+
// Quick checks for element nodes
|
|
3663
|
+
if (node.nodeType === Node.ELEMENT_NODE && !isElementAccepted(node)) {
|
|
3664
|
+
return null;
|
|
3665
|
+
}
|
|
3666
|
+
const element = node;
|
|
3667
|
+
const nodeData = {
|
|
3668
|
+
tagName: element.tagName.toLowerCase(),
|
|
3669
|
+
attributes: {},
|
|
3670
|
+
xpath: getXPathTree(element, true),
|
|
3671
|
+
children: [],
|
|
3672
|
+
};
|
|
3673
|
+
// Get attributes for interactive elements or potential text containers
|
|
3674
|
+
if (element.tagName.toLowerCase() === 'iframe' || element.tagName.toLowerCase() === 'body') {
|
|
3675
|
+
const attributeNames = element.getAttributeNames?.() || [];
|
|
3676
|
+
for (const name of attributeNames) {
|
|
3677
|
+
const value = element.getAttribute(name);
|
|
3678
|
+
nodeData.attributes[name] = value;
|
|
3679
|
+
}
|
|
3680
|
+
}
|
|
3681
|
+
let highlightResult = { highlighted: false, rect: null };
|
|
3682
|
+
// Perform visibility, interactivity, and highlighting checks
|
|
3683
|
+
if (node.nodeType === Node.ELEMENT_NODE) {
|
|
3684
|
+
if (alwaysHighlightFileInput && element.tagName.toLowerCase() === 'input' && element.type === 'file') {
|
|
3685
|
+
nodeData.isTopElement = true;
|
|
3686
|
+
if (nodeData.isTopElement) {
|
|
3687
|
+
nodeData.isInteractive = true;
|
|
3688
|
+
nodeData.isInViewport = true; // File inputs should always be considered in viewport
|
|
3689
|
+
// Call the dedicated highlighting function
|
|
3690
|
+
highlightResult = handleHighlighting(nodeData, element, parentIframe, parentHighlightedRect);
|
|
3691
|
+
}
|
|
3692
|
+
}
|
|
3693
|
+
else {
|
|
3694
|
+
nodeData.isVisible = isElementVisible(element); // isElementVisible uses offsetWidth/Height, which is fine
|
|
3695
|
+
if (nodeData.isVisible) {
|
|
3696
|
+
nodeData.isTopElement = isTopElement(element);
|
|
3697
|
+
if (nodeData.isTopElement) {
|
|
3698
|
+
let isScrollable = isElementScrollable(element);
|
|
3699
|
+
nodeData.isInteractive = isInteractiveElement(element) || isScrollable;
|
|
3700
|
+
nodeData.isScrollable = isScrollable;
|
|
3701
|
+
nodeData.markAsClickable = shouldMarkAsClickable(element);
|
|
3702
|
+
// Call the dedicated highlighting function
|
|
3703
|
+
highlightResult = handleHighlighting(nodeData, element, parentIframe, parentHighlightedRect);
|
|
3704
|
+
}
|
|
3705
|
+
}
|
|
3706
|
+
}
|
|
3707
|
+
}
|
|
3708
|
+
// Determine the rect to pass to children: use this element's rect if highlighted, otherwise parent's
|
|
3709
|
+
const rectForChildren = highlightResult.highlighted ? highlightResult.rect : parentHighlightedRect;
|
|
3710
|
+
// Process children, with special handling for iframes and rich text editors
|
|
3711
|
+
if (element.tagName) {
|
|
3712
|
+
const tagName = element.tagName.toLowerCase();
|
|
3713
|
+
// Handle iframes
|
|
3714
|
+
if (tagName === "iframe") {
|
|
3715
|
+
try {
|
|
3716
|
+
const iframeEl = element;
|
|
3717
|
+
const iframeDoc = iframeEl.contentDocument || iframeEl.contentWindow?.document;
|
|
3718
|
+
if (iframeDoc) {
|
|
3719
|
+
for (const child of iframeDoc.childNodes) {
|
|
3720
|
+
const domElement = buildDomTree(child, iframeEl, null); // iframes start fresh
|
|
3721
|
+
if (domElement)
|
|
3722
|
+
nodeData.children.push(domElement);
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
3726
|
+
catch (e) {
|
|
3727
|
+
console.warn("Unable to access iframe:", e);
|
|
3728
|
+
}
|
|
3729
|
+
}
|
|
3730
|
+
// Handle rich text editors and contenteditable elements
|
|
3731
|
+
else if (element.isContentEditable ||
|
|
3732
|
+
element.getAttribute("contenteditable") === "true" ||
|
|
3733
|
+
element.id === "tinymce" ||
|
|
3734
|
+
element.classList.contains("mce-content-body") ||
|
|
3735
|
+
(tagName === "body" && element.getAttribute("data-id")?.startsWith("mce_"))) {
|
|
3736
|
+
// Process all child nodes to capture formatted text
|
|
3737
|
+
for (const child of element.childNodes) {
|
|
3738
|
+
const domElement = buildDomTree(child, parentIframe, rectForChildren);
|
|
3739
|
+
if (domElement)
|
|
3740
|
+
nodeData.children.push(domElement);
|
|
3741
|
+
}
|
|
3742
|
+
}
|
|
3743
|
+
else {
|
|
3744
|
+
// Handle shadow DOM
|
|
3745
|
+
if (element.shadowRoot) {
|
|
3746
|
+
nodeData.shadowRoot = true;
|
|
3747
|
+
for (const child of element.shadowRoot.childNodes) {
|
|
3748
|
+
const domElement = buildDomTree(child, parentIframe, rectForChildren);
|
|
3749
|
+
if (domElement)
|
|
3750
|
+
nodeData.children.push(domElement);
|
|
3751
|
+
}
|
|
3752
|
+
}
|
|
3753
|
+
// Handle regular elements
|
|
3754
|
+
for (const child of element.childNodes) {
|
|
3755
|
+
const domElement = buildDomTree(child, parentIframe, rectForChildren);
|
|
3756
|
+
if (domElement)
|
|
3757
|
+
nodeData.children.push(domElement);
|
|
3758
|
+
}
|
|
3759
|
+
}
|
|
3760
|
+
}
|
|
3761
|
+
const id = \`\${ID.current++}\`;
|
|
3762
|
+
DOM_HASH_MAP[id] = nodeData;
|
|
3763
|
+
return id;
|
|
3764
|
+
}
|
|
3765
|
+
const rootId = buildDomTree(document.body);
|
|
3766
|
+
// After traversal, render all highlights
|
|
3767
|
+
let treeEntries = [];
|
|
3768
|
+
if (doHighlightElements) {
|
|
3769
|
+
if (phase === 'labels') {
|
|
3770
|
+
// Labels phase: use provided element data to place labels only
|
|
3771
|
+
processLabelsPhase();
|
|
3772
|
+
}
|
|
3773
|
+
else if (elementsToHighlight.length > 0) {
|
|
3774
|
+
// Boxes phase or legacy mode: traverse tree to draw boxes and optionally labels
|
|
3775
|
+
treeEntries = buildTreeEntries();
|
|
3776
|
+
// Use recursive tree processing for correct hot map behavior:
|
|
3777
|
+
// - Pre-order: Mark bounding boxes in hot map (so children avoid parent borders)
|
|
3778
|
+
// - Post-order: Place labels (so parent labels avoid children's labels, skipped in boxes phase)
|
|
3779
|
+
processElementTreeRecursively(elementsToHighlight);
|
|
3780
|
+
}
|
|
3781
|
+
}
|
|
3782
|
+
// Clear the cache before returning
|
|
3783
|
+
DOM_CACHE.clearCache();
|
|
3784
|
+
return {
|
|
3785
|
+
rootId,
|
|
3786
|
+
map: DOM_HASH_MAP,
|
|
3787
|
+
debugLogs,
|
|
3788
|
+
treeEntries,
|
|
3789
|
+
highlightCount: highlightIndex,
|
|
3790
|
+
// Return the final grayscale image state (after all boxes and labels marked) for debugging
|
|
3791
|
+
finalGrayscaleImage: grayscaleImage ?? undefined,
|
|
3792
|
+
// Note: labelSnapshots are now streamed via onSnapshot callback instead of returned
|
|
3793
|
+
// Return element data from boxes phase for use in labels phase
|
|
3794
|
+
elementData: phase === 'boxes' ? collectedElementData : undefined,
|
|
3795
|
+
};
|
|
3796
|
+
})`;function te(e){return e==="about:blank"||e==="chrome://newtab/"||e==="edge://newtab/"||e==="about:newtab"}var Me=class{constructor(e={}){h.debug("\u{1F333} Initializing DomService with options:",e),this.useDomTreeTs=e.useDomTreeTs??!1,this.jsCode=this.useDomTreeTs?ee:Z}async getClickableElements(e,i={}){let{highlightElements:o=!0,focusElement:l=-1,viewportExpansion:t=0,interactiveClassNames:r=[],alwaysHighlightFileInput:n=!1,sameRectIoUThreshold:d,actionIntent:c="all"}=i,[u,a]=await this.buildDomTree(e,o,l,t,r,n,d,c);return{elementTree:u,selectorMap:a}}async getClickableElementsWithScreenshot(e,i={}){if(i.useAccessibilityTree)return this.getClickableElementsWithAXTree(e,i);let o;i.useCleanScreenshot&&(o=await e.screenshot({type:"png",fullPage:!1}));let l=await this.getClickableElements(e,i);await e.waitForTimeout(100),i.useCleanScreenshot||(o=await e.screenshot({type:"png",fullPage:!1})),await this.removeHighlights(e);let t=o.toString("base64"),r;if(i.useSlicedScreenshots)try{r=(await D(o,{resize:i.resizeSlicedScreenshots})).map(d=>d.toString("base64"))}catch(n){h.warn("Failed to slice screenshot:",n)}return{domState:l,screenshotBase64:t,slicedScreenshotsBase64:r}}async getClickableElementsWithAXTree(e,i={}){h.debug("\u{1F333} Using CDP Accessibility Tree for element detection");let o;i.useCleanScreenshot&&(o=await e.screenshot({type:"png",fullPage:!1}));let l=await e.context().newCDPSession(e),{nodes:t}=await l.send("Accessibility.getFullAXTree",{depth:-1});h.debug(`\u{1F4CA} Got ${t.length} AXNodes from accessibility tree`);let r=t.filter(s=>{if(s.ignored)return!1;let f=s.role?.value;return!(!f||!X.has(f)||s.properties?.find(y=>y.name==="disabled")?.value?.value||!s.backendDOMNodeId)});h.debug(`\u2705 Found ${r.length} interactive elements from AXTree`);let n=t.filter(s=>s.role?.value==="button");h.debug(`\u{1F518} Total buttons in AXTree: ${n.length}`);for(let s of n){let f=[];s.ignored&&f.push("ignored"),s.backendDOMNodeId||f.push("no-backendDOMNodeId"),s.properties?.find(y=>y.name==="disabled")?.value?.value&&f.push("disabled"),h.debug(` - "${s.name?.value||"(no name)"}" ${f.length>0?`[SKIPPED: ${f.join(", ")}]`:"[INCLUDED]"}`)}let d=new Set(r.map(s=>s.backendDOMNodeId)),c=await this.getElementsWithEventListeners(l,i.eventListenerLimit??500),u=0;for(let s of c)d.has(s.backendNodeId)||(r.push({nodeId:`synthetic-${s.backendNodeId}`,ignored:!1,backendDOMNodeId:s.backendNodeId,role:{type:"role",value:"generic"},name:{type:"string",value:""},properties:[{name:"eventListeners",value:{type:"string",value:s.eventTypes.join(",")}}]}),d.add(s.backendNodeId),u++);h.debug(`\u{1F3AF} Added ${u} elements from event listeners (total: ${r.length})`);let a=await this.resolveAXNodesToDOM(l,r),m=a.filter(s=>s.isVisible&&s.isInViewport&&s.isTopElement&&s.boundingRect),w=a.filter(s=>!(s.isVisible&&s.isInViewport&&s.isTopElement&&s.boundingRect));if(w.length>0){h.debug(`\u{1F6AB} Filtered out ${w.length} elements:`);for(let s of w){let f=[];s.isVisible||f.push("not-visible"),s.isInViewport||f.push("not-in-viewport"),s.isTopElement||f.push("not-top-element"),s.boundingRect||f.push("no-bounding-rect"),h.debug(` - <${s.tagName}> "${s.axNode.name?.value||""}" [${f.join(", ")}]`)}}h.debug(`\u{1F441}\uFE0F ${m.length} elements are visible and in viewport`);let{domState:E,highlightIndex:p}=await this.buildDomStateFromAXTree(m);i.highlightElements!==!1&&p>0&&(await this.renderHighlightsForAXElements(e,m.slice(0,p)),await e.waitForTimeout(100)),i.useCleanScreenshot||(o=await e.screenshot({type:"png",fullPage:!1})),await this.removeHighlights(e);try{await l.detach()}catch{}let b=o.toString("base64"),g;if(i.useSlicedScreenshots)try{g=(await D(o,{resize:i.resizeSlicedScreenshots})).map(f=>f.toString("base64"))}catch(s){h.warn("Failed to slice screenshot:",s)}return{domState:E,screenshotBase64:b,slicedScreenshotsBase64:g}}async resolveAXNodesToDOM(e,i){let o=[],l=i.map(r=>r.backendDOMNodeId).filter(r=>r!==void 0);if(l.length===0)return o;let t=[];for(let r of l)try{let{object:n}=await e.send("DOM.resolveNode",{backendNodeId:r});t.push(n.objectId||null)}catch{t.push(null)}for(let r=0;r<i.length;r++){let n=i[r],d=t[r];if(d)try{let{result:c}=await e.send("Runtime.callFunctionOn",{objectId:d,functionDeclaration:`function() {
|
|
3797
|
+
const el = this;
|
|
3798
|
+
const rect = el.getBoundingClientRect();
|
|
3799
|
+
const rects = el.getClientRects();
|
|
3800
|
+
const style = window.getComputedStyle(el);
|
|
3801
|
+
|
|
3802
|
+
// Check visibility
|
|
3803
|
+
const isVisible =
|
|
3804
|
+
el.offsetWidth > 0 &&
|
|
3805
|
+
el.offsetHeight > 0 &&
|
|
3806
|
+
style.visibility !== 'hidden' &&
|
|
3807
|
+
style.display !== 'none' &&
|
|
3808
|
+
style.opacity !== '0';
|
|
3809
|
+
|
|
3810
|
+
// Check if in viewport
|
|
3811
|
+
const viewportWidth = window.innerWidth;
|
|
3812
|
+
const viewportHeight = window.innerHeight;
|
|
3813
|
+
const isInViewport = !(
|
|
3814
|
+
rect.bottom < 0 ||
|
|
3815
|
+
rect.top > viewportHeight ||
|
|
3816
|
+
rect.right < 0 ||
|
|
3817
|
+
rect.left > viewportWidth
|
|
3818
|
+
);
|
|
3819
|
+
|
|
3820
|
+
// Check if topmost element
|
|
3821
|
+
let isTopElement = false;
|
|
3822
|
+
if (isVisible && isInViewport) {
|
|
3823
|
+
const centerX = rect.left + rect.width / 2;
|
|
3824
|
+
const centerY = rect.top + rect.height / 2;
|
|
3825
|
+
const topEl = document.elementFromPoint(centerX, centerY);
|
|
3826
|
+
if (topEl) {
|
|
3827
|
+
let current = topEl;
|
|
3828
|
+
while (current && current !== document.documentElement) {
|
|
3829
|
+
if (current === el) {
|
|
3830
|
+
isTopElement = true;
|
|
3831
|
+
break;
|
|
3832
|
+
}
|
|
3833
|
+
current = current.parentElement;
|
|
3834
|
+
}
|
|
3835
|
+
}
|
|
3836
|
+
}
|
|
3837
|
+
|
|
3838
|
+
// Get XPath
|
|
3839
|
+
function getXPath(element) {
|
|
3840
|
+
const segments = [];
|
|
3841
|
+
let current = element;
|
|
3842
|
+
while (current && current.nodeType === Node.ELEMENT_NODE) {
|
|
3843
|
+
const tagName = current.nodeName.toLowerCase();
|
|
3844
|
+
const siblings = current.parentElement
|
|
3845
|
+
? Array.from(current.parentElement.children).filter(c => c.nodeName.toLowerCase() === tagName)
|
|
3846
|
+
: [];
|
|
3847
|
+
const index = siblings.length > 1 ? siblings.indexOf(current) + 1 : 0;
|
|
3848
|
+
segments.unshift(index > 0 ? tagName + '[' + index + ']' : tagName);
|
|
3849
|
+
current = current.parentNode;
|
|
3850
|
+
}
|
|
3851
|
+
return segments.join('/');
|
|
3852
|
+
}
|
|
3853
|
+
|
|
3854
|
+
// Get attributes
|
|
3855
|
+
const attributes = {};
|
|
3856
|
+
for (const attr of el.attributes) {
|
|
3857
|
+
attributes[attr.name] = attr.value;
|
|
3858
|
+
}
|
|
3859
|
+
|
|
3860
|
+
// Convert client rects to plain objects
|
|
3861
|
+
const clientRectsArray = [];
|
|
3862
|
+
for (const r of rects) {
|
|
3863
|
+
clientRectsArray.push({
|
|
3864
|
+
x: r.x, y: r.y, width: r.width, height: r.height
|
|
3865
|
+
});
|
|
3866
|
+
}
|
|
3867
|
+
|
|
3868
|
+
return {
|
|
3869
|
+
tagName: el.tagName.toLowerCase(),
|
|
3870
|
+
xpath: getXPath(el),
|
|
3871
|
+
attributes: attributes,
|
|
3872
|
+
isVisible: isVisible,
|
|
3873
|
+
isInViewport: isInViewport,
|
|
3874
|
+
isTopElement: isTopElement,
|
|
3875
|
+
boundingRect: rect.width > 0 && rect.height > 0 ? {
|
|
3876
|
+
x: rect.x, y: rect.y, width: rect.width, height: rect.height
|
|
3877
|
+
} : null,
|
|
3878
|
+
clientRects: clientRectsArray
|
|
3879
|
+
};
|
|
3880
|
+
}`,returnByValue:!0});c.value&&o.push({axNode:n,...c.value})}catch(c){h.debug(`Failed to resolve element: ${c}`)}}return o}async getElementsWithEventListeners(e,i=G){let o=[];try{let{root:l}=await e.send("DOM.getDocument",{depth:0}),t=B.join(","),{nodeIds:r}=await e.send("DOM.querySelectorAll",{nodeId:l.nodeId,selector:t});h.debug(`\u{1F50D} Checking ${Math.min(r.length,i)} elements for event listeners`);for(let n of r.slice(0,i))try{let{object:d}=await e.send("DOM.resolveNode",{nodeId:n});if(!d.objectId)continue;let{listeners:c}=await e.send("DOMDebugger.getEventListeners",{objectId:d.objectId}),u=c.filter(a=>$.has(a.type));if(u.length>0){let{node:a}=await e.send("DOM.describeNode",{nodeId:n});o.push({backendNodeId:a.backendNodeId,eventTypes:u.map(m=>m.type)})}await e.send("Runtime.releaseObject",{objectId:d.objectId})}catch{}h.debug(`\u2705 Found ${o.length} elements with interaction event listeners`)}catch(l){h.warn("Failed to get elements with event listeners:",l)}return o}async buildDomStateFromAXTree(e){let i=new Map,o=new C("body","/body",{},[],!0,!1,!1,!1,!0,!0,!1,null),l=0;for(let t of e){let r=t.axNode.role?.value||"",n=t.axNode.name?.value||"",d=["button","link","menuitem","tab","switch"].includes(r),c=new C(t.tagName,t.xpath,t.attributes,[],t.isVisible,!0,r==="scrollbar",d,t.isTopElement,t.isInViewport,!1,l,t.boundingRect?{topLeft:{x:t.boundingRect.x,y:t.boundingRect.y},topRight:{x:t.boundingRect.x+t.boundingRect.width,y:t.boundingRect.y},bottomLeft:{x:t.boundingRect.x,y:t.boundingRect.y+t.boundingRect.height},bottomRight:{x:t.boundingRect.x+t.boundingRect.width,y:t.boundingRect.y+t.boundingRect.height},center:{x:t.boundingRect.x+t.boundingRect.width/2,y:t.boundingRect.y+t.boundingRect.height/2},width:t.boundingRect.width,height:t.boundingRect.height}:null,null,null,o);if(n){let u=new N(n,!0,c);c.children.push(u)}o.children.push(c),i.set(l,c),l++}return{domState:{elementTree:o,selectorMap:i},highlightIndex:l}}async renderHighlightsForAXElements(e,i){let o=["#FF0000","#00FF00","#0000FF","#FFA500","#800080","#008080","#FF69B4","#4B0082","#FF4500","#2E8B57","#DC143C","#4682B4"];await e.evaluate(({elements:l,colors:t})=>{let r="playwright-highlight-container",n=document.getElementById(r);n||(n=document.createElement("div"),n.id=r,n.style.position="fixed",n.style.pointerEvents="none",n.style.top="0",n.style.left="0",n.style.width="100%",n.style.height="100%",n.style.zIndex="2147483647",n.style.backgroundColor="transparent",document.body.appendChild(n)),l.forEach((d,c)=>{if(!d.boundingRect)return;let u=t[c%t.length],a=d.boundingRect,m=document.createElement("div");m.style.position="fixed",m.style.border=`1px solid ${u}`,m.style.backgroundColor="transparent",m.style.pointerEvents="none",m.style.boxSizing="border-box",m.style.top=`${a.y}px`,m.style.left=`${a.x}px`,m.style.width=`${a.width}px`,m.style.height=`${a.height}px`,n.appendChild(m);let w=document.createElement("div");w.style.position="fixed",w.style.background=u,w.style.color="white",w.style.padding="1px 4px",w.style.borderRadius="4px",w.style.fontSize=c>=100?"8px":"12px",w.textContent=String(c);let E=Math.max(0,a.y-16),p=Math.max(0,Math.min(a.x+a.width-20,window.innerWidth-25));w.style.top=`${E}px`,w.style.left=`${p}px`,n.appendChild(w)})},{elements:i,colors:o})}async removeHighlights(e){try{await e.evaluate(()=>{let i=document.getElementById("playwright-highlight-container");i&&i.remove(),window._highlightCleanupFunctions&&(window._highlightCleanupFunctions.forEach(o=>o()),window._highlightCleanupFunctions=[])}),h.debug("\u2705 Highlights removed from page")}catch(i){h.warn("Failed to remove highlights:",i.message)}}async getCrossOriginIframes(e){let i=await e.locator("iframe").filter({hasNot:e.locator(":visible")}).evaluateAll(d=>d.map(c=>c.src)),o=d=>{try{let c=new URL(d);return["doubleclick.net","adroll.com","googletagmanager.com"].some(a=>c.hostname.includes(a))}catch{return!1}},l=e.url(),t=new URL(l).hostname,r=e.frames(),n=[];for(let d of r){let c=d.url();try{let u=new URL(c).hostname;u&&u!==t&&!i.includes(c)&&!o(c)&&n.push(c)}catch{continue}}return n}async buildDomTree(e,i,o,l,t,r,n,d="all"){if(await e.evaluate("1+1")!==2)throw new Error("The page cannot evaluate javascript code properly");if(te(e.url()))return[new C("body","",{},[],!1,!1,!1,!1,!1,!1,!1,null),new Map];let u={doHighlightElements:i,focusHighlightIndex:o,viewportExpansion:l,debugMode:!1,interactiveClassNames:t,alwaysHighlightFileInput:r,sameRectIoUThreshold:n,actionIntent:d};h.debug(`\u{1F527} Starting JavaScript DOM analysis for ${e.url().slice(0,50)}...`);let a,m=null;if(this.useDomTreeTs&&i)try{let p={...u,phase:"boxes",grayscaleImage:null},b=await e.evaluate(({code:v,argsObj:x})=>new Function("return "+v)()(x),{code:this.jsCode,argsObj:p});h.debug(`\u{1F4E6} Phase 1: Drew ${b.elementData?.length||0} bounding boxes`);let g=await e.screenshot({type:"png",fullPage:!1});h.debug("\u{1F4F8} Captured screenshot with bounding boxes");let s=performance.now(),f=await V(g);m=f.pixels;let T=Math.round(performance.now()-s);h.debug(`\u{1F5BC}\uFE0F Generated grayscale image (${f.width}x${f.height}) in ${T}ms`);let y={...u,phase:"labels",grayscaleImage:m,elementData:b.elementData};a=await e.evaluate(({code:v,argsObj:x})=>new Function("return "+v)()(x),{code:this.jsCode,argsObj:y}),a.map=b.map,a.rootId=b.rootId,a.highlightCount=b.highlightCount,a.perfMetrics=b.perfMetrics,h.debug("\u2705 Phase 2: Labels placed using grayscale-based positioning")}catch(p){h.warn("Two-phase rendering failed, falling back to legacy mode:",p.message),m=null;let b={...u,grayscaleImage:null};a=await e.evaluate(({code:g,argsObj:s})=>new Function("return "+g)()(s),{code:this.jsCode,argsObj:b}),h.debug("\u2705 JavaScript DOM analysis completed (legacy mode)")}else try{a=await e.evaluate(({code:p,argsObj:b})=>new Function("return "+p)()(b),{code:this.jsCode,argsObj:u}),h.debug("\u2705 JavaScript DOM analysis completed")}catch(p){throw h.error("Error evaluating JavaScript:",p.message),p}if(!a||typeof a!="object")throw h.error("JavaScript returned invalid result:",a),new Error("JavaScript DOM analysis returned invalid result");if(!a.map||!a.rootId)throw h.error("JavaScript result missing map or rootId:",JSON.stringify(a,null,2)),new Error("JavaScript result missing required fields (map or rootId)");if(d!=="all"&&a.highlightCount===0){h.debug(`\u26A0\uFE0F No elements matched intent '${d}', falling back to 'all'`);let p={...u,actionIntent:"all"};a=await e.evaluate(({code:b,argsObj:g})=>new Function("return "+b)()(g),{code:this.jsCode,argsObj:p})}if(a&&a.perfMetrics){let b=a.perfMetrics.nodeMetrics?.totalNodes??0,g=0;if(a.map)for(let f of Object.values(a.map))typeof f=="object"&&f!==null&&f.isInteractive&&g++;let s=e.url().length>50?e.url().slice(0,50)+"...":e.url();h.debug(`\u{1F50E} Ran buildDOMTree.js interactive element detection on: ${s} interactive=${g}/${b}`)}h.debug("\u{1F504} Starting TypeScript DOM tree construction...");let E=await this.constructDomTree(a);return h.debug("\u2705 TypeScript DOM tree construction completed"),E}async constructDomTree(e){let i=e.map,o=e.rootId,l=new Map,t=new Map;for(let[n,d]of Object.entries(i)){let[c,u]=this.parseNode(d);if(c!==null&&(t.set(n,c),c instanceof C&&c.highlightIndex!==null&&l.set(c.highlightIndex,c),c instanceof C))for(let a of u){let m=t.get(a);m&&(m.parent=c,c.children.push(m))}}let r=t.get(o);if(!r||!(r instanceof C))throw new Error("Failed to parse HTML to dictionary");return[r,l]}parseNode(e){if(!e)return[null,[]];if(e.type==="TEXT_NODE")return[new N(e.text,e.isVisible,null),[]];let i=null;e.viewport&&(i={width:e.viewport.width,height:e.viewport.height,scrollX:e.viewport.scrollX,scrollY:e.viewport.scrollY});let o=new C(e.tagName,e.xpath,e.attributes||{},[],e.isVisible??!1,e.isInteractive??!1,e.isScrollable??!1,e.markAsClickable??!1,e.isTopElement??!1,e.isInViewport??!1,e.shadowRoot??!1,e.highlightIndex??null,e.viewportCoordinates??null,e.pageCoordinates??null,i,null),l=e.children||[];return[o,l]}};export{ie as a,be as b,ye as c,Me as d};
|