@shiplightai/sdk 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3880 @@
1
+ import{a as h}from"./chunk-YR4E7JSB.js";import{g as R}from"./chunk-UFLZ3URR.js";import{z as I}from"zod";var q=class{constructor(){this.tools=new Map}register(e){if(this.tools.has(e.name))throw new Error(`Tool '${e.name}' is already registered`);let i={name:e.name,description:e.description,schema:e.schema,execute:e.execute,usesElementIndex:e.usesElementIndex??!1,availability:{openai:e.availability?.openai??!0,mcp:e.availability?.mcp??!0}};this.tools.set(e.name,i)}get(e){return this.tools.get(e)}has(e){return this.tools.has(e)}getToolNames(){return Array.from(this.tools.keys())}getTools(){return Array.from(this.tools.values())}async execute(e,i,o){let l=this.tools.get(e);if(!l)throw new Error(`Tool not found: ${e}`);try{let t=i?.description,r={...i};delete r.description;let n=l.schema.parse(r),d={...o,actionDescription:t};return await l.execute(n,d)}catch(t){if(t instanceof I.ZodError){let r=t.issues.map(n=>`${n.path.join(".")}: ${n.message}`).join(", ");return{success:!1,error:`Invalid arguments for tool '${e}': ${r}`,actionEntity:{action_description:i?.description||`${e} (validation failed)`,action_data:{action_name:e,kwargs:i},feedback:`Validation error: ${r}`}}}throw t}}clear(){this.tools.clear()}size(){return this.tools.size}buildActionUnionSchema(){let e=this.getTools().filter(i=>i.availability.openai);return this.buildUnionSchemaFromTools(e)}buildActionUnionSchemaForTools(e){let i=new Set(e),o=this.getTools().filter(l=>i.has(l.name));return this.buildUnionSchemaFromTools(o)}buildUnionSchemaFromTools(e){if(e.length===0)return I.object({done:I.any()});let i=e.map(r=>{let n=r.schema;if(n instanceof I.ZodObject){let d=n._def.shape();Object.keys(d).length===0&&(n=I.object({_empty:I.boolean().optional()}))}return I.object({[r.name]:n})});if(i.length===1)return i[0];let[o,l,...t]=i;return I.union([o,l,...t])}},ie=new q;import{createAnthropic as K}from"@ai-sdk/anthropic";function H(e){let o=R().env?.ANTHROPIC_API_KEY;if(!o)throw new Error("ANTHROPIC_API_KEY not configured in SDK config");return h.debug(`Using Anthropic provider: model=${e}`),K({apiKey:o})(e)}function M(e){return{anthropic:{structuredOutputMode:"jsonTool"}}}import{createGoogleGenerativeAI as J}from"@ai-sdk/google";import{createVertex as Q}from"@ai-sdk/google-vertex";var O={MEDIA_RESOLUTION_HIGH:"MEDIA_RESOLUTION_HIGH",MEDIA_RESOLUTION_MEDIUM:"MEDIA_RESOLUTION_MEDIUM",MEDIA_RESOLUTION_LOW:"MEDIA_RESOLUTION_LOW"};function _(){let o=(R().env||{}).GOOGLE_GENAI_USE_VERTEXAI;return o==="True"||o==="true"}function P(e){let o=R().env||{};if(_()){let r=o.GOOGLE_CLOUD_PROJECT;if(!r)throw new Error("GOOGLE_CLOUD_PROJECT is required when using Vertex AI");let n=e==="gemini-3-flash-preview"?"global":o.GOOGLE_CLOUD_LOCATION;if(!n)throw new Error("GOOGLE_CLOUD_LOCATION is required when using Vertex AI");return h.debug(`Using Vertex AI provider: model=${e}, location=${n}`),Q({project:r,location:n})(e)}let l=o.GOOGLE_API_KEY||o.GOOGLE_GENERATIVE_AI_API_KEY;if(!l)throw new Error("Google API key is missing. Set GOOGLE_API_KEY or GOOGLE_GENERATIVE_AI_API_KEY in SDK config or environment.");return h.debug(`Using Google AI provider (API key): model=${e}`),J({apiKey:l})(e)}function W(e,i){let o={thinkingConfig:{thinkingBudget:512,includeThoughts:!0}},l={thinkingConfig:{thinkingLevel:"minimal",includeThoughts:!0},mediaResolution:O.MEDIA_RESOLUTION_HIGH},t;switch(i){case"gemini-3-flash-preview":t={...l};break;default:t={...o},e===1&&(t.mediaResolution=O.MEDIA_RESOLUTION_HIGH)}return _()?{vertex:t}:{google:t}}function be(e){if(e.startsWith("claude-"))return H(e);if(e.startsWith("gemini-"))return P(e);throw new Error(`Unsupported model: ${e}. Use 'claude-*' or 'gemini-*' models.`)}function ye(e,i){return e.startsWith("claude-")?M(e):W(i,e)}import S from"sharp";var F=768;async function D(e,i){let l=await S(e).metadata(),t=l.width||0,r=l.height||0;if(t===0||r===0)throw new Error("Invalid image dimensions");let n=r,d=0,c=Math.floor((t-n)/2),u=Math.max(0,t-n),a=(p,b)=>{let g=S(e).extract({left:p,top:0,width:b,height:n});return i?.resize&&(g=g.resize(F,F)),g.png().toBuffer()},[m,w,E]=await Promise.all([a(d,Math.min(n,t)),a(c,Math.min(n,t-c)),a(u,Math.min(n,t-u))]);return[m,w,E]}async function V(e){let i=S(e),o=await i.metadata(),l=o.width||0,t=o.height||0;if(l===0||t===0)throw new Error("Invalid image dimensions");let{data:r}=await i.grayscale().raw().toBuffer({resolveWithObject:!0}),n=[];for(let d=0;d<t;d++){n[d]=[];for(let c=0;c<l;c++)n[d][c]=r[d*l+c]}return{pixels:n,width:l,height:t}}var X=new Set(["button","link","textbox","checkbox","radio","combobox","listbox","menuitem","menuitemcheckbox","menuitemradio","option","tab","switch","slider","spinbutton","searchbox","scrollbar","treeitem","gridcell"]),$=new Set(["click","mousedown","mouseup","dblclick","pointerdown","pointerup","touchstart","touchend"]),B=["[onclick]","[onmousedown]","[ontouchstart]","div","span","li","tr","td","[role]",'[class*="btn"]','[class*="button"]','[class*="click"]',"[data-action]","[data-click]"],G=500;function Y(e,i){return e.length>i?e.slice(0,i)+"...":e}var z=["title","type","checked","name","role","value","placeholder","data-date-format","alt","aria-label","aria-expanded","data-state","aria-checked","data-id","data-testid","data-test-id","data-handlepos","data-item-id"],U={"react-flow__(\\S+)":"$1"};var j=class{constructor(e,i=null){this.isVisible=e,this.parent=i}},N=class extends j{constructor(e,i,o=null){super(i,o),this.text=e,this.type="TEXT_NODE"}hasParentWithHighlightIndex(){let e=this.parent;for(;e!==null;){if(e.highlightIndex!==null)return!0;e=e.parent}return!1}isParentInViewport(){return this.parent===null?!1:this.parent.isInViewport}isParentTopElement(){return this.parent===null?!1:this.parent.isTopElement}},C=class L extends j{constructor(i,o,l,t,r,n=!1,d=!1,c=!1,u=!1,a=!1,m=!1,w=null,E=null,p=null,b=null,g=null){super(r,g),this.tagName=i,this.xpath=o,this.attributes=l,this.children=t,this.isInteractive=n,this.isScrollable=d,this.markAsClickable=c,this.isTopElement=u,this.isInViewport=a,this.shadowRoot=m,this.highlightIndex=w,this.viewportCoordinates=E,this.pageCoordinates=p,this.viewportInfo=b,this.isNew=null}getAllTextTillNextClickableElement(i=-1){let o=[],l=(t,r)=>{if(!(i!==-1&&r>i)&&!(t instanceof L&&t!==this&&t.highlightIndex!==null)){if(t instanceof N)o.push(t.text);else if(t instanceof L)for(let n of t.children)l(n,r+1)}};return l(this,0),o.join(`
2
+ `).trim()}clickableElementsToString(i){let o=i?.includeAttributes??z,l=i?.includeClassesWithRename??U,t=[],r=(n,d)=>{let c=d,u=" ".repeat(d);if(n instanceof L){if(n.highlightIndex!==null){c+=1;let a=n.isScrollable?"":n.getAllTextTillNextClickableElement(),m=null;if(o.length>0){let s={};for(let y of Object.keys(n.attributes))if(o.includes(y)){let v=n.attributes[y].trim();v!==""&&(s[y]=v)}let f=o.filter(y=>y in s);if(f.length>1){let y=new Set,v={};for(let x of f){let k=s[x];k.length>5&&(k in v?y.add(x):v[k]=x)}for(let x of y)delete s[x]}n.tagName===s.role&&delete s.role;let T=["aria-label","placeholder","title"];for(let y of T)s[y]&&s[y].trim().toLowerCase()===a.trim().toLowerCase()&&delete s[y];Object.keys(s).length>0&&(m=Object.entries(s).map(([y,v])=>`${y}=${Y(v,200)}`).join(" "))}let w=n.isNew?`*[${n.highlightIndex}]`:`[${n.highlightIndex}]`,E=[];if(Object.keys(l).length>0&&n.attributes.class){let f=n.attributes.class.split(/\s+/);for(let T of f)for(let[y,v]of Object.entries(l))try{let x=new RegExp(`^${y}$`);if(T.match(x)){let A=T.replace(x,v);A&&E.push(A);break}}catch{continue}}let p=n.isScrollable?" (SCROLLABLE)":"",b=n.markAsClickable?" (CLICKABLE)":"",g=`${u}${w}${p}${b}<${n.tagName}`;if(E.length>0&&(g+=` ${E.join(" ")}`),m&&(g+=` ${m}`),a){let s=a.trim();m||(g+=" "),g+=`>${s}`}else m||(g+=" ");g+=" />",t.push(g)}for(let a of n.children)r(a,c)}else if(n instanceof N){if(n.hasParentWithHighlightIndex())return;n.parent&&n.parent.isVisible&&n.parent.isTopElement&&t.push(`${u}${n.text}`)}};return r(this,0),t.join(`
3
+ `)}};var Z=`(
4
+ args = {
5
+ doHighlightElements: true,
6
+ focusHighlightIndex: -1,
7
+ viewportExpansion: 0,
8
+ debugMode: false,
9
+ interactiveClassNames: [],
10
+ alwaysHighlightFileInput: false,
11
+ }
12
+ ) => {
13
+ const EVENT_LISTENER_MAPPING = {
14
+ 'onclick': 'click',
15
+ 'onmousedown': 'mousedown',
16
+ 'onmouseup': 'mouseup',
17
+ 'ondblclick': 'dblclick',
18
+ 'onmouseenter': 'mouseenter',
19
+ 'onmouseleave': 'mouseleave',
20
+ 'onmousemove': 'mousemove',
21
+ 'onmouseout': 'mouseout',
22
+ 'onmouseover': 'mouseover',
23
+ 'onmouseup': 'mouseup',
24
+ 'onmousewheel': 'mousewheel',
25
+ 'onscroll': 'scroll',
26
+ 'onselect': 'select',
27
+ 'onchange': 'change',
28
+ 'onfocus': 'focus',
29
+ 'onblur': 'blur',
30
+ 'onkeydown': 'keydown',
31
+ 'onkeyup': 'keyup',
32
+ 'onkeypress': 'keypress',
33
+ 'oninput': 'input',
34
+ }
35
+
36
+ const INTERACTION_EVENTS = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave'];
37
+
38
+ const {
39
+ doHighlightElements,
40
+ focusHighlightIndex,
41
+ viewportExpansion,
42
+ debugMode,
43
+ interactiveClassNames,
44
+ alwaysHighlightFileInput,
45
+ } = args;
46
+
47
+ const buttonClassNames = ['button', 'dropdown-toggle'];
48
+ const heuristicClassPattern = /\\b(btn|const clickable|menu|item|entry|link)\\b/i;
49
+ const containerSelectors = 'button,a,[role="button"],.menu,.dropdown,.list,.toolbar';
50
+
51
+ let highlightIndex = 0; // Reset highlight index
52
+
53
+ /**
54
+ * Helper function to check if element has any of the specified class names.
55
+ *
56
+ * @param {HTMLElement} element - The element to check.
57
+ * @param {string[]} classNames - Array of class names to check for.
58
+ * @returns {boolean} Whether the element has any of the specified class names.
59
+ */
60
+ function hasAnyClassName(element, classNames) {
61
+ if (!element.classList || !classNames || classNames.length === 0) return false;
62
+ return classNames.some(className => element.classList.contains(className));
63
+ }
64
+
65
+ // Add caching mechanisms at the top level
66
+ const DOM_CACHE = {
67
+ boundingRects: new WeakMap(),
68
+ clientRects: new WeakMap(),
69
+ computedStyles: new WeakMap(),
70
+ nodeEventListeners: new WeakMap(),
71
+ clearCache: () => {
72
+ DOM_CACHE.boundingRects = new WeakMap();
73
+ DOM_CACHE.clientRects = new WeakMap();
74
+ DOM_CACHE.computedStyles = new WeakMap();
75
+ DOM_CACHE.nodeEventListeners = new WeakMap();
76
+ }
77
+ };
78
+
79
+ /**
80
+ * Gets the cached bounding rect for an element.
81
+ *
82
+ * @param {HTMLElement} element - The element to get the bounding rect for.
83
+ * @returns {DOMRect | null} The cached bounding rect, or null if the element is not found.
84
+ */
85
+ function getCachedBoundingRect(element) {
86
+ if (!element) return null;
87
+
88
+ if (DOM_CACHE.boundingRects.has(element)) {
89
+ return DOM_CACHE.boundingRects.get(element);
90
+ }
91
+
92
+ const rect = element.getBoundingClientRect();
93
+
94
+ if (rect) {
95
+ DOM_CACHE.boundingRects.set(element, rect);
96
+ }
97
+ return rect;
98
+ }
99
+
100
+ /**
101
+ * Gets the cached computed style for an element.
102
+ *
103
+ * @param {HTMLElement} element - The element to get the computed style for.
104
+ * @returns {CSSStyleDeclaration | null} The cached computed style, or null if the element is not found.
105
+ */
106
+ function getCachedComputedStyle(element) {
107
+ if (!element) return null;
108
+
109
+ if (DOM_CACHE.computedStyles.has(element)) {
110
+ return DOM_CACHE.computedStyles.get(element);
111
+ }
112
+
113
+ const style = window.getComputedStyle(element);
114
+
115
+ if (style) {
116
+ DOM_CACHE.computedStyles.set(element, style);
117
+ }
118
+ return style;
119
+ }
120
+
121
+ /**
122
+ * Gets the cached client rects for an element.
123
+ *
124
+ * @param {HTMLElement} element - The element to get the client rects for.
125
+ * @returns {DOMRectList | null} The cached client rects, or null if the element is not found.
126
+ */
127
+ function getCachedClientRects(element) {
128
+ if (!element) return null;
129
+
130
+ if (DOM_CACHE.clientRects.has(element)) {
131
+ return DOM_CACHE.clientRects.get(element);
132
+ }
133
+
134
+ const rects = element.getClientRects();
135
+
136
+ if (rects) {
137
+ DOM_CACHE.clientRects.set(element, rects);
138
+ }
139
+ return rects;
140
+ }
141
+
142
+ /**
143
+ * Gets the event listeners for a node.
144
+ *
145
+ * @param {HTMLElement} element - The element to get the event listeners for.
146
+ * @returns {string[]} The event listeners for the element.
147
+ */
148
+ function getNodeEventListeners(element) {
149
+ const set = new Set();
150
+ try {
151
+ if (typeof getEventListeners === 'function') {
152
+ const listeners = getEventListeners(element);
153
+ for (const eventType in listeners) {
154
+ if (listeners[eventType] && listeners[eventType].length > 0) {
155
+ set.add(eventType);
156
+ }
157
+ }
158
+ }
159
+
160
+ const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
161
+ if (typeof getEventListenersForNode === 'function') {
162
+ const listeners = getEventListenersForNode(element);
163
+ for (const listener of listeners) {
164
+ if (listener.type) {
165
+ set.add(listener.type);
166
+ }
167
+ }
168
+ }
169
+
170
+ for (const attr in EVENT_LISTENER_MAPPING) {
171
+ if (element.hasAttribute(attr) || typeof element[attr] === 'function') {
172
+ set.add(EVENT_LISTENER_MAPPING[attr]);
173
+ }
174
+ }
175
+ } catch (error) {
176
+ }
177
+
178
+ const listenedEvents = Array.from(set);
179
+ return listenedEvents;
180
+ }
181
+
182
+ function getCachedNodeEventListeners(element) {
183
+ if (!element) return null;
184
+ if (DOM_CACHE.nodeEventListeners.has(element)) {
185
+ return DOM_CACHE.nodeEventListeners.get(element);
186
+ }
187
+ const listenedEvents = getNodeEventListeners(element);
188
+ if (listenedEvents) {
189
+ DOM_CACHE.nodeEventListeners.set(element, listenedEvents);
190
+ }
191
+ return listenedEvents;
192
+ }
193
+
194
+ /**
195
+ * Hash map of DOM nodes indexed by their highlight index.
196
+ *
197
+ * @type {Object<string, any>}
198
+ */
199
+ const DOM_HASH_MAP = {};
200
+
201
+ const ID = { current: 0 };
202
+
203
+ const HIGHLIGHT_CONTAINER_ID = "playwright-highlight-container";
204
+
205
+ // Add a WeakMap cache for XPath strings
206
+ const xpathCache = new WeakMap();
207
+
208
+ const existingLabelBoundingBoxes = [];
209
+
210
+ /**
211
+ * Highlights an element in the DOM and returns the index of the next element.
212
+ *
213
+ * @param {HTMLElement} element - The element to highlight.
214
+ * @param {number} index - The index of the element.
215
+ * @param {HTMLElement | null} parentIframe - The parent iframe node.
216
+ * @returns {number} The index of the next element.
217
+ */
218
+ function highlightElement(element, index, parentIframe = null) {
219
+ if (!element) return index;
220
+
221
+ const overlays = [];
222
+ /**
223
+ * @type {HTMLElement | null}
224
+ */
225
+ let label = null;
226
+ let labelWidth = 20;
227
+ let labelHeight = 16;
228
+ let cleanupFn = null;
229
+
230
+ try {
231
+ // Create or get highlight container
232
+ let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
233
+ if (!container) {
234
+ container = document.createElement("div");
235
+ container.id = HIGHLIGHT_CONTAINER_ID;
236
+ container.style.position = "fixed";
237
+ container.style.pointerEvents = "none";
238
+ container.style.top = "0";
239
+ container.style.left = "0";
240
+ container.style.width = "100%";
241
+ container.style.height = "100%";
242
+ // Use the maximum valid value in zIndex to ensure the element is not blocked by overlapping elements.
243
+ container.style.zIndex = "2147483647";
244
+ container.style.backgroundColor = 'transparent';
245
+ document.body.appendChild(container);
246
+ }
247
+
248
+ // Get element client rects
249
+ let rects = element.getClientRects(); // Use getClientRects()
250
+
251
+ if (!rects || rects.length === 0) return index; // Exit if no rects
252
+
253
+ // If element is inside an iframe, we need to transform the rects to main document coordinates
254
+ if (parentIframe) {
255
+ const transformedRects = [];
256
+ const iframeRect = parentIframe.getBoundingClientRect();
257
+
258
+ // Get iframe's content area offset and CSS transforms
259
+ const iframeStyle = window.getComputedStyle(parentIframe);
260
+ const borderLeft = parseFloat(iframeStyle.borderLeftWidth) || 0;
261
+ const borderTop = parseFloat(iframeStyle.borderTopWidth) || 0;
262
+ const paddingLeft = parseFloat(iframeStyle.paddingLeft) || 0;
263
+ const paddingTop = parseFloat(iframeStyle.paddingTop) || 0;
264
+
265
+ const contentOffsetX = borderLeft + paddingLeft;
266
+ const contentOffsetY = borderTop + paddingTop;
267
+
268
+ // Extract scale factor from CSS transform
269
+ let scaleX = 1, scaleY = 1;
270
+ const transform = iframeStyle.transform;
271
+ if (transform && transform !== 'none') {
272
+ // Parse scale from transform matrix or scale() function
273
+ const scaleMatch = transform.match(/scale\\(([^)]+)\\)/);
274
+ if (scaleMatch) {
275
+ const scaleValues = scaleMatch[1].split(',').map(v => parseFloat(v.trim()));
276
+ scaleX = scaleValues[0] || 1;
277
+ scaleY = scaleValues[1] || scaleX; // If only one value, use it for both X and Y
278
+ } else {
279
+ // Try to parse matrix() transform
280
+ const matrixMatch = transform.match(/matrix\\(([^)]+)\\)/);
281
+ if (matrixMatch) {
282
+ const values = matrixMatch[1].split(',').map(v => parseFloat(v.trim()));
283
+ if (values.length >= 6) {
284
+ scaleX = values[0]; // a value in matrix(a, b, c, d, e, f)
285
+ scaleY = values[3]; // d value in matrix(a, b, c, d, e, f)
286
+ }
287
+ }
288
+ }
289
+ }
290
+
291
+ // Get iframe scroll position
292
+ let scrollLeft = 0, scrollTop = 0;
293
+ try {
294
+ const iframeDoc = parentIframe.contentDocument || parentIframe.contentWindow?.document;
295
+ if (iframeDoc) {
296
+ scrollLeft = iframeDoc.documentElement?.scrollLeft || iframeDoc.body?.scrollLeft || 0;
297
+ scrollTop = iframeDoc.documentElement?.scrollTop || iframeDoc.body?.scrollTop || 0;
298
+ }
299
+ } catch (e) {
300
+ console.warn("Cannot access iframe scroll position (cross-origin):", e);
301
+ }
302
+
303
+ // Transform each rect accounting for iframe scaling
304
+ for (const rect of rects) {
305
+ // Apply scale factor to coordinates and dimensions
306
+ const scaledWidth = rect.width * scaleX;
307
+ const scaledHeight = rect.height * scaleY;
308
+ const scaledTop = rect.top * scaleY;
309
+ const scaledLeft = rect.left * scaleX;
310
+ const scaledScrollTop = scrollTop * scaleY;
311
+ const scaledScrollLeft = scrollLeft * scaleX;
312
+
313
+ const transformedRect = {
314
+ top: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop,
315
+ left: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
316
+ bottom: scaledTop + scaledHeight + iframeRect.top + contentOffsetY - scaledScrollTop,
317
+ right: scaledLeft + scaledWidth + iframeRect.left + contentOffsetX - scaledScrollLeft,
318
+ width: scaledWidth,
319
+ height: scaledHeight,
320
+ x: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
321
+ y: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop
322
+ };
323
+ transformedRects.push(transformedRect);
324
+ }
325
+
326
+ rects = transformedRects;
327
+ }
328
+
329
+ // Generate a color based on the index
330
+ const colors = [
331
+ "#FF0000",
332
+ "#00FF00",
333
+ "#0000FF",
334
+ "#FFA500",
335
+ "#800080",
336
+ "#008080",
337
+ "#FF69B4",
338
+ "#4B0082",
339
+ "#FF4500",
340
+ "#2E8B57",
341
+ "#DC143C",
342
+ "#4682B4",
343
+ ];
344
+ const colorIndex = index % colors.length;
345
+ const baseColor = colors[colorIndex];
346
+ const backgroundColor = baseColor + "1A"; // 10% opacity version of the color
347
+
348
+ // No need for iframe offset calculation since rects are already transformed to main document coordinates
349
+ const iframeOffset = { x: 0, y: 0 };
350
+
351
+ // Create fragment to hold overlay elements
352
+ const fragment = document.createDocumentFragment();
353
+
354
+ // Create highlight overlays for each client rect
355
+ for (const rect of rects) {
356
+ if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects
357
+
358
+ const overlay = document.createElement("div");
359
+ overlay.style.position = "fixed";
360
+ overlay.style.border = \`1px solid \${baseColor}\`;
361
+ overlay.style.backgroundColor = "none";
362
+ overlay.style.pointerEvents = "none";
363
+ overlay.style.boxSizing = "border-box";
364
+
365
+ const top = rect.top + iframeOffset.y;
366
+ const left = rect.left + iframeOffset.x;
367
+
368
+ overlay.style.top = \`\${top}px\`;
369
+ overlay.style.left = \`\${left}px\`;
370
+ overlay.style.width = \`\${rect.width}px\`;
371
+ overlay.style.height = \`\${rect.height}px\`;
372
+
373
+ fragment.appendChild(overlay);
374
+ overlays.push({ element: overlay, initialRect: rect }); // Store overlay and its rect
375
+ }
376
+
377
+ // Create and position a single label relative to the first rect
378
+ const firstRect = rects[0];
379
+ label = document.createElement("div");
380
+ label.className = "playwright-highlight-label";
381
+ label.style.position = "fixed";
382
+ label.style.background = baseColor;
383
+ label.style.color = "white";
384
+ label.style.padding = "1px 4px";
385
+ label.style.borderRadius = "4px";
386
+ const fontSize = Math.min(index >= 100 ? 8 : 12, Math.max(8, firstRect.height / 2));
387
+ label.style.fontSize = \`\${fontSize}px\`;
388
+ label.textContent = index;
389
+
390
+ // labelWidth = label.offsetWidth > 0 ? label.offsetWidth : labelWidth; // Update actual width if possible
391
+ // labelHeight = label.offsetHeight > 0 ? label.offsetHeight : labelHeight; // Update actual height if possible
392
+ labelWidth *= fontSize / 12;
393
+ labelHeight *= fontSize / 12;
394
+ const digits = index.toString().length;
395
+ labelWidth += (digits - 2) * fontSize / 1.5;
396
+ labelHeight *= fontSize / 12;
397
+
398
+ const firstRectTop = firstRect.top + iframeOffset.y;
399
+ const firstRectLeft = firstRect.left + iframeOffset.x;
400
+
401
+ let labelTop = firstRectTop - labelHeight - 4 * 12 / fontSize;
402
+ let labelLeft = firstRectLeft + firstRect.width - labelWidth - 2;
403
+
404
+ // Adjust label position if first rect is too small
405
+ if (firstRect.width < labelWidth + 4 || firstRect.height < labelHeight + 4) {
406
+ labelTop = firstRectTop - labelHeight;
407
+ labelLeft = firstRectLeft + firstRect.width - labelWidth; // Align with right edge
408
+ if (labelLeft < iframeOffset.x) labelLeft = firstRectLeft; // Prevent going off-left
409
+ }
410
+
411
+ // // Check if the label is too close to any existing label
412
+ // const minDistance = 2; // Minimum distance between labels
413
+ // let hasOverlap = true;
414
+ // let attempts = 0;
415
+ // const maxAttempts = 5;
416
+
417
+ // const alignmentPositions = [
418
+ // // align with right edge (current default)
419
+ // firstRectLeft + firstRect.width - labelWidth - 2,
420
+ // // 1/4 of the way from the right edge
421
+ // firstRectLeft + (firstRect.width - labelWidth) / 2 + firstRect.width / 4,
422
+ // // in middle
423
+ // firstRectLeft + (firstRect.width - labelWidth) / 2,
424
+ // firstRectLeft + (firstRect.width - labelWidth) / 2 - firstRect.width / 4,
425
+ // // align with left edge
426
+ // firstRectLeft,
427
+ // ];
428
+
429
+ // while (hasOverlap && attempts < maxAttempts) {
430
+ // hasOverlap = false;
431
+
432
+ // // Use the current alignment position
433
+ // labelLeft = alignmentPositions[attempts];
434
+
435
+ // // Current label bounding box
436
+ // const currentLabel = {
437
+ // xmin: labelLeft,
438
+ // ymin: labelTop,
439
+ // xmax: labelLeft + labelWidth,
440
+ // ymax: labelTop + labelHeight,
441
+ // };
442
+
443
+ // for (const existingLabel of existingLabelBoundingBoxes) {
444
+ // // Check if labels overlap or are too close (using minDistance buffer)
445
+ // if (!(currentLabel.xmax + minDistance < existingLabel.xmin ||
446
+ // currentLabel.xmin > existingLabel.xmax + minDistance ||
447
+ // currentLabel.ymax + minDistance < existingLabel.ymin ||
448
+ // currentLabel.ymin > existingLabel.ymax + minDistance)) {
449
+ // hasOverlap = true;
450
+ // break;
451
+ // }
452
+ // }
453
+
454
+ // if (hasOverlap) {
455
+ // attempts++;
456
+ // }
457
+ // }
458
+
459
+ // Ensure label stays within viewport bounds slightly better
460
+ labelTop = Math.max(0, Math.min(labelTop, window.innerHeight - labelHeight));
461
+ labelLeft = Math.max(0, Math.min(labelLeft, window.innerWidth - labelWidth - 2));
462
+
463
+
464
+ label.style.top = \`\${labelTop}px\`;
465
+ label.style.left = \`\${labelLeft}px\`;
466
+
467
+ // Add the label to the existing label bounding boxes
468
+ existingLabelBoundingBoxes.push({
469
+ xmin: labelLeft,
470
+ ymin: labelTop,
471
+ xmax: labelLeft + labelWidth,
472
+ ymax: labelTop + labelHeight,
473
+ });
474
+
475
+ fragment.appendChild(label);
476
+
477
+ // Update positions on scroll/resize
478
+ const updatePositions = () => {
479
+ let newRects = element.getClientRects(); // Get fresh rects
480
+
481
+ // Transform rects if element is inside an iframe (same logic as initial highlighting)
482
+ if (parentIframe) {
483
+ const transformedRects = [];
484
+ const iframeRect = parentIframe.getBoundingClientRect();
485
+
486
+ // Get iframe's content area offset and CSS transforms
487
+ const iframeStyle = window.getComputedStyle(parentIframe);
488
+ const borderLeft = parseFloat(iframeStyle.borderLeftWidth) || 0;
489
+ const borderTop = parseFloat(iframeStyle.borderTopWidth) || 0;
490
+ const paddingLeft = parseFloat(iframeStyle.paddingLeft) || 0;
491
+ const paddingTop = parseFloat(iframeStyle.paddingTop) || 0;
492
+
493
+ const contentOffsetX = borderLeft + paddingLeft;
494
+ const contentOffsetY = borderTop + paddingTop;
495
+
496
+ // Extract scale factor from CSS transform
497
+ let scaleX = 1, scaleY = 1;
498
+ const transform = iframeStyle.transform;
499
+ if (transform && transform !== 'none') {
500
+ // Parse scale from transform matrix or scale() function
501
+ const scaleMatch = transform.match(/scale\\(([^)]+)\\)/);
502
+ if (scaleMatch) {
503
+ const scaleValues = scaleMatch[1].split(',').map(v => parseFloat(v.trim()));
504
+ scaleX = scaleValues[0] || 1;
505
+ scaleY = scaleValues[1] || scaleX; // If only one value, use it for both X and Y
506
+ } else {
507
+ // Try to parse matrix() transform
508
+ const matrixMatch = transform.match(/matrix\\(([^)]+)\\)/);
509
+ if (matrixMatch) {
510
+ const values = matrixMatch[1].split(',').map(v => parseFloat(v.trim()));
511
+ if (values.length >= 6) {
512
+ scaleX = values[0]; // a value in matrix(a, b, c, d, e, f)
513
+ scaleY = values[3]; // d value in matrix(a, b, c, d, e, f)
514
+ }
515
+ }
516
+ }
517
+ }
518
+
519
+ // Get iframe scroll position
520
+ let scrollLeft = 0, scrollTop = 0;
521
+ try {
522
+ const iframeDoc = parentIframe.contentDocument || parentIframe.contentWindow?.document;
523
+ if (iframeDoc) {
524
+ scrollLeft = iframeDoc.documentElement?.scrollLeft || iframeDoc.body?.scrollLeft || 0;
525
+ scrollTop = iframeDoc.documentElement?.scrollTop || iframeDoc.body?.scrollTop || 0;
526
+ }
527
+ } catch (e) {
528
+ console.warn("Cannot access iframe scroll position (cross-origin):", e);
529
+ }
530
+
531
+ // Transform each rect accounting for iframe scaling
532
+ for (const rect of newRects) {
533
+ // Apply scale factor to coordinates and dimensions
534
+ const scaledWidth = rect.width * scaleX;
535
+ const scaledHeight = rect.height * scaleY;
536
+ const scaledTop = rect.top * scaleY;
537
+ const scaledLeft = rect.left * scaleX;
538
+ const scaledScrollTop = scrollTop * scaleY;
539
+ const scaledScrollLeft = scrollLeft * scaleX;
540
+
541
+ const transformedRect = {
542
+ top: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop,
543
+ left: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
544
+ bottom: scaledTop + scaledHeight + iframeRect.top + contentOffsetY - scaledScrollTop,
545
+ right: scaledLeft + scaledWidth + iframeRect.left + contentOffsetX - scaledScrollLeft,
546
+ width: scaledWidth,
547
+ height: scaledHeight,
548
+ x: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
549
+ y: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop
550
+ };
551
+ transformedRects.push(transformedRect);
552
+ }
553
+
554
+ newRects = transformedRects;
555
+ }
556
+
557
+ const newIframeOffset = { x: 0, y: 0 }; // No offset needed since rects are already transformed
558
+
559
+ // Update each overlay
560
+ overlays.forEach((overlayData, i) => {
561
+ if (i < newRects.length) { // Check if rect still exists
562
+ const newRect = newRects[i];
563
+ const newTop = newRect.top + newIframeOffset.y;
564
+ const newLeft = newRect.left + newIframeOffset.x;
565
+
566
+ overlayData.element.style.top = \`\${newTop}px\`;
567
+ overlayData.element.style.left = \`\${newLeft}px\`;
568
+ overlayData.element.style.width = \`\${newRect.width}px\`;
569
+ overlayData.element.style.height = \`\${newRect.height}px\`;
570
+ overlayData.element.style.display = (newRect.width === 0 || newRect.height === 0) ? 'none' : 'block';
571
+ } else {
572
+ // If fewer rects now, hide extra overlays
573
+ overlayData.element.style.display = 'none';
574
+ }
575
+ });
576
+
577
+ // If there are fewer new rects than overlays, hide the extras
578
+ if (newRects.length < overlays.length) {
579
+ for (let i = newRects.length; i < overlays.length; i++) {
580
+ overlays[i].element.style.display = 'none';
581
+ }
582
+ }
583
+
584
+ // Update label position based on the first new rect
585
+ if (label && newRects.length > 0) {
586
+ const firstNewRect = newRects[0];
587
+ const firstNewRectTop = firstNewRect.top + newIframeOffset.y;
588
+ const firstNewRectLeft = firstNewRect.left + newIframeOffset.x;
589
+
590
+ let newLabelTop = firstNewRectTop + 2;
591
+ let newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth - 2;
592
+
593
+ if (firstNewRect.width < labelWidth + 4 || firstNewRect.height < labelHeight + 4) {
594
+ newLabelTop = firstNewRectTop - labelHeight - 2;
595
+ newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth;
596
+ if (newLabelLeft < newIframeOffset.x) newLabelLeft = firstNewRectLeft;
597
+ }
598
+
599
+ // Ensure label stays within viewport bounds
600
+ newLabelTop = Math.max(0, Math.min(newLabelTop, window.innerHeight - labelHeight));
601
+ newLabelLeft = Math.max(0, Math.min(newLabelLeft, window.innerWidth - labelWidth));
602
+
603
+ label.style.top = \`\${newLabelTop}px\`;
604
+ label.style.left = \`\${newLabelLeft}px\`;
605
+ label.style.display = 'block';
606
+ } else if (label) {
607
+ // Hide label if element has no rects anymore
608
+ label.style.display = 'none';
609
+ }
610
+ };
611
+
612
+ const throttleFunction = (func, delay) => {
613
+ let lastCall = 0;
614
+ return (...args) => {
615
+ const now = performance.now();
616
+ if (now - lastCall < delay) return;
617
+ lastCall = now;
618
+ return func(...args);
619
+ };
620
+ };
621
+
622
+ // const throttledUpdatePositions = throttleFunction(updatePositions, 16); // ~60fps
623
+ // window.addEventListener('scroll', throttledUpdatePositions, true);
624
+ // window.addEventListener('resize', throttledUpdatePositions);
625
+
626
+ // Add cleanup function
627
+ cleanupFn = () => {
628
+ // window.removeEventListener('scroll', throttledUpdatePositions, true);
629
+ // window.removeEventListener('resize', throttledUpdatePositions);
630
+ // Remove overlay elements if needed
631
+ overlays.forEach(overlay => overlay.element.remove());
632
+ if (label) label.remove();
633
+ };
634
+
635
+ // Then add fragment to container in one operation
636
+ container.appendChild(fragment);
637
+
638
+ return index + 1;
639
+ } finally {
640
+ // Store cleanup function for later use
641
+ if (cleanupFn) {
642
+ // Keep a reference to cleanup functions in a global array
643
+ (window._highlightCleanupFunctions = window._highlightCleanupFunctions || []).push(cleanupFn);
644
+ }
645
+ }
646
+ }
647
+
648
+
649
+ /**
650
+ * Gets the position of an element in its parent.
651
+ *
652
+ * @param {HTMLElement} currentElement - The element to get the position for.
653
+ * @returns {number} The position of the element in its parent.
654
+ */
655
+ function getElementPosition(currentElement) {
656
+ if (!currentElement.parentElement) {
657
+ return 0; // No parent means no siblings
658
+ }
659
+
660
+ const tagName = currentElement.nodeName.toLowerCase();
661
+
662
+ const siblings = Array.from(currentElement.parentElement.children)
663
+ .filter((sib) => sib.nodeName.toLowerCase() === tagName);
664
+
665
+ if (siblings.length === 1) {
666
+ return 0; // Only element of its type
667
+ }
668
+
669
+ const index = siblings.indexOf(currentElement) + 1; // 1-based index
670
+ return index;
671
+ }
672
+
673
+
674
+ function getXPathTree(element, stopAtBoundary = true) {
675
+ if (xpathCache.has(element)) return xpathCache.get(element);
676
+
677
+ const segments = [];
678
+ let currentElement = element;
679
+
680
+ while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) {
681
+ // Stop if we hit a shadow root or iframe
682
+ if (
683
+ stopAtBoundary &&
684
+ (currentElement.parentNode instanceof ShadowRoot ||
685
+ currentElement.parentNode instanceof HTMLIFrameElement)
686
+ ) {
687
+ break;
688
+ }
689
+
690
+ const position = getElementPosition(currentElement);
691
+ const tagName = currentElement.nodeName.toLowerCase();
692
+ const xpathIndex = position > 0 ? \`[\${position}]\` : "";
693
+ segments.unshift(\`\${tagName}\${xpathIndex}\`);
694
+
695
+ currentElement = currentElement.parentNode;
696
+ }
697
+
698
+ const result = segments.join("/");
699
+ xpathCache.set(element, result);
700
+ return result;
701
+ }
702
+
703
+ /**
704
+ * Checks if a text node is visible.
705
+ *
706
+ * @param {Text} textNode - The text node to check.
707
+ * @returns {boolean} Whether the text node is visible.
708
+ */
709
+ function isTextNodeVisible(textNode) {
710
+ try {
711
+ // Special case: when viewportExpansion is -1, consider all text nodes as visible
712
+ if (viewportExpansion === -1) {
713
+ // Still check parent visibility for basic filtering
714
+ const parentElement = textNode.parentElement;
715
+ if (!parentElement) return false;
716
+
717
+ try {
718
+ return parentElement.checkVisibility({
719
+ checkOpacity: true,
720
+ checkVisibilityCSS: true,
721
+ });
722
+ } catch (e) {
723
+ // Fallback if checkVisibility is not supported
724
+ const style = window.getComputedStyle(parentElement);
725
+ return style.display !== 'none' &&
726
+ style.visibility !== 'hidden' &&
727
+ style.opacity !== '0';
728
+ }
729
+ }
730
+
731
+ const range = document.createRange();
732
+ range.selectNodeContents(textNode);
733
+ const rects = range.getClientRects(); // Use getClientRects for Range
734
+
735
+ if (!rects || rects.length === 0) {
736
+ return false;
737
+ }
738
+
739
+ let isAnyRectVisible = false;
740
+ let isAnyRectInViewport = false;
741
+
742
+ for (const rect of rects) {
743
+ // Check size
744
+ if (rect.width > 0 && rect.height > 0) {
745
+ isAnyRectVisible = true;
746
+
747
+ // Viewport check for this rect
748
+ if (!(
749
+ rect.bottom < -viewportExpansion ||
750
+ rect.top > window.innerHeight + viewportExpansion ||
751
+ rect.right < -viewportExpansion ||
752
+ rect.left > window.innerWidth + viewportExpansion
753
+ )) {
754
+ isAnyRectInViewport = true;
755
+ break; // Found a visible rect in viewport, no need to check others
756
+ }
757
+ }
758
+ }
759
+
760
+ if (!isAnyRectVisible || !isAnyRectInViewport) {
761
+ return false;
762
+ }
763
+
764
+ // Check parent visibility
765
+ const parentElement = textNode.parentElement;
766
+ if (!parentElement) return false;
767
+
768
+ try {
769
+ return parentElement.checkVisibility({
770
+ checkOpacity: true,
771
+ checkVisibilityCSS: true,
772
+ });
773
+ } catch (e) {
774
+ // Fallback if checkVisibility is not supported
775
+ const style = window.getComputedStyle(parentElement);
776
+ return style.display !== 'none' &&
777
+ style.visibility !== 'hidden' &&
778
+ style.opacity !== '0';
779
+ }
780
+ } catch (e) {
781
+ console.warn('Error checking text node visibility:', e);
782
+ return false;
783
+ }
784
+ }
785
+
786
+ /**
787
+ * Checks if an element is accepted.
788
+ *
789
+ * @param {HTMLElement} element - The element to check.
790
+ * @returns {boolean} Whether the element is accepted.
791
+ */
792
+ function isElementAccepted(element) {
793
+ if (!element || !element.tagName) return false;
794
+
795
+ // Always accept body and common container elements
796
+ const alwaysAccept = new Set([
797
+ "body", "div", "main", "article", "section", "nav", "header", "footer"
798
+ ]);
799
+ const tagName = element.tagName.toLowerCase();
800
+
801
+ if (alwaysAccept.has(tagName)) return true;
802
+
803
+ const leafElementDenyList = new Set([
804
+ "svg",
805
+ "script",
806
+ "style",
807
+ "link",
808
+ "meta",
809
+ "noscript",
810
+ "template",
811
+ ]);
812
+
813
+ return !leafElementDenyList.has(tagName);
814
+ }
815
+
816
+ /**
817
+ * Checks if an element is visible.
818
+ *
819
+ * @param {HTMLElement} element - The element to check.
820
+ * @returns {boolean} Whether the element is visible.
821
+ */
822
+ function isElementVisible(element) {
823
+ if (element.tagName.toLowerCase() === "input" && element.type === "date") {
824
+ return true;
825
+ }
826
+
827
+ if (alwaysHighlightFileInput && element.tagName.toLowerCase() === "input" && element.type === "file") return true;
828
+
829
+ // SVG elements need special handling for visibility
830
+ if (element.tagName.toLowerCase() === "svg") {
831
+ const rect = getCachedBoundingRect(element);
832
+ const style = getCachedComputedStyle(element);
833
+ return (
834
+ rect &&
835
+ rect.width > 0 &&
836
+ rect.height > 0 &&
837
+ style?.visibility !== "hidden" &&
838
+ style?.display !== "none"
839
+ );
840
+ }
841
+
842
+ const style = getCachedComputedStyle(element);
843
+ return (
844
+ element.offsetWidth > 0 &&
845
+ element.offsetHeight > 0 &&
846
+ style?.visibility !== "hidden" &&
847
+ style?.display !== "none"
848
+ );
849
+ }
850
+
851
+ /**
852
+ * Checks if an element is clickable (responds to click events).
853
+ *
854
+ * @param {HTMLElement} element - The element to check.
855
+ * @returns {boolean} Whether the element is clickable.
856
+ */
857
+ function shouldMarkAsClickable(element) {
858
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) {
859
+ return false;
860
+ }
861
+
862
+ const tagName = element.tagName.toLowerCase();
863
+
864
+ // Primarily clickable elements
865
+ const primaryClickableElements = new Set([
866
+ "a", // Links
867
+ "button", // Buttons
868
+ "details", // Expandable details
869
+ "summary", // Summary element (clickable part of details)
870
+ "label", // Form labels (often clickable)
871
+ "option", // Select options
872
+ "optgroup", // Option groups
873
+ ]);
874
+
875
+ if (primaryClickableElements.has(tagName)) {
876
+ return false;
877
+ }
878
+
879
+ const role = element.getAttribute("role");
880
+
881
+ // Clickable roles
882
+ const clickableRoles = new Set([
883
+ 'button', // Directly clickable element
884
+ 'link', // Clickable link
885
+ 'menuitem', // Clickable menu item
886
+ 'menuitemradio', // Radio-style menu item (selectable)
887
+ 'menuitemcheckbox', // Checkbox-style menu item (toggleable)
888
+ 'radio', // Radio button (selectable)
889
+ 'checkbox', // Checkbox (toggleable)
890
+ 'tab', // Tab (clickable to switch content)
891
+ 'switch', // Toggle switch (clickable to change state)
892
+ 'option', // Selectable option in a list
893
+ ]);
894
+
895
+ if (role && clickableRoles.has(role)) {
896
+ return true;
897
+ }
898
+
899
+ // Check for dropdown indicators
900
+ if (hasAnyClassName(element, buttonClassNames)) {
901
+ return true; // Return true for dropdown elements
902
+ }
903
+
904
+ if (
905
+ element.getAttribute('data-toggle') === 'dropdown' ||
906
+ element.getAttribute('aria-haspopup')
907
+ ) {
908
+ return true;
909
+ }
910
+
911
+ const clickEvents = ['click', 'mousedown', 'mouseup', 'dblclick'];
912
+ const listenedEvents = getCachedNodeEventListeners(element);
913
+ if (listenedEvents && listenedEvents.length > 0) {
914
+ for (const eventType of clickEvents) {
915
+ if (listenedEvents.includes(eventType)) {
916
+ return true;
917
+ }
918
+ }
919
+ }
920
+
921
+ return false;
922
+ }
923
+
924
+ /**
925
+ * Checks if an element is interactive.
926
+ *
927
+ * lots of comments, and uncommented code - to show the logic of what we already tried
928
+ *
929
+ * One of the things we tried at the beginning was also to use event listeners, and other fancy class, style stuff -> what actually worked best was just combining most things with computed cursor style :)
930
+ *
931
+ * @param {HTMLElement} element - The element to check.
932
+ */
933
+ function isInteractiveElement(element) {
934
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) {
935
+ return false;
936
+ }
937
+
938
+ // Cache the tagName and style lookups
939
+ const tagName = element.tagName.toLowerCase();
940
+ const style = getCachedComputedStyle(element);
941
+
942
+ // Define interactive cursors
943
+ const interactiveCursors = new Set([
944
+ 'pointer', // Link/clickable elements
945
+ 'move', // Movable elements
946
+ 'text', // Text selection
947
+ 'grab', // Grabbable elements
948
+ 'grabbing', // Currently grabbing
949
+ 'cell', // Table cell selection
950
+ 'copy', // Copy operation
951
+ 'alias', // Alias creation
952
+ 'all-scroll', // Scrollable content
953
+ 'col-resize', // Column resize
954
+ 'context-menu', // Context menu available
955
+ 'crosshair', // Precise selection
956
+ 'e-resize', // East resize
957
+ 'ew-resize', // East-west resize
958
+ 'help', // Help available
959
+ 'n-resize', // North resize
960
+ 'ne-resize', // Northeast resize
961
+ 'nesw-resize', // Northeast-southwest resize
962
+ 'ns-resize', // North-south resize
963
+ 'nw-resize', // Northwest resize
964
+ 'nwse-resize', // Northwest-southeast resize
965
+ 'row-resize', // Row resize
966
+ 's-resize', // South resize
967
+ 'se-resize', // Southeast resize
968
+ 'sw-resize', // Southwest resize
969
+ 'vertical-text', // Vertical text selection
970
+ 'w-resize', // West resize
971
+ 'zoom-in', // Zoom in
972
+ 'zoom-out' // Zoom out
973
+ ]);
974
+
975
+ // Define non-interactive cursors
976
+ const nonInteractiveCursors = new Set([
977
+ 'not-allowed', // Action not allowed
978
+ 'no-drop', // Drop not allowed
979
+ 'wait', // Processing
980
+ 'progress', // In progress
981
+ 'initial', // Initial value
982
+ 'inherit' // Inherited value
983
+ //? Let's just include all potentially clickable elements that are not specifically blocked
984
+ // 'none', // No cursor
985
+ // 'default', // Default cursor
986
+ // 'auto', // Browser default
987
+ ]);
988
+
989
+ /**
990
+ * Checks if an element has an interactive pointer.
991
+ *
992
+ * @param {HTMLElement} element - The element to check.
993
+ * @returns {boolean} Whether the element has an interactive pointer.
994
+ */
995
+ function doesElementHaveInteractivePointer(element) {
996
+ if (element.tagName.toLowerCase() === "html") return false;
997
+
998
+ if (style?.cursor && interactiveCursors.has(style.cursor)) return true;
999
+
1000
+ return false;
1001
+ }
1002
+ // Disabled for now, since it adds too many false positives
1003
+ // let isInteractiveCursor = doesElementHaveInteractivePointer(element);
1004
+
1005
+ // // Genius fix for almost all interactive elements
1006
+ // if (isInteractiveCursor) {
1007
+ // return true;
1008
+ // }
1009
+
1010
+ const interactiveElements = new Set([
1011
+ "a", // Links
1012
+ "button", // Buttons
1013
+ "input", // All input types (text, checkbox, radio, etc.)
1014
+ "select", // Dropdown menus
1015
+ "textarea", // Text areas
1016
+ "details", // Expandable details
1017
+ "summary", // Summary element (clickable part of details)
1018
+ "label", // Form labels (often clickable)
1019
+ "option", // Select options
1020
+ "optgroup", // Option groups
1021
+ "fieldset", // Form fieldsets (can be interactive with legend)
1022
+ "legend", // Fieldset legends
1023
+ ]);
1024
+
1025
+ // Define explicit disable attributes and properties
1026
+ const explicitDisableTags = new Set([
1027
+ 'disabled', // Standard disabled attribute
1028
+ // 'aria-disabled', // ARIA disabled state
1029
+ // 'readonly', // Read-only state
1030
+ // 'aria-readonly', // ARIA read-only state
1031
+ // 'aria-hidden', // Hidden from accessibility
1032
+ // 'hidden', // Hidden attribute
1033
+ // 'inert', // Inert attribute
1034
+ // 'aria-inert', // ARIA inert state
1035
+ // 'tabindex="-1"', // Removed from tab order
1036
+ // 'aria-hidden="true"' // Hidden from screen readers
1037
+ ]);
1038
+
1039
+ // Check for non-interactive cursor
1040
+ if (style?.cursor && nonInteractiveCursors.has(style.cursor)) {
1041
+ return false;
1042
+ }
1043
+
1044
+ // handle inputs, select, checkbox, radio, textarea, button and make sure they are not cursor style disabled/not-allowed
1045
+ if (interactiveElements.has(tagName)) {
1046
+ // Check for explicit disable attributes
1047
+ for (const disableTag of explicitDisableTags) {
1048
+ if (element.hasAttribute(disableTag) ||
1049
+ element.getAttribute(disableTag) === 'true' ||
1050
+ element.getAttribute(disableTag) === '') {
1051
+ return false;
1052
+ }
1053
+ }
1054
+
1055
+ // Check for disabled property on form elements
1056
+ if (element.disabled) {
1057
+ return false;
1058
+ }
1059
+
1060
+ // Don't mark as non-interactive yet
1061
+ // Check for readonly property on form elements
1062
+ if (element.readOnly) {
1063
+ // return false;
1064
+ }
1065
+
1066
+ // Check for inert property
1067
+ if (element.inert) {
1068
+ return false;
1069
+ }
1070
+
1071
+ return true;
1072
+ }
1073
+
1074
+ const role = element.getAttribute("role");
1075
+ const ariaRole = element.getAttribute("aria-role");
1076
+
1077
+ // Check for contenteditable attribute
1078
+ if (element.getAttribute("contenteditable") === "true" || element.isContentEditable) {
1079
+ return true;
1080
+ }
1081
+
1082
+ // Added enhancement to capture dropdown interactive elements
1083
+ if (hasAnyClassName(element, buttonClassNames) ||
1084
+ hasAnyClassName(element, interactiveClassNames) ||
1085
+ element.getAttribute('data-index') ||
1086
+ element.getAttribute('data-toggle') === 'dropdown' ||
1087
+ element.getAttribute('aria-haspopup')) {
1088
+ return true;
1089
+ }
1090
+
1091
+
1092
+ const interactiveRoles = new Set([
1093
+ 'button', // Directly clickable element
1094
+ 'link', // Clickable link
1095
+ 'menuitem', // Clickable menu item
1096
+ 'menuitemradio', // Radio-style menu item (selectable)
1097
+ 'menuitemcheckbox', // Checkbox-style menu item (toggleable)
1098
+ 'radio', // Radio button (selectable)
1099
+ 'checkbox', // Checkbox (toggleable)
1100
+ 'tab', // Tab (clickable to switch content)
1101
+ 'switch', // Toggle switch (clickable to change state)
1102
+ 'slider', // Slider control (draggable)
1103
+ 'spinbutton', // Number input with up/down controls
1104
+ 'combobox', // Dropdown with text input
1105
+ 'searchbox', // Search input field
1106
+ 'textbox', // Text input field
1107
+ 'listbox', // Selectable list
1108
+ 'option', // Selectable option in a list
1109
+ 'scrollbar' // Scrollable control
1110
+ ]);
1111
+
1112
+
1113
+ // Basic role/attribute checks
1114
+ const hasInteractiveRole =
1115
+ (role && interactiveRoles.has(role)) ||
1116
+ (ariaRole && interactiveRoles.has(ariaRole));
1117
+
1118
+ if (hasInteractiveRole) return true;
1119
+
1120
+ const listenedEvents = getCachedNodeEventListeners(element);
1121
+ if (listenedEvents && listenedEvents.length > 0) {
1122
+ for (const eventType of INTERACTION_EVENTS) {
1123
+ if (listenedEvents.includes(eventType)) {
1124
+ return true;
1125
+ }
1126
+ }
1127
+ }
1128
+
1129
+ return false
1130
+ }
1131
+
1132
+
1133
+ /**
1134
+ * Checks if an element is the topmost element at its position.
1135
+ *
1136
+ * @param {HTMLElement} element - The element to check.
1137
+ * @returns {boolean} Whether the element is the topmost element at its position.
1138
+ */
1139
+ function isTopElement(element) {
1140
+ if (element.tagName.toLowerCase() === "input" && element.type === "date") {
1141
+ return true;
1142
+ }
1143
+ // Special case: when viewportExpansion is -1, consider all elements as "top" elements
1144
+ if (viewportExpansion === -1) {
1145
+ return true;
1146
+ }
1147
+
1148
+ const rects = getCachedClientRects(element); // Replace element.getClientRects()
1149
+
1150
+ if (!rects || rects.length === 0) {
1151
+ return false; // No geometry, cannot be top
1152
+ }
1153
+
1154
+ let isAnyRectInViewport = false;
1155
+ for (const rect of rects) {
1156
+ // Use the same logic as isInExpandedViewport check
1157
+ if (rect.width > 0 && rect.height > 0 && !( // Only check non-empty rects
1158
+ rect.bottom < -viewportExpansion ||
1159
+ rect.top > window.innerHeight + viewportExpansion ||
1160
+ rect.right < -viewportExpansion ||
1161
+ rect.left > window.innerWidth + viewportExpansion
1162
+ )) {
1163
+ isAnyRectInViewport = true;
1164
+ break;
1165
+ }
1166
+ }
1167
+
1168
+ if (!isAnyRectInViewport) {
1169
+ return false; // All rects are outside the viewport area
1170
+ }
1171
+
1172
+
1173
+ // Find the correct document context and root element
1174
+ let doc = element.ownerDocument;
1175
+
1176
+ // If we're in an iframe, elements are considered top by default
1177
+ if (doc !== window.document) {
1178
+ return true;
1179
+ }
1180
+
1181
+ // For shadow DOM, we need to check within its own root context
1182
+ const shadowRoot = element.getRootNode();
1183
+ if (shadowRoot instanceof ShadowRoot) {
1184
+ const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
1185
+ const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
1186
+
1187
+ try {
1188
+ const topEl = shadowRoot.elementFromPoint(centerX, centerY);
1189
+ if (!topEl) return false;
1190
+
1191
+ let current = topEl;
1192
+ while (current && current !== shadowRoot) {
1193
+ if (current === element) return true;
1194
+ current = current.parentElement;
1195
+ }
1196
+ return false;
1197
+ } catch (e) {
1198
+ return true;
1199
+ }
1200
+ }
1201
+
1202
+ // For elements in viewport, check if they're topmost
1203
+ const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
1204
+ const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
1205
+
1206
+ try {
1207
+ const topEl = document.elementFromPoint(centerX, centerY);
1208
+ if (!topEl) return false;
1209
+
1210
+ let current = topEl;
1211
+ while (current && current !== document.documentElement) {
1212
+ if (current === element) return true;
1213
+ current = current.parentElement;
1214
+ }
1215
+ return false;
1216
+ } catch (e) {
1217
+ return true;
1218
+ }
1219
+ }
1220
+
1221
+ /**
1222
+ * Checks if an element is within the expanded viewport.
1223
+ *
1224
+ * @param {HTMLElement} element - The element to check.
1225
+ * @param {number} viewportExpansion - The viewport expansion.
1226
+ * @returns {boolean} Whether the element is within the expanded viewport.
1227
+ */
1228
+ function isInExpandedViewport(element, viewportExpansion) {
1229
+ if (viewportExpansion === -1) {
1230
+ return true;
1231
+ }
1232
+
1233
+ const rects = element.getClientRects(); // Use getClientRects
1234
+
1235
+ if (!rects || rects.length === 0) {
1236
+ // Fallback to getBoundingClientRect if getClientRects is empty,
1237
+ // useful for elements like <svg> that might not have client rects but have a bounding box.
1238
+ const boundingRect = getCachedBoundingRect(element);
1239
+ if (!boundingRect || boundingRect.width === 0 || boundingRect.height === 0) {
1240
+ return false;
1241
+ }
1242
+ return !(
1243
+ boundingRect.bottom < -viewportExpansion ||
1244
+ boundingRect.top > window.innerHeight + viewportExpansion ||
1245
+ boundingRect.right < -viewportExpansion ||
1246
+ boundingRect.left > window.innerWidth + viewportExpansion
1247
+ );
1248
+ }
1249
+
1250
+ // Check if *any* client rect is within the viewport
1251
+ for (const rect of rects) {
1252
+ if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects
1253
+
1254
+ if (!(
1255
+ rect.bottom < -viewportExpansion ||
1256
+ rect.top > window.innerHeight + viewportExpansion ||
1257
+ rect.right < -viewportExpansion ||
1258
+ rect.left > window.innerWidth + viewportExpansion
1259
+ )) {
1260
+ return true; // Found at least one rect in the viewport
1261
+ }
1262
+ }
1263
+
1264
+ return false; // No rects were found in the viewport
1265
+ }
1266
+
1267
+ // /**
1268
+ // * Gets the effective scroll of an element.
1269
+ // *
1270
+ // * @param {HTMLElement} element - The element to get the effective scroll for.
1271
+ // * @returns {Object} The effective scroll of the element.
1272
+ // */
1273
+ // function getEffectiveScroll(element) {
1274
+ // let currentEl = element;
1275
+ // let scrollX = 0;
1276
+ // let scrollY = 0;
1277
+
1278
+ // while (currentEl && currentEl !== document.documentElement) {
1279
+ // if (currentEl.scrollLeft || currentEl.scrollTop) {
1280
+ // scrollX += currentEl.scrollLeft;
1281
+ // scrollY += currentEl.scrollTop;
1282
+ // }
1283
+ // currentEl = currentEl.parentElement;
1284
+ // }
1285
+
1286
+ // scrollX += window.scrollX;
1287
+ // scrollY += window.scrollY;
1288
+
1289
+ // return { scrollX, scrollY };
1290
+ // }
1291
+
1292
+ /**
1293
+ * Checks if an element is an interactive candidate.
1294
+ *
1295
+ * @param {HTMLElement} element - The element to check.
1296
+ * @returns {boolean} Whether the element is an interactive candidate.
1297
+ */
1298
+ function isInteractiveCandidate(element) {
1299
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
1300
+
1301
+ const tagName = element.tagName.toLowerCase();
1302
+
1303
+ // Fast-path for common interactive elements
1304
+ const interactiveElements = new Set([
1305
+ "a", "button", "input", "select", "textarea", "details", "summary", "label"
1306
+ ]);
1307
+
1308
+ if (interactiveElements.has(tagName)) return true;
1309
+
1310
+ // Quick attribute checks without getting full lists
1311
+ const hasQuickInteractiveAttr = element.hasAttribute("onclick") ||
1312
+ element.hasAttribute("role") ||
1313
+ element.hasAttribute("tabindex") ||
1314
+ element.hasAttribute("aria-") ||
1315
+ element.hasAttribute("data-action") ||
1316
+ element.getAttribute("contenteditable") === "true";
1317
+
1318
+ return hasQuickInteractiveAttr;
1319
+ }
1320
+
1321
+ // --- Define constants for distinct interaction check ---
1322
+ const DISTINCT_INTERACTIVE_TAGS = new Set([
1323
+ 'a', 'button', 'input', 'select', 'textarea', 'summary', 'details', 'label', 'option'
1324
+ ]);
1325
+ const INTERACTIVE_ROLES = new Set([
1326
+ 'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox',
1327
+ 'radio', 'checkbox', 'tab', 'switch', 'slider', 'spinbutton',
1328
+ 'combobox', 'searchbox', 'textbox', 'listbox', 'option', 'scrollbar'
1329
+ ]);
1330
+
1331
+
1332
+ /**
1333
+ * Heuristically determines if an element should be considered as independently interactive,
1334
+ * even if it's nested inside another interactive container.
1335
+ *
1336
+ * This function helps detect deeply nested actionable elements (e.g., menu items within a button)
1337
+ * that may not be picked up by strict interactivity checks.
1338
+ *
1339
+ * @param {HTMLElement} element - The element to check.
1340
+ * @returns {boolean} Whether the element is heuristically interactive.
1341
+ */
1342
+ function isHeuristicallyInteractive(element) {
1343
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
1344
+
1345
+ // Skip non-visible elements early for performance
1346
+ if (!isElementVisible(element)) return false;
1347
+
1348
+ // Check for common attributes that often indicate interactivity
1349
+ const hasInteractiveAttributes =
1350
+ element.hasAttribute('role') ||
1351
+ element.hasAttribute('tabindex') ||
1352
+ element.hasAttribute('onclick') ||
1353
+ typeof element.onclick === 'function';
1354
+
1355
+ // Check for semantic class names suggesting interactivity
1356
+ const hasInteractiveClass = heuristicClassPattern.test(element.className || '');
1357
+
1358
+ // Determine whether the element is inside a known interactive container
1359
+ const isInKnownContainer = Boolean(
1360
+ element.closest(containerSelectors)
1361
+ );
1362
+
1363
+ // Ensure the element has at least one visible child (to avoid marking empty wrappers)
1364
+ const hasVisibleChildren = [...element.children].some(isElementVisible);
1365
+
1366
+ // Avoid highlighting elements whose parent is <body> (top-level wrappers)
1367
+ const isParentBody = element.parentElement && element.parentElement.isSameNode(document.body);
1368
+
1369
+ return (
1370
+ (isInteractiveElement(element) || hasInteractiveAttributes || hasInteractiveClass) &&
1371
+ hasVisibleChildren &&
1372
+ isInKnownContainer &&
1373
+ !isParentBody
1374
+ );
1375
+ }
1376
+
1377
+
1378
+ /**
1379
+ * Checks if an element likely represents a distinct interaction
1380
+ * separate from its parent (if the parent is also interactive).
1381
+ *
1382
+ * @param {HTMLElement} element - The element to check.
1383
+ * @param {Object} nodeData - The node data object.
1384
+ * @returns {boolean} Whether the element is a distinct interaction.
1385
+ */
1386
+ function isElementDistinctInteraction(element, nodeData) {
1387
+ if (nodeData.isScrollable) {
1388
+ return true;
1389
+ }
1390
+
1391
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) {
1392
+ return false;
1393
+ }
1394
+
1395
+ const tagName = element.tagName.toLowerCase();
1396
+ const role = element.getAttribute('role');
1397
+
1398
+ // Check if it's an iframe - always distinct boundary
1399
+ if (tagName === 'iframe') {
1400
+ return true;
1401
+ }
1402
+
1403
+ // Check tag name
1404
+ if (DISTINCT_INTERACTIVE_TAGS.has(tagName)) {
1405
+ return true;
1406
+ }
1407
+ // Check interactive roles
1408
+ if (role && INTERACTIVE_ROLES.has(role)) {
1409
+ return true;
1410
+ }
1411
+ // Check contenteditable
1412
+ if (element.isContentEditable || element.getAttribute('contenteditable') === 'true') {
1413
+ return true;
1414
+ }
1415
+ // Check for common testing/automation attributes
1416
+ if (element.hasAttribute('data-testid') || element.hasAttribute('data-cy') || element.hasAttribute('data-test')) {
1417
+ return true;
1418
+ }
1419
+ // Check for explicit onclick handler (attribute or property)
1420
+ if (element.hasAttribute('onclick') || typeof element.onclick === 'function') {
1421
+ return true;
1422
+ }
1423
+
1424
+ if (element.hasAttribute('aria-haspopup')) {
1425
+ return true;
1426
+ }
1427
+
1428
+ if (hasAnyClassName(element, interactiveClassNames)) {
1429
+ return true;
1430
+ }
1431
+
1432
+ // return false
1433
+
1434
+ // Check for other common interaction event listeners
1435
+ try {
1436
+ const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
1437
+ if (typeof getEventListenersForNode === 'function') {
1438
+ const listeners = getEventListenersForNode(element);
1439
+ const interactionEvents = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave', 'keydown', 'keyup', 'submit', 'change', 'focus', 'blur'];
1440
+ for (const eventType of interactionEvents) {
1441
+ for (const listener of listeners) {
1442
+ if (listener.type === eventType) {
1443
+ return true; // Found a common interaction listener
1444
+ }
1445
+ }
1446
+ }
1447
+ }
1448
+ // Fallback: Check common event attributes if getEventListeners is not available (getEventListenersForNode doesn't work in page.evaluate context)
1449
+ const commonEventAttrs = ['onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', 'onsubmit', 'onmouseenter', 'onmouseleave', 'onchange', 'oninput', 'onfocus', 'onblur'];
1450
+ if (commonEventAttrs.some(attr => element.hasAttribute(attr))) {
1451
+ return true;
1452
+ }
1453
+ } catch (e) {
1454
+ // console.warn(\`Could not check event listeners for \${element.tagName}:\`, e);
1455
+ // If checking listeners fails, rely on other checks
1456
+ }
1457
+
1458
+
1459
+
1460
+ // if the element is not strictly interactive but appears clickable based on heuristic signals
1461
+ if (isHeuristicallyInteractive(element)) {
1462
+ return true;
1463
+ }
1464
+
1465
+ // Default to false: if it's interactive but doesn't match above,
1466
+ // assume it triggers the same action as the parent.
1467
+ return false;
1468
+ }
1469
+ // --- End distinct interaction check ---
1470
+
1471
+ /**
1472
+ * Handles the logic for deciding whether to highlight an element and performing the highlight.
1473
+ * @param {
1474
+ {
1475
+ tagName: string;
1476
+ attributes: Record<string, string>;
1477
+ xpath: any;
1478
+ children: never[];
1479
+ isVisible?: boolean;
1480
+ isTopElement?: boolean;
1481
+ isInteractive?: boolean;
1482
+ isInViewport?: boolean;
1483
+ highlightIndex?: number;
1484
+ shadowRoot?: boolean;
1485
+ }} nodeData - The node data object.
1486
+ * @param {HTMLElement} node - The node to highlight.
1487
+ * @param {HTMLElement | null} parentIframe - The parent iframe node.
1488
+ * @param {boolean} isParentHighlighted - Whether the parent node is highlighted.
1489
+ * @returns {boolean} Whether the element was highlighted.
1490
+ */
1491
+ function handleHighlighting(nodeData, node, parentIframe, isParentHighlighted) {
1492
+ if (!nodeData.isInteractive) return false; // Not interactive, definitely don't highlight
1493
+
1494
+ let shouldHighlight = false;
1495
+ if (!isParentHighlighted) {
1496
+ // Parent wasn't highlighted, this interactive node can be highlighted.
1497
+ shouldHighlight = true;
1498
+ } else {
1499
+ // Parent *was* highlighted. Only highlight this node if it represents a distinct interaction.
1500
+ if (isElementDistinctInteraction(node, nodeData)) {
1501
+ shouldHighlight = true;
1502
+ } else {
1503
+ // console.log(\`Skipping highlight for \${nodeData.tagName} (parent highlighted)\`);
1504
+ shouldHighlight = false;
1505
+ }
1506
+ }
1507
+
1508
+ if (shouldHighlight) {
1509
+ const attributeNames = node.getAttributeNames?.() || [];
1510
+ for (const name of attributeNames) {
1511
+ const value = node.getAttribute(name);
1512
+ nodeData.attributes[name] = value;
1513
+ }
1514
+
1515
+ // Check viewport status before assigning index and highlighting
1516
+ if (nodeData.isInViewport === undefined) {
1517
+ nodeData.isInViewport = isInExpandedViewport(node, viewportExpansion);
1518
+ }
1519
+
1520
+ // When viewportExpansion is -1, all interactive elements should get a highlight index
1521
+ // regardless of viewport status
1522
+ if (nodeData.isInViewport || viewportExpansion === -1) {
1523
+ nodeData.highlightIndex = highlightIndex++;
1524
+
1525
+ if (doHighlightElements) {
1526
+ if (focusHighlightIndex >= 0) {
1527
+ if (focusHighlightIndex === nodeData.highlightIndex) {
1528
+ highlightElement(node, nodeData.highlightIndex, parentIframe);
1529
+ }
1530
+ } else {
1531
+ highlightElement(node, nodeData.highlightIndex, parentIframe);
1532
+ }
1533
+ return true; // Successfully highlighted
1534
+ }
1535
+ } else {
1536
+ // console.log(\`Skipping highlight for \${nodeData.tagName} (outside viewport)\`);
1537
+ }
1538
+ }
1539
+
1540
+ return false; // Did not highlight
1541
+ }
1542
+
1543
+ function isElementScrollable(element) {
1544
+ const listenedEvents = getCachedNodeEventListeners(element);
1545
+ if (listenedEvents && listenedEvents.includes('scroll')) {
1546
+ const hasScrollableX = element.scrollWidth > element.clientWidth;
1547
+ const hasScrollableY = element.scrollHeight > element.clientHeight;
1548
+ return hasScrollableX || hasScrollableY;
1549
+ }
1550
+
1551
+ const style = getCachedComputedStyle(element);
1552
+ const hasScrollableX = ['auto', 'scroll'].includes(style.overflowX) &&
1553
+ element.scrollWidth > element.clientWidth;
1554
+ const hasScrollableY = ['auto', 'scroll'].includes(style.overflowY) &&
1555
+ element.scrollHeight > element.clientHeight;
1556
+ return hasScrollableX || hasScrollableY;
1557
+ }
1558
+
1559
+ /**
1560
+ * Creates a node data object for a given node and its descendants.
1561
+ *
1562
+ * @param {HTMLElement} node - The node to process.
1563
+ * @param {HTMLElement | null} parentIframe - The parent iframe node.
1564
+ * @param {boolean} isParentHighlighted - Whether the parent node is highlighted.
1565
+ * @returns {string | null} The ID of the node data object, or null if the node is not processed.
1566
+ */
1567
+ function buildDomTree(node, parentIframe = null, isParentHighlighted = false) {
1568
+
1569
+ // Fast rejection checks first
1570
+ if (!node || node.id === HIGHLIGHT_CONTAINER_ID) {
1571
+ return null;
1572
+ }
1573
+
1574
+ if (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE) {
1575
+ return null;
1576
+ }
1577
+
1578
+ // Special handling for root node (body)
1579
+ if (node === document.body) {
1580
+ const nodeData = {
1581
+ tagName: 'body',
1582
+ attributes: {},
1583
+ xpath: '/body',
1584
+ children: [],
1585
+ };
1586
+
1587
+ // Process children of body
1588
+ for (const child of node.childNodes) {
1589
+ const domElement = buildDomTree(child, parentIframe, false); // Body's children have no highlighted parent initially
1590
+ if (domElement) nodeData.children.push(domElement);
1591
+ }
1592
+
1593
+ const id = \`\${ID.current++}\`;
1594
+ DOM_HASH_MAP[id] = nodeData;
1595
+ return id;
1596
+ }
1597
+
1598
+ // Process text nodes
1599
+ if (node.nodeType === Node.TEXT_NODE) {
1600
+ const textContent = node.textContent?.trim();
1601
+ if (!textContent) {
1602
+ return null;
1603
+ }
1604
+
1605
+ // Only check visibility for text nodes that might be visible
1606
+ const parentElement = node.parentElement;
1607
+ if (!parentElement || parentElement.tagName.toLowerCase() === 'script') {
1608
+ return null;
1609
+ }
1610
+
1611
+ const id = \`\${ID.current++}\`;
1612
+ DOM_HASH_MAP[id] = {
1613
+ type: "TEXT_NODE",
1614
+ text: textContent,
1615
+ isVisible: isTextNodeVisible(node),
1616
+ };
1617
+ return id;
1618
+ }
1619
+
1620
+ // Quick checks for element nodes
1621
+ if (node.nodeType === Node.ELEMENT_NODE && !isElementAccepted(node)) {
1622
+ return null;
1623
+ }
1624
+
1625
+ /**
1626
+ * @type {
1627
+ {
1628
+ tagName: string;
1629
+ attributes: Record<string, string | null>;
1630
+ xpath: any;
1631
+ children: never[];
1632
+ isVisible?: boolean;
1633
+ isTopElement?: boolean;
1634
+ isInteractive?: boolean;
1635
+ isInViewport?: boolean;
1636
+ highlightIndex?: number;
1637
+ shadowRoot?: boolean;
1638
+ }
1639
+ } nodeData - The node data object.
1640
+ */
1641
+ const nodeData = {
1642
+ tagName: node.tagName.toLowerCase(),
1643
+ attributes: {},
1644
+ xpath: getXPathTree(node, true),
1645
+ children: [],
1646
+ };
1647
+
1648
+ // Get attributes for interactive elements or potential text containers
1649
+ if (node.tagName.toLowerCase() === 'iframe' || node.tagName.toLowerCase() === 'body') {
1650
+ const attributeNames = node.getAttributeNames?.() || [];
1651
+ for (const name of attributeNames) {
1652
+ const value = node.getAttribute(name);
1653
+ nodeData.attributes[name] = value;
1654
+ }
1655
+ }
1656
+
1657
+ let nodeWasHighlighted = false;
1658
+ // Perform visibility, interactivity, and highlighting checks
1659
+ if (node.nodeType === Node.ELEMENT_NODE) {
1660
+ if (alwaysHighlightFileInput && node.tagName.toLowerCase() === 'input' && node.type === 'file') {
1661
+ nodeData.isTopElement = true;
1662
+ if (nodeData.isTopElement) {
1663
+ nodeData.isInteractive = true;
1664
+ nodeData.isInViewport = true; // File inputs should always be considered in viewport
1665
+ // Call the dedicated highlighting function
1666
+ nodeWasHighlighted = handleHighlighting(nodeData, node, parentIframe, isParentHighlighted);
1667
+ }
1668
+ } else {
1669
+ nodeData.isVisible = isElementVisible(node); // isElementVisible uses offsetWidth/Height, which is fine
1670
+
1671
+ if (nodeData.isVisible) {
1672
+ nodeData.isTopElement = isTopElement(node);
1673
+ if (nodeData.isTopElement) {
1674
+ let isScrollable = isElementScrollable(node);
1675
+ nodeData.isInteractive = isInteractiveElement(node) || isScrollable;
1676
+ nodeData.isScrollable = isScrollable;
1677
+ nodeData.markAsClickable = shouldMarkAsClickable(node);
1678
+ // Call the dedicated highlighting function
1679
+ nodeWasHighlighted = handleHighlighting(nodeData, node, parentIframe, isParentHighlighted);
1680
+ }
1681
+ }
1682
+ }
1683
+ }
1684
+
1685
+ // Process children, with special handling for iframes and rich text editors
1686
+ if (node.tagName) {
1687
+ const tagName = node.tagName.toLowerCase();
1688
+
1689
+ // Handle iframes
1690
+ if (tagName === "iframe") {
1691
+ try {
1692
+ const iframeDoc = node.contentDocument || node.contentWindow?.document;
1693
+ if (iframeDoc) {
1694
+ for (const child of iframeDoc.childNodes) {
1695
+ const domElement = buildDomTree(child, node, false);
1696
+ if (domElement) nodeData.children.push(domElement);
1697
+ }
1698
+ }
1699
+ } catch (e) {
1700
+ console.warn("Unable to access iframe:", e);
1701
+ }
1702
+ }
1703
+ // Handle rich text editors and contenteditable elements
1704
+ else if (
1705
+ node.isContentEditable ||
1706
+ node.getAttribute("contenteditable") === "true" ||
1707
+ node.id === "tinymce" ||
1708
+ node.classList.contains("mce-content-body") ||
1709
+ (tagName === "body" && node.getAttribute("data-id")?.startsWith("mce_"))
1710
+ ) {
1711
+ // Process all child nodes to capture formatted text
1712
+ for (const child of node.childNodes) {
1713
+ const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted);
1714
+ if (domElement) nodeData.children.push(domElement);
1715
+ }
1716
+ }
1717
+ else {
1718
+ // Handle shadow DOM
1719
+ if (node.shadowRoot) {
1720
+ nodeData.shadowRoot = true;
1721
+ for (const child of node.shadowRoot.childNodes) {
1722
+ const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted);
1723
+ if (domElement) nodeData.children.push(domElement);
1724
+ }
1725
+ }
1726
+ // Handle regular elements
1727
+ for (const child of node.childNodes) {
1728
+ // Pass the highlighted status of the *current* node to its children
1729
+ const passHighlightStatusToChild = nodeWasHighlighted || isParentHighlighted;
1730
+ const domElement = buildDomTree(child, parentIframe, passHighlightStatusToChild);
1731
+ if (domElement) nodeData.children.push(domElement);
1732
+ }
1733
+ }
1734
+ }
1735
+
1736
+ const id = \`\${ID.current++}\`;
1737
+ DOM_HASH_MAP[id] = nodeData;
1738
+ return id;
1739
+ }
1740
+
1741
+ const rootId = buildDomTree(document.body);
1742
+
1743
+ // Clear the cache before starting
1744
+ DOM_CACHE.clearCache();
1745
+
1746
+ return { rootId, map: DOM_HASH_MAP };
1747
+ }
1748
+ `,ee=`((args = {
1749
+ doHighlightElements: true,
1750
+ focusHighlightIndex: -1,
1751
+ viewportExpansion: 0,
1752
+ debugMode: false,
1753
+ interactiveClassNames: [],
1754
+ alwaysHighlightFileInput: false,
1755
+ sameRectIoUThreshold: 0.85,
1756
+ }) => {
1757
+ // Default threshold if not provided
1758
+ const sameRectIoUThreshold = args.sameRectIoUThreshold ?? 0.85;
1759
+ const EVENT_LISTENER_MAPPING = {
1760
+ 'onclick': 'click',
1761
+ 'onmousedown': 'mousedown',
1762
+ 'onmouseup': 'mouseup',
1763
+ 'ondblclick': 'dblclick',
1764
+ 'onmouseenter': 'mouseenter',
1765
+ 'onmouseleave': 'mouseleave',
1766
+ 'onmousemove': 'mousemove',
1767
+ 'onmouseout': 'mouseout',
1768
+ 'onmouseover': 'mouseover',
1769
+ 'onmousewheel': 'mousewheel',
1770
+ 'onscroll': 'scroll',
1771
+ 'onselect': 'select',
1772
+ 'onchange': 'change',
1773
+ 'onfocus': 'focus',
1774
+ 'onblur': 'blur',
1775
+ 'onkeydown': 'keydown',
1776
+ 'onkeyup': 'keyup',
1777
+ 'onkeypress': 'keypress',
1778
+ 'oninput': 'input',
1779
+ };
1780
+ const INTERACTION_EVENTS = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave'];
1781
+ const { doHighlightElements, focusHighlightIndex, viewportExpansion, debugMode, interactiveClassNames, alwaysHighlightFileInput, grayscaleImage, uniformityTolerance = 32, captureDebugSnapshots = false, onSnapshot, onLog, phase, elementData: inputElementData, } = args;
1782
+ // Helper to stream logs if callback provided
1783
+ const streamLog = (msg) => {
1784
+ if (onLog)
1785
+ onLog(msg);
1786
+ };
1787
+ streamLog(\`[dom-tree] Starting phase=\${phase || 'legacy'}, grayscaleImage=\${!!grayscaleImage}, captureDebugSnapshots=\${captureDebugSnapshots}\`);
1788
+ const buttonClassNames = ['button', 'dropdown-toggle'];
1789
+ const cursorPointerClassNames = ['cursor-pointer', 'tw-cursor-pointer', 'clickable'];
1790
+ const heuristicClassPattern = /\\b(btn|const clickable|menu|item|entry|link)\\b/i;
1791
+ const containerSelectors = 'button,a,[role="button"],.menu,.dropdown,.list,.toolbar';
1792
+ let highlightIndex = 0; // Reset highlight index
1793
+ /**
1794
+ * Helper function to check if element has any of the specified class names.
1795
+ */
1796
+ function hasAnyClassName(element, classNames) {
1797
+ if (!element.classList || !classNames || classNames.length === 0)
1798
+ return false;
1799
+ return classNames.some(className => element.classList.contains(className));
1800
+ }
1801
+ // Add caching mechanisms at the top level
1802
+ const DOM_CACHE = {
1803
+ boundingRects: new WeakMap(),
1804
+ clientRects: new WeakMap(),
1805
+ computedStyles: new WeakMap(),
1806
+ nodeEventListeners: new WeakMap(),
1807
+ clearCache: () => {
1808
+ DOM_CACHE.boundingRects = new WeakMap();
1809
+ DOM_CACHE.clientRects = new WeakMap();
1810
+ DOM_CACHE.computedStyles = new WeakMap();
1811
+ DOM_CACHE.nodeEventListeners = new WeakMap();
1812
+ }
1813
+ };
1814
+ /**
1815
+ * Gets the cached bounding rect for an element.
1816
+ */
1817
+ function getCachedBoundingRect(element) {
1818
+ if (!element)
1819
+ return null;
1820
+ if (DOM_CACHE.boundingRects.has(element)) {
1821
+ return DOM_CACHE.boundingRects.get(element);
1822
+ }
1823
+ const rect = element.getBoundingClientRect();
1824
+ if (rect) {
1825
+ DOM_CACHE.boundingRects.set(element, rect);
1826
+ }
1827
+ return rect;
1828
+ }
1829
+ /**
1830
+ * Gets the cached computed style for an element.
1831
+ */
1832
+ function getCachedComputedStyle(element) {
1833
+ if (!element)
1834
+ return null;
1835
+ if (DOM_CACHE.computedStyles.has(element)) {
1836
+ return DOM_CACHE.computedStyles.get(element);
1837
+ }
1838
+ const style = window.getComputedStyle(element);
1839
+ if (style) {
1840
+ DOM_CACHE.computedStyles.set(element, style);
1841
+ }
1842
+ return style;
1843
+ }
1844
+ /**
1845
+ * Gets the cached client rects for an element.
1846
+ */
1847
+ function getCachedClientRects(element) {
1848
+ if (!element)
1849
+ return null;
1850
+ if (DOM_CACHE.clientRects.has(element)) {
1851
+ return DOM_CACHE.clientRects.get(element);
1852
+ }
1853
+ const rects = element.getClientRects();
1854
+ if (rects) {
1855
+ DOM_CACHE.clientRects.set(element, rects);
1856
+ }
1857
+ return rects;
1858
+ }
1859
+ /**
1860
+ * Gets the event listeners for a node.
1861
+ */
1862
+ function getNodeEventListeners(element) {
1863
+ const set = new Set();
1864
+ try {
1865
+ if (typeof getEventListeners === 'function') {
1866
+ const listeners = getEventListeners(element);
1867
+ for (const eventType in listeners) {
1868
+ if (listeners[eventType] && listeners[eventType].length > 0) {
1869
+ set.add(eventType);
1870
+ }
1871
+ }
1872
+ }
1873
+ const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
1874
+ if (typeof getEventListenersForNode === 'function') {
1875
+ const listeners = getEventListenersForNode(element);
1876
+ for (const listener of listeners) {
1877
+ if (listener.type) {
1878
+ set.add(listener.type);
1879
+ }
1880
+ }
1881
+ }
1882
+ for (const attr in EVENT_LISTENER_MAPPING) {
1883
+ if (element.hasAttribute(attr) || typeof element[attr] === 'function') {
1884
+ set.add(EVENT_LISTENER_MAPPING[attr]);
1885
+ }
1886
+ }
1887
+ }
1888
+ catch (error) {
1889
+ // Silently ignore errors
1890
+ }
1891
+ const listenedEvents = Array.from(set);
1892
+ return listenedEvents;
1893
+ }
1894
+ function getCachedNodeEventListeners(element) {
1895
+ if (!element)
1896
+ return null;
1897
+ if (DOM_CACHE.nodeEventListeners.has(element)) {
1898
+ return DOM_CACHE.nodeEventListeners.get(element);
1899
+ }
1900
+ const listenedEvents = getNodeEventListeners(element);
1901
+ if (listenedEvents) {
1902
+ DOM_CACHE.nodeEventListeners.set(element, listenedEvents);
1903
+ }
1904
+ return listenedEvents;
1905
+ }
1906
+ // ============================================================================
1907
+ // Action Intent Predicates
1908
+ // ============================================================================
1909
+ const CLICKABLE_TAGS = new Set(['a', 'button', 'summary', 'label', 'option', 'optgroup']);
1910
+ const CLICKABLE_ROLES = new Set([
1911
+ 'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox',
1912
+ 'radio', 'checkbox', 'tab', 'switch', 'option', 'treeitem'
1913
+ ]);
1914
+ const CLICK_EVENTS = ['click', 'mousedown', 'mouseup', 'dblclick'];
1915
+ const TEXT_INPUT_TYPES = new Set([
1916
+ 'text', 'email', 'password', 'search', 'tel', 'url', 'number',
1917
+ 'date', 'datetime-local', 'month', 'week', 'time'
1918
+ ]);
1919
+ const INPUT_ROLES = new Set(['textbox', 'searchbox', 'spinbutton', 'combobox']);
1920
+ /**
1921
+ * Checks if an element matches the 'click' intent.
1922
+ * Includes: buttons, links, elements with click handlers, clickable roles
1923
+ */
1924
+ function isClickIntentElement(element) {
1925
+ const tagName = element.tagName.toLowerCase();
1926
+ if (CLICKABLE_TAGS.has(tagName))
1927
+ return true;
1928
+ const role = element.getAttribute('role');
1929
+ if (role && CLICKABLE_ROLES.has(role))
1930
+ return true;
1931
+ // Check for click event listeners
1932
+ const listeners = getCachedNodeEventListeners(element);
1933
+ if (listeners?.some(e => CLICK_EVENTS.includes(e)))
1934
+ return true;
1935
+ // Dropdown/popup triggers
1936
+ if (element.getAttribute('aria-haspopup') ||
1937
+ element.getAttribute('data-toggle') === 'dropdown')
1938
+ return true;
1939
+ return false;
1940
+ }
1941
+ /**
1942
+ * Checks if an element matches the 'input' intent.
1943
+ * Includes: text inputs, textareas, contenteditable elements
1944
+ */
1945
+ function isInputIntentElement(element) {
1946
+ const tagName = element.tagName.toLowerCase();
1947
+ if (tagName === 'textarea')
1948
+ return true;
1949
+ if (tagName === 'input') {
1950
+ const type = element.type?.toLowerCase() || 'text';
1951
+ return TEXT_INPUT_TYPES.has(type);
1952
+ }
1953
+ if (element.isContentEditable ||
1954
+ element.getAttribute('contenteditable') === 'true')
1955
+ return true;
1956
+ const role = element.getAttribute('role');
1957
+ return role ? INPUT_ROLES.has(role) : false;
1958
+ }
1959
+ /**
1960
+ * Checks if an element matches the 'scroll' intent.
1961
+ * Delegates to existing isElementScrollable function.
1962
+ */
1963
+ function isScrollIntentElement(element) {
1964
+ // Note: isElementScrollable is defined later but hoisted due to function declaration
1965
+ return isElementScrollable(element);
1966
+ }
1967
+ /**
1968
+ * Checks if an element matches the specified action intent.
1969
+ */
1970
+ function matchesActionIntent(element, intent) {
1971
+ if (intent === 'all')
1972
+ return true;
1973
+ if (intent === 'click')
1974
+ return isClickIntentElement(element);
1975
+ if (intent === 'input')
1976
+ return isInputIntentElement(element);
1977
+ if (intent === 'scroll')
1978
+ return isScrollIntentElement(element);
1979
+ return true;
1980
+ }
1981
+ /**
1982
+ * Hash map of DOM nodes indexed by their highlight index.
1983
+ */
1984
+ const DOM_HASH_MAP = {};
1985
+ const ID = { current: 0 };
1986
+ const HIGHLIGHT_CONTAINER_ID = "playwright-highlight-container";
1987
+ // Add a WeakMap cache for XPath strings
1988
+ const xpathCache = new WeakMap();
1989
+ const debugLogs = [];
1990
+ const debugLog = (msg) => {
1991
+ debugLogs.push(msg);
1992
+ };
1993
+ // ============================================================================
1994
+ // Grayscale Image Label Placement (Dynamic Convolution)
1995
+ // ============================================================================
1996
+ /**
1997
+ * 1D sliding window min/max using monotonic deque (Lemire algorithm).
1998
+ * O(n) time complexity - each element is pushed and popped at most once.
1999
+ *
2000
+ * @param arr - Array of values
2001
+ * @param k - Window size
2002
+ * @returns Object with maxResults and minResults arrays
2003
+ */
2004
+ function slidingWindowMinMax(arr, k) {
2005
+ if (k <= 0 || arr.length === 0 || k > arr.length) {
2006
+ return { maxResults: [], minResults: [] };
2007
+ }
2008
+ const maxDeque = []; // Indices, values in descending order
2009
+ const minDeque = []; // Indices, values in ascending order
2010
+ const maxResults = [];
2011
+ const minResults = [];
2012
+ for (let i = 0; i < arr.length; i++) {
2013
+ const val = arr[i];
2014
+ // Update max deque - remove smaller elements from back
2015
+ while (maxDeque.length > 0 && arr[maxDeque[maxDeque.length - 1]] <= val) {
2016
+ maxDeque.pop();
2017
+ }
2018
+ maxDeque.push(i);
2019
+ // Remove front if outside window
2020
+ if (maxDeque[0] <= i - k)
2021
+ maxDeque.shift();
2022
+ // Update min deque - remove larger elements from back
2023
+ while (minDeque.length > 0 && arr[minDeque[minDeque.length - 1]] >= val) {
2024
+ minDeque.pop();
2025
+ }
2026
+ minDeque.push(i);
2027
+ // Remove front if outside window
2028
+ if (minDeque[0] <= i - k)
2029
+ minDeque.shift();
2030
+ // Record result once window is full
2031
+ if (i >= k - 1) {
2032
+ maxResults.push(arr[maxDeque[0]]);
2033
+ minResults.push(arr[minDeque[0]]);
2034
+ }
2035
+ }
2036
+ return { maxResults, minResults };
2037
+ }
2038
+ /**
2039
+ * Compute 2D min/max over sliding windows within a bounded region.
2040
+ * Uses two-pass decomposition: horizontal pass then vertical pass.
2041
+ *
2042
+ * @param image - Full grayscale image (2D array where image[y][x] is intensity 0-255)
2043
+ * @param regionX - Starting X of the region to scan
2044
+ * @param regionY - Starting Y of the region to scan
2045
+ * @param regionWidth - Width of the region to scan
2046
+ * @param regionHeight - Height of the region to scan
2047
+ * @param labelWidth - Width of the label (kernel width)
2048
+ * @param labelHeight - Height of the label (kernel height)
2049
+ * @returns Object with windowMax and windowMin 2D arrays
2050
+ */
2051
+ function compute2DMinMaxInRegion(image, regionX, regionY, regionWidth, regionHeight, labelWidth, labelHeight) {
2052
+ // Handle edge cases
2053
+ if (regionWidth < labelWidth || regionHeight < labelHeight) {
2054
+ return { windowMax: [], windowMin: [] };
2055
+ }
2056
+ const imageHeight = image.length;
2057
+ const imageWidth = image[0]?.length || 0;
2058
+ // Step 1: Horizontal pass - compute row-wise min/max for each row in region
2059
+ const rowMax = [];
2060
+ const rowMin = [];
2061
+ for (let dy = 0; dy < regionHeight; dy++) {
2062
+ const y = regionY + dy;
2063
+ if (y < 0 || y >= imageHeight) {
2064
+ // Out of bounds - use empty arrays
2065
+ rowMax[dy] = [];
2066
+ rowMin[dy] = [];
2067
+ continue;
2068
+ }
2069
+ // Extract the row slice from the region
2070
+ const rowSlice = [];
2071
+ for (let dx = 0; dx < regionWidth; dx++) {
2072
+ const x = regionX + dx;
2073
+ // Use 0 for out of bounds pixels (they'll fail uniformity check)
2074
+ rowSlice.push(x >= 0 && x < imageWidth ? (image[y][x] ?? 0) : 0);
2075
+ }
2076
+ const { maxResults, minResults } = slidingWindowMinMax(rowSlice, labelWidth);
2077
+ rowMax[dy] = maxResults;
2078
+ rowMin[dy] = minResults;
2079
+ }
2080
+ // Step 2: Vertical pass - compute column-wise min/max on intermediate buffers
2081
+ const resultWidth = regionWidth - labelWidth + 1;
2082
+ const resultHeight = regionHeight - labelHeight + 1;
2083
+ if (resultWidth <= 0 || resultHeight <= 0) {
2084
+ return { windowMax: [], windowMin: [] };
2085
+ }
2086
+ const windowMax = [];
2087
+ const windowMin = [];
2088
+ for (let x = 0; x < resultWidth; x++) {
2089
+ // Extract column from intermediate buffers
2090
+ const colMax = [];
2091
+ const colMin = [];
2092
+ for (let y = 0; y < regionHeight; y++) {
2093
+ colMax.push(rowMax[y]?.[x] ?? 0);
2094
+ colMin.push(rowMin[y]?.[x] ?? 255);
2095
+ }
2096
+ const { maxResults: colMaxResults } = slidingWindowMinMax(colMax, labelHeight);
2097
+ const { minResults: colMinResults } = slidingWindowMinMax(colMin, labelHeight);
2098
+ for (let y = 0; y < resultHeight; y++) {
2099
+ if (!windowMax[y])
2100
+ windowMax[y] = [];
2101
+ if (!windowMin[y])
2102
+ windowMin[y] = [];
2103
+ windowMax[y][x] = colMaxResults[y] ?? 0;
2104
+ windowMin[y][x] = colMinResults[y] ?? 255;
2105
+ }
2106
+ }
2107
+ return { windowMax, windowMin };
2108
+ }
2109
+ /**
2110
+ * Draw a border on the grayscale image.
2111
+ * Used to mark element bounding boxes after they've been processed.
2112
+ *
2113
+ * @param image - Grayscale image (modified in place)
2114
+ * @param x - Left edge of the border
2115
+ * @param y - Top edge of the border
2116
+ * @param width - Width of the bordered region
2117
+ * @param height - Height of the bordered region
2118
+ * @param borderWidth - Width of the border in pixels (default 2)
2119
+ * @param borderColor - Gray value for the border (default 128)
2120
+ */
2121
+ function drawBorderOnImage(image, x, y, width, height, borderWidth = 2, borderColor = 128) {
2122
+ const imageHeight = image.length;
2123
+ const imageWidth = image[0]?.length || 0;
2124
+ const x1 = Math.floor(x);
2125
+ const y1 = Math.floor(y);
2126
+ const x2 = Math.floor(x + width);
2127
+ const y2 = Math.floor(y + height);
2128
+ // Top edge
2129
+ for (let dy = 0; dy < borderWidth; dy++) {
2130
+ const py = y1 + dy;
2131
+ if (py >= 0 && py < imageHeight) {
2132
+ for (let px = x1; px < x2; px++) {
2133
+ if (px >= 0 && px < imageWidth) {
2134
+ image[py][px] = borderColor;
2135
+ }
2136
+ }
2137
+ }
2138
+ }
2139
+ // Bottom edge
2140
+ for (let dy = 0; dy < borderWidth; dy++) {
2141
+ const py = y2 - 1 - dy;
2142
+ if (py >= 0 && py < imageHeight) {
2143
+ for (let px = x1; px < x2; px++) {
2144
+ if (px >= 0 && px < imageWidth) {
2145
+ image[py][px] = borderColor;
2146
+ }
2147
+ }
2148
+ }
2149
+ }
2150
+ // Left edge (excluding corners)
2151
+ for (let py = y1 + borderWidth; py < y2 - borderWidth; py++) {
2152
+ if (py >= 0 && py < imageHeight) {
2153
+ for (let dx = 0; dx < borderWidth; dx++) {
2154
+ const px = x1 + dx;
2155
+ if (px >= 0 && px < imageWidth) {
2156
+ image[py][px] = borderColor;
2157
+ }
2158
+ }
2159
+ }
2160
+ }
2161
+ // Right edge (excluding corners)
2162
+ for (let py = y1 + borderWidth; py < y2 - borderWidth; py++) {
2163
+ if (py >= 0 && py < imageHeight) {
2164
+ for (let dx = 0; dx < borderWidth; dx++) {
2165
+ const px = x2 - 1 - dx;
2166
+ if (px >= 0 && px < imageWidth) {
2167
+ image[py][px] = borderColor;
2168
+ }
2169
+ }
2170
+ }
2171
+ }
2172
+ }
2173
+ /**
2174
+ * Mark a region as occupied with a striped pattern (0, 255, 0, 255...).
2175
+ * This guarantees max - min = 255 > tolerance, blocking any future label placement.
2176
+ *
2177
+ * Called after an element is done processing (bbox + label placed).
2178
+ *
2179
+ * @param image - Grayscale image (modified in place)
2180
+ * @param x1 - Left edge of element bbox
2181
+ * @param y1 - Top edge of element bbox
2182
+ * @param x2 - Right edge of element bbox
2183
+ * @param y2 - Bottom edge of element bbox
2184
+ * @param labelX - Label top-left X
2185
+ * @param labelY - Label top-left Y
2186
+ * @param labelW - Label width
2187
+ * @param labelH - Label height
2188
+ */
2189
+ function markRegionAsOccupied(image, x1, y1, x2, y2, labelX, labelY, labelW, labelH) {
2190
+ const imageHeight = image.length;
2191
+ const imageWidth = image[0]?.length || 0;
2192
+ // Compute combined region (element + label)
2193
+ const minX = Math.floor(Math.min(x1, labelX));
2194
+ const minY = Math.floor(Math.min(y1, labelY));
2195
+ const maxX = Math.floor(Math.max(x2, labelX + labelW));
2196
+ const maxY = Math.floor(Math.max(y2, labelY + labelH));
2197
+ // Fill with interleaving 0, 255 pattern to guarantee non-uniformity
2198
+ for (let py = minY; py < maxY; py++) {
2199
+ if (py >= 0 && py < imageHeight) {
2200
+ for (let px = minX; px < maxX; px++) {
2201
+ if (px >= 0 && px < imageWidth) {
2202
+ image[py][px] = ((px + py) % 2 === 0) ? 0 : 255;
2203
+ }
2204
+ }
2205
+ }
2206
+ }
2207
+ }
2208
+ /**
2209
+ * Find a valid label position using dynamic convolution on grayscale image.
2210
+ *
2211
+ * Algorithm:
2212
+ * 1. Define candidate region based on element bbox and label dimensions
2213
+ * 2. Compute 2D min/max over the region using sliding window
2214
+ * 3. Find first position where max - min <= tolerance (uniform region)
2215
+ * 4. Return that position or fallback
2216
+ *
2217
+ * @param elementRect - The element's bounding rect
2218
+ * @param labelW - Label width in pixels
2219
+ * @param labelH - Label height in pixels
2220
+ * @returns Position { x, y } for label top-left corner and whether grayscale was used
2221
+ */
2222
+ function findLabelPosition(elementRect, labelW, labelH) {
2223
+ // If no grayscale image available, return fallback
2224
+ if (!grayscaleImage || grayscaleImage.length === 0) {
2225
+ return {
2226
+ x: Math.max(0, elementRect.x),
2227
+ y: Math.max(0, elementRect.y),
2228
+ usedGrayscale: false,
2229
+ };
2230
+ }
2231
+ const imageHeight = grayscaleImage.length;
2232
+ const imageWidth = grayscaleImage[0]?.length || 0;
2233
+ if (imageWidth === 0) {
2234
+ return {
2235
+ x: Math.max(0, elementRect.x),
2236
+ y: Math.max(0, elementRect.y),
2237
+ usedGrayscale: false,
2238
+ };
2239
+ }
2240
+ const { x: X, y: Y, width: N, height: M } = elementRect;
2241
+ const x1 = Math.floor(X);
2242
+ const y1 = Math.floor(Y);
2243
+ const x2 = Math.floor(X + N);
2244
+ const y2 = Math.floor(Y + M);
2245
+ // Define candidate region:
2246
+ // Label can be placed from (x1 - labelW, y1 - labelH) to (x2, y2)
2247
+ // This ensures label can touch any edge of the element
2248
+ // +1 on right/bottom to allow labels adjacent to (not overlapping) the border
2249
+ const regionX = Math.max(0, x1 - labelW);
2250
+ const regionY = Math.max(0, y1 - labelH);
2251
+ const regionX2 = Math.min(imageWidth, x2 + labelW);
2252
+ const regionY2 = Math.min(imageHeight, y2 + labelH);
2253
+ const regionWidth = regionX2 - regionX;
2254
+ const regionHeight = regionY2 - regionY;
2255
+ // Compute 2D min/max for the candidate region
2256
+ const { windowMax, windowMin } = compute2DMinMaxInRegion(grayscaleImage, regionX, regionY, regionWidth, regionHeight, labelW, labelH);
2257
+ if (windowMax.length === 0 || windowMin.length === 0) {
2258
+ // Region too small for label, use fallback
2259
+ const fallbackX = Math.max(0, Math.min(imageWidth - labelW, x2 - labelW));
2260
+ const fallbackY = Math.max(0, y1 - labelH);
2261
+ return { x: fallbackX, y: fallbackY, usedGrayscale: true };
2262
+ }
2263
+ // Search for a uniform position (max - min <= tolerance)
2264
+ // Prefer positions near element center
2265
+ const resultHeight = windowMax.length;
2266
+ const resultWidth = windowMax[0]?.length || 0;
2267
+ // Calculate center of candidate region (in result coordinates)
2268
+ const centerResultX = Math.floor(resultWidth / 2);
2269
+ const centerResultY = Math.floor(resultHeight / 2);
2270
+ // BFS from center to find nearest uniform position
2271
+ const visited = new Set();
2272
+ const queue = [{ rx: centerResultX, ry: centerResultY }];
2273
+ while (queue.length > 0) {
2274
+ const pos = queue.shift();
2275
+ const key = \`\${pos.rx},\${pos.ry}\`;
2276
+ if (visited.has(key))
2277
+ continue;
2278
+ visited.add(key);
2279
+ // Skip if already visited too many positions (performance limit)
2280
+ if (visited.size > 5000)
2281
+ break;
2282
+ // Check bounds
2283
+ if (pos.rx < 0 || pos.rx >= resultWidth || pos.ry < 0 || pos.ry >= resultHeight)
2284
+ continue;
2285
+ // Convert result coordinates to absolute image coordinates
2286
+ const absX = regionX + pos.rx;
2287
+ const absY = regionY + pos.ry;
2288
+ // Check uniformity: max - min <= tolerance
2289
+ const maxVal = windowMax[pos.ry]?.[pos.rx] ?? 255;
2290
+ const minVal = windowMin[pos.ry]?.[pos.rx] ?? 0;
2291
+ const diff = maxVal - minVal;
2292
+ if (diff <= uniformityTolerance) {
2293
+ // Uniform position found!
2294
+ return { x: absX, y: absY, usedGrayscale: true };
2295
+ }
2296
+ // Position not uniform, keep searching
2297
+ queue.push({ rx: pos.rx - 1, ry: pos.ry });
2298
+ queue.push({ rx: pos.rx + 1, ry: pos.ry });
2299
+ queue.push({ rx: pos.rx, ry: pos.ry - 1 });
2300
+ queue.push({ rx: pos.rx, ry: pos.ry + 1 });
2301
+ }
2302
+ // No uniform position found - fallback to outside element (top-right)
2303
+ const fallbackX = Math.max(0, Math.min(imageWidth - labelW, x2 - labelW));
2304
+ const fallbackY = Math.max(0, y1 - labelH);
2305
+ return { x: fallbackX, y: fallbackY, usedGrayscale: true };
2306
+ }
2307
+ /**
2308
+ * Mark both the element and label region as occupied after processing.
2309
+ * This prevents future labels from overlapping with already-labeled elements.
2310
+ *
2311
+ * In post-order traversal, children are labeled first. By marking the combined
2312
+ * region as occupied, we ensure parent labels don't cross child element areas.
2313
+ */
2314
+ function markElementAndLabelAsOccupied(elementRect, labelX, labelY, labelW, labelH) {
2315
+ if (!grayscaleImage)
2316
+ return;
2317
+ const { x, y, width, height } = elementRect;
2318
+ markRegionAsOccupied(grayscaleImage, Math.floor(x), Math.floor(y), Math.floor(x + width), Math.floor(y + height), Math.floor(labelX), Math.floor(labelY), labelW, labelH);
2319
+ }
2320
+ const elementsToHighlight = [];
2321
+ /**
2322
+ * Reorder elements for post-order traversal (children before parents).
2323
+ * This ensures child elements get their labels placed before their parent containers.
2324
+ *
2325
+ * Builds a tree based on actual DOM ancestry between highlighted elements,
2326
+ * not the layout parent grouping used for flow detection.
2327
+ */
2328
+ function getPostOrderElements(elements) {
2329
+ if (elements.length === 0)
2330
+ return [];
2331
+ // Build set of highlighted elements for quick lookup
2332
+ const highlightedSet = new Set();
2333
+ for (const info of elements) {
2334
+ highlightedSet.add(info.element);
2335
+ }
2336
+ // For each element, find its nearest highlighted ancestor (if any)
2337
+ // This builds a tree of highlighted elements based on DOM ancestry
2338
+ const childrenOfHighlighted = new Map();
2339
+ for (const info of elements) {
2340
+ let highlightedParent = null;
2341
+ let current = info.element.parentElement;
2342
+ // Walk up the DOM to find nearest highlighted ancestor
2343
+ while (current && current !== document.body) {
2344
+ if (highlightedSet.has(current)) {
2345
+ highlightedParent = current;
2346
+ break;
2347
+ }
2348
+ current = current.parentElement;
2349
+ }
2350
+ // Group elements by their highlighted parent
2351
+ if (!childrenOfHighlighted.has(highlightedParent)) {
2352
+ childrenOfHighlighted.set(highlightedParent, []);
2353
+ }
2354
+ childrenOfHighlighted.get(highlightedParent).push(info.element);
2355
+ }
2356
+ // Build element -> ElementToHighlight lookup
2357
+ const elementToInfo = new Map();
2358
+ for (const info of elements) {
2359
+ elementToInfo.set(info.element, info);
2360
+ }
2361
+ // Recursive post-order collection
2362
+ const result = [];
2363
+ const visited = new Set();
2364
+ function visit(element) {
2365
+ if (visited.has(element))
2366
+ return;
2367
+ visited.add(element);
2368
+ // First, visit all highlighted children (post-order: children before parent)
2369
+ const children = childrenOfHighlighted.get(element) || [];
2370
+ for (const child of children) {
2371
+ visit(child);
2372
+ }
2373
+ // Then add this element
2374
+ const info = elementToInfo.get(element);
2375
+ if (info)
2376
+ result.push(info);
2377
+ }
2378
+ // Start from roots (elements with no highlighted parent)
2379
+ const roots = childrenOfHighlighted.get(null) || [];
2380
+ for (const root of roots) {
2381
+ visit(root);
2382
+ }
2383
+ return result;
2384
+ }
2385
+ /**
2386
+ * Process elements in recursive tree order:
2387
+ * - Pre-order: Create bounding boxes and mark in hot map
2388
+ * - Post-order: Create labels and mark in hot map
2389
+ *
2390
+ * This ensures children's boxes are visible before parent places its label,
2391
+ * and children's labels are placed before parent's label.
2392
+ */
2393
+ // Collected element data during boxes phase, for return
2394
+ const collectedElementData = [];
2395
+ function processElementTreeRecursively(elements) {
2396
+ if (elements.length === 0)
2397
+ return;
2398
+ // Build tree structure based on DOM ancestry (same as getPostOrderElements)
2399
+ const highlightedSet = new Set();
2400
+ for (const info of elements) {
2401
+ highlightedSet.add(info.element);
2402
+ }
2403
+ const childrenOfHighlighted = new Map();
2404
+ for (const info of elements) {
2405
+ let highlightedParent = null;
2406
+ let current = info.element.parentElement;
2407
+ while (current && current !== document.body) {
2408
+ if (highlightedSet.has(current)) {
2409
+ highlightedParent = current;
2410
+ break;
2411
+ }
2412
+ current = current.parentElement;
2413
+ }
2414
+ if (!childrenOfHighlighted.has(highlightedParent)) {
2415
+ childrenOfHighlighted.set(highlightedParent, []);
2416
+ }
2417
+ childrenOfHighlighted.get(highlightedParent).push(info.element);
2418
+ }
2419
+ // Build element -> ElementToHighlight lookup
2420
+ const elementToInfo = new Map();
2421
+ for (const info of elements) {
2422
+ elementToInfo.set(info.element, info);
2423
+ }
2424
+ // Get or create highlight container
2425
+ let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
2426
+ if (!container) {
2427
+ container = document.createElement("div");
2428
+ container.id = HIGHLIGHT_CONTAINER_ID;
2429
+ container.style.position = "fixed";
2430
+ container.style.pointerEvents = "none";
2431
+ container.style.top = "0";
2432
+ container.style.left = "0";
2433
+ container.style.width = "100%";
2434
+ container.style.height = "100%";
2435
+ container.style.zIndex = "2147483647";
2436
+ container.style.backgroundColor = 'transparent';
2437
+ document.body.appendChild(container);
2438
+ }
2439
+ // Store element info after box creation for later label creation
2440
+ const renderInfoMap = new Map();
2441
+ // Recursive function that processes elements in correct order
2442
+ function processElement(element) {
2443
+ const info = elementToInfo.get(element);
2444
+ if (!info)
2445
+ return;
2446
+ // === PRE-ORDER: Create bounding box ===
2447
+ const renderInfo = createBoundingBoxForElement(info.element, info.index, info.parentIframe, container);
2448
+ if (renderInfo) {
2449
+ renderInfoMap.set(element, renderInfo);
2450
+ // In boxes phase, collect element data for return (no grayscale drawing needed)
2451
+ if (phase === 'boxes') {
2452
+ collectedElementData.push({
2453
+ index: renderInfo.index,
2454
+ xpath: getXPathTree(info.element),
2455
+ rect: { ...renderInfo.elementRect },
2456
+ color: renderInfo.color,
2457
+ });
2458
+ }
2459
+ // In legacy mode (no phase), draw border on grayscale image
2460
+ // This ensures child labels don't get placed crossing this element's border
2461
+ if (!phase && grayscaleImage) {
2462
+ drawBorderOnImage(grayscaleImage, renderInfo.elementRect.x, renderInfo.elementRect.y, renderInfo.elementRect.width, renderInfo.elementRect.height, 2, // borderWidth
2463
+ 128 // borderColor (mid-gray to break uniformity)
2464
+ );
2465
+ }
2466
+ }
2467
+ // === RECURSE: Process children ===
2468
+ const children = childrenOfHighlighted.get(element) || [];
2469
+ for (const child of children) {
2470
+ processElement(child);
2471
+ }
2472
+ // === POST-ORDER: Create label (skip in boxes phase) ===
2473
+ if (renderInfo && phase !== 'boxes') {
2474
+ createLabelForElement(renderInfo);
2475
+ }
2476
+ }
2477
+ // Process roots (elements with no highlighted parent)
2478
+ const roots = childrenOfHighlighted.get(null) || [];
2479
+ for (const root of roots) {
2480
+ processElement(root);
2481
+ }
2482
+ }
2483
+ /**
2484
+ * Process labels phase using pre-collected element data.
2485
+ * Called when phase === 'labels' with inputElementData.
2486
+ */
2487
+ function processLabelsPhase() {
2488
+ if (!inputElementData || inputElementData.length === 0)
2489
+ return;
2490
+ streamLog(\`[dom-tree] Labels phase: processing \${inputElementData.length} elements\`);
2491
+ // Get or create highlight container (should already exist from boxes phase)
2492
+ let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
2493
+ if (!container) {
2494
+ container = document.createElement("div");
2495
+ container.id = HIGHLIGHT_CONTAINER_ID;
2496
+ container.style.position = "fixed";
2497
+ container.style.pointerEvents = "none";
2498
+ container.style.top = "0";
2499
+ container.style.left = "0";
2500
+ container.style.width = "100%";
2501
+ container.style.height = "100%";
2502
+ container.style.zIndex = "2147483647";
2503
+ container.style.backgroundColor = 'transparent';
2504
+ document.body.appendChild(container);
2505
+ }
2506
+ // Sort by index to maintain correct order
2507
+ const sortedData = [...inputElementData].sort((a, b) => a.index - b.index);
2508
+ // Create labels for each element
2509
+ for (const data of sortedData) {
2510
+ const renderInfo = {
2511
+ element: null, // Not needed for label creation
2512
+ index: data.index,
2513
+ parentIframe: null,
2514
+ elementRect: data.rect,
2515
+ color: data.color,
2516
+ container,
2517
+ };
2518
+ createLabelForElement(renderInfo);
2519
+ }
2520
+ }
2521
+ /**
2522
+ * Create bounding box overlays for an element and append to container.
2523
+ * Returns element render info for later label creation.
2524
+ */
2525
+ function createBoundingBoxForElement(element, index, parentIframe, container) {
2526
+ if (!element)
2527
+ return null;
2528
+ // Get element client rects
2529
+ let rects = element.getClientRects();
2530
+ if (!rects || rects.length === 0)
2531
+ return null;
2532
+ // Transform rects if inside an iframe
2533
+ if (parentIframe) {
2534
+ const transformedRects = [];
2535
+ const iframeRect = parentIframe.getBoundingClientRect();
2536
+ const iframeStyle = window.getComputedStyle(parentIframe);
2537
+ const borderLeft = parseFloat(iframeStyle.borderLeftWidth) || 0;
2538
+ const borderTop = parseFloat(iframeStyle.borderTopWidth) || 0;
2539
+ const paddingLeft = parseFloat(iframeStyle.paddingLeft) || 0;
2540
+ const paddingTop = parseFloat(iframeStyle.paddingTop) || 0;
2541
+ const contentOffsetX = borderLeft + paddingLeft;
2542
+ const contentOffsetY = borderTop + paddingTop;
2543
+ let scaleX = 1, scaleY = 1;
2544
+ const transform = iframeStyle.transform;
2545
+ if (transform && transform !== 'none') {
2546
+ const scaleMatch = transform.match(/scale\\(([^)]+)\\)/);
2547
+ if (scaleMatch) {
2548
+ const scaleValues = scaleMatch[1].split(',').map(v => parseFloat(v.trim()));
2549
+ scaleX = scaleValues[0] || 1;
2550
+ scaleY = scaleValues[1] || scaleX;
2551
+ }
2552
+ else {
2553
+ const matrixMatch = transform.match(/matrix\\(([^)]+)\\)/);
2554
+ if (matrixMatch) {
2555
+ const values = matrixMatch[1].split(',').map(v => parseFloat(v.trim()));
2556
+ if (values.length >= 6) {
2557
+ scaleX = values[0];
2558
+ scaleY = values[3];
2559
+ }
2560
+ }
2561
+ }
2562
+ }
2563
+ let scrollLeft = 0, scrollTop = 0;
2564
+ try {
2565
+ const iframeDoc = parentIframe.contentDocument || parentIframe.contentWindow?.document;
2566
+ if (iframeDoc) {
2567
+ scrollLeft = iframeDoc.documentElement?.scrollLeft || iframeDoc.body?.scrollLeft || 0;
2568
+ scrollTop = iframeDoc.documentElement?.scrollTop || iframeDoc.body?.scrollTop || 0;
2569
+ }
2570
+ }
2571
+ catch (e) {
2572
+ console.warn("Cannot access iframe scroll position:", e);
2573
+ }
2574
+ for (const rect of rects) {
2575
+ const scaledWidth = rect.width * scaleX;
2576
+ const scaledHeight = rect.height * scaleY;
2577
+ const scaledTop = rect.top * scaleY;
2578
+ const scaledLeft = rect.left * scaleX;
2579
+ const scaledScrollTop = scrollTop * scaleY;
2580
+ const scaledScrollLeft = scrollLeft * scaleX;
2581
+ transformedRects.push({
2582
+ top: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop,
2583
+ left: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
2584
+ bottom: scaledTop + scaledHeight + iframeRect.top + contentOffsetY - scaledScrollTop,
2585
+ right: scaledLeft + scaledWidth + iframeRect.left + contentOffsetX - scaledScrollLeft,
2586
+ width: scaledWidth,
2587
+ height: scaledHeight,
2588
+ x: scaledLeft + iframeRect.left + contentOffsetX - scaledScrollLeft,
2589
+ y: scaledTop + iframeRect.top + contentOffsetY - scaledScrollTop
2590
+ });
2591
+ }
2592
+ rects = transformedRects;
2593
+ }
2594
+ // Generate color based on index
2595
+ const colors = [
2596
+ "#E53935", "#1E88E5", "#7B1FA2", "#00897B", "#F4511E",
2597
+ "#3949AB", "#C2185B", "#00796B", "#5E35B1", "#D81B60",
2598
+ "#039BE5", "#388E3C"
2599
+ ];
2600
+ const color = colors[index % colors.length];
2601
+ // Create bounding box overlays
2602
+ for (const rect of rects) {
2603
+ if (rect.width === 0 || rect.height === 0)
2604
+ continue;
2605
+ const overlay = document.createElement("div");
2606
+ overlay.style.position = "fixed";
2607
+ overlay.style.border = \`1px solid \${color}\`;
2608
+ overlay.style.backgroundColor = "none";
2609
+ overlay.style.pointerEvents = "none";
2610
+ overlay.style.boxSizing = "border-box";
2611
+ overlay.setAttribute("data-element-index", String(index));
2612
+ overlay.style.top = \`\${rect.top}px\`;
2613
+ overlay.style.left = \`\${rect.left}px\`;
2614
+ overlay.style.width = \`\${rect.width}px\`;
2615
+ overlay.style.height = \`\${rect.height}px\`;
2616
+ container.appendChild(overlay);
2617
+ }
2618
+ // Return element info for label creation
2619
+ const firstRect = rects[0];
2620
+ return {
2621
+ element,
2622
+ index,
2623
+ parentIframe,
2624
+ elementRect: {
2625
+ x: firstRect.left,
2626
+ y: firstRect.top,
2627
+ width: firstRect.width,
2628
+ height: firstRect.height
2629
+ },
2630
+ color,
2631
+ container
2632
+ };
2633
+ }
2634
+ /**
2635
+ * Create and position label for an element and append to container.
2636
+ */
2637
+ function createLabelForElement(renderInfo) {
2638
+ const { index, elementRect, color, container } = renderInfo;
2639
+ // Capture BEFORE snapshot if debug mode enabled (stream via callback)
2640
+ streamLog(\`[dom-tree] Element \${index}: captureDebugSnapshots=\${captureDebugSnapshots}, hasGrayscale=\${!!grayscaleImage}, hasOnSnapshot=\${!!onSnapshot}\`);
2641
+ if (captureDebugSnapshots && grayscaleImage && onSnapshot) {
2642
+ streamLog(\`[dom-tree] Calling onSnapshot for element \${index} (before)\`);
2643
+ onSnapshot({ elementIndex: index, type: 'before', data: grayscaleImage.map(row => [...row]) });
2644
+ }
2645
+ // Calculate label dimensions based on digit count
2646
+ const digits = index.toString().length;
2647
+ const labelWidth = (digits * 8) + 8;
2648
+ const labelHeight = 16;
2649
+ // Find label position using grayscale image or fallback to heuristics
2650
+ let labelTop = 0;
2651
+ let labelLeft = 0;
2652
+ const grayscaleResult = findLabelPosition(elementRect, labelWidth, labelHeight);
2653
+ if (grayscaleResult.usedGrayscale) {
2654
+ labelTop = grayscaleResult.y;
2655
+ labelLeft = grayscaleResult.x;
2656
+ }
2657
+ else {
2658
+ // Fallback to heuristic positioning (assume vertical flow)
2659
+ const isNarrow = elementRect.width <= 32;
2660
+ const centerX = elementRect.x + elementRect.width / 2;
2661
+ let positions;
2662
+ if (isNarrow) {
2663
+ positions = [
2664
+ { top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x, hAlign: 'right', name: 'left' },
2665
+ { top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x + elementRect.width, hAlign: 'left', name: 'right' },
2666
+ { top: elementRect.y - labelHeight, anchorX: centerX, hAlign: 'center', name: 'top' },
2667
+ { top: elementRect.y + elementRect.height, anchorX: centerX, hAlign: 'center', name: 'bottom' },
2668
+ ];
2669
+ }
2670
+ else {
2671
+ positions = [
2672
+ { top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x, hAlign: 'right', name: 'left' },
2673
+ { top: elementRect.y + (elementRect.height - labelHeight) / 2, anchorX: elementRect.x + elementRect.width, hAlign: 'left', name: 'right' },
2674
+ { top: elementRect.y - labelHeight, anchorX: elementRect.x + elementRect.width, hAlign: 'right', name: 'top-right' },
2675
+ { top: elementRect.y - labelHeight, anchorX: elementRect.x, hAlign: 'left', name: 'top-left' },
2676
+ ];
2677
+ }
2678
+ // Use first position as default
2679
+ const pos = positions[0];
2680
+ labelTop = Math.max(0, Math.min(pos.top, window.innerHeight - labelHeight));
2681
+ if (pos.hAlign === 'right') {
2682
+ labelLeft = pos.anchorX - labelWidth;
2683
+ }
2684
+ else if (pos.hAlign === 'center') {
2685
+ labelLeft = pos.anchorX - labelWidth / 2;
2686
+ }
2687
+ else {
2688
+ labelLeft = pos.anchorX;
2689
+ }
2690
+ labelLeft = Math.max(0, Math.min(labelLeft, window.innerWidth - labelWidth));
2691
+ }
2692
+ // Always mark the label position on grayscale image to prevent future overlaps
2693
+ markElementAndLabelAsOccupied(elementRect, labelLeft, labelTop, labelWidth, labelHeight);
2694
+ // Capture AFTER snapshot if debug mode enabled (stream via callback)
2695
+ if (captureDebugSnapshots && grayscaleImage && onSnapshot) {
2696
+ streamLog(\`[dom-tree] Calling onSnapshot for element \${index} (after)\`);
2697
+ onSnapshot({ elementIndex: index, type: 'after', data: grayscaleImage.map(row => [...row]) });
2698
+ }
2699
+ // Create label element with explicit dimensions matching theoretical calculation
2700
+ const label = document.createElement("div");
2701
+ label.className = "playwright-highlight-label";
2702
+ label.style.position = "fixed";
2703
+ label.style.background = color;
2704
+ label.style.color = "white";
2705
+ label.style.padding = "1px 4px";
2706
+ label.style.fontSize = "12px";
2707
+ label.style.width = \`\${labelWidth}px\`;
2708
+ label.style.height = \`\${labelHeight}px\`;
2709
+ label.style.boxSizing = "border-box";
2710
+ label.style.textAlign = "center";
2711
+ label.style.lineHeight = \`\${labelHeight - 2}px\`;
2712
+ label.textContent = String(index);
2713
+ label.setAttribute("data-element-index", String(index));
2714
+ label.style.top = \`\${labelTop}px\`;
2715
+ label.style.left = \`\${labelLeft}px\`;
2716
+ container.appendChild(label);
2717
+ }
2718
+ /**
2719
+ * Build tree entries for debug output showing element hierarchy.
2720
+ *
2721
+ * The tree collapses single-child chains: if A contains only B, which contains only C (highlighted),
2722
+ * then A, B, C collapse into just C.
2723
+ *
2724
+ * Parents are only included if they have more than one highlighted descendant.
2725
+ */
2726
+ function buildTreeEntries() {
2727
+ const treeEntries = [];
2728
+ if (elementsToHighlight.length === 0)
2729
+ return treeEntries;
2730
+ // For each highlighted element, find all ancestors and count highlighted descendants
2731
+ const ancestorCounts = new Map();
2732
+ for (const { element } of elementsToHighlight) {
2733
+ let current = element.parentElement;
2734
+ while (current && current !== document.body) {
2735
+ ancestorCounts.set(current, (ancestorCounts.get(current) || 0) + 1);
2736
+ current = current.parentElement;
2737
+ }
2738
+ }
2739
+ // Build tree structure: map each element to its layout parent
2740
+ const childrenMap = new Map(); // parent -> children
2741
+ for (const { element } of elementsToHighlight) {
2742
+ let layoutParent = null;
2743
+ let current = element.parentElement;
2744
+ while (current && current !== document.body) {
2745
+ const count = ancestorCounts.get(current) || 0;
2746
+ if (count > 1) {
2747
+ layoutParent = current;
2748
+ break;
2749
+ }
2750
+ current = current.parentElement;
2751
+ }
2752
+ // Group children by parent
2753
+ if (!childrenMap.has(layoutParent)) {
2754
+ childrenMap.set(layoutParent, []);
2755
+ }
2756
+ childrenMap.get(layoutParent).push(element);
2757
+ }
2758
+ // Output tree structure to debug logs and build tree entries
2759
+ const elementToIndex = new Map();
2760
+ for (const { element, index } of elementsToHighlight) {
2761
+ elementToIndex.set(element, index);
2762
+ }
2763
+ const describeElement = (el) => {
2764
+ const tag = el.tagName.toLowerCase();
2765
+ const id = el.id ? \`#\${el.id}\` : '';
2766
+ const cls = el.className && typeof el.className === 'string'
2767
+ ? '.' + el.className.split(' ').slice(0, 2).join('.')
2768
+ : '';
2769
+ return \`<\${tag}\${id}\${cls.slice(0, 30)}>\`;
2770
+ };
2771
+ debugLog(\`[Tree] Detected elements tree:\`);
2772
+ // Get unique layout parents and sort by DOM order
2773
+ const parents = Array.from(childrenMap.keys());
2774
+ for (const parent of parents) {
2775
+ const children = childrenMap.get(parent) || [];
2776
+ const parentDesc = parent ? describeElement(parent) : '(root)';
2777
+ debugLog(\`[Tree] \${parentDesc}\`);
2778
+ // Add parent entry to tree
2779
+ treeEntries.push({
2780
+ type: 'parent',
2781
+ label: parentDesc,
2782
+ });
2783
+ for (const child of children) {
2784
+ const idx = elementToIndex.get(child) ?? -1;
2785
+ debugLog(\`[Tree] [\${idx}] \${describeElement(child)}\`);
2786
+ // Add element entry to tree
2787
+ treeEntries.push({
2788
+ type: 'element',
2789
+ highlightIndex: idx,
2790
+ label: describeElement(child),
2791
+ });
2792
+ }
2793
+ }
2794
+ return treeEntries;
2795
+ }
2796
+ /**
2797
+ * Gets the position of an element in its parent.
2798
+ */
2799
+ function getElementPosition(currentElement) {
2800
+ if (!currentElement.parentElement) {
2801
+ return 0; // No parent means no siblings
2802
+ }
2803
+ const tagName = currentElement.nodeName.toLowerCase();
2804
+ const siblings = Array.from(currentElement.parentElement.children)
2805
+ .filter((sib) => sib.nodeName.toLowerCase() === tagName);
2806
+ if (siblings.length === 1) {
2807
+ return 0; // Only element of its type
2808
+ }
2809
+ const index = siblings.indexOf(currentElement) + 1; // 1-based index
2810
+ return index;
2811
+ }
2812
+ function getXPathTree(element, stopAtBoundary = true) {
2813
+ if (xpathCache.has(element))
2814
+ return xpathCache.get(element);
2815
+ const segments = [];
2816
+ let currentElement = element;
2817
+ while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) {
2818
+ // Stop if we hit a shadow root or iframe
2819
+ if (stopAtBoundary &&
2820
+ (currentElement.parentNode instanceof ShadowRoot ||
2821
+ currentElement.parentNode instanceof HTMLIFrameElement)) {
2822
+ break;
2823
+ }
2824
+ const position = getElementPosition(currentElement);
2825
+ const tagName = currentElement.nodeName.toLowerCase();
2826
+ const xpathIndex = position > 0 ? \`[\${position}]\` : "";
2827
+ segments.unshift(\`\${tagName}\${xpathIndex}\`);
2828
+ currentElement = currentElement.parentNode;
2829
+ }
2830
+ const result = segments.join("/");
2831
+ xpathCache.set(element, result);
2832
+ return result;
2833
+ }
2834
+ /**
2835
+ * Checks if a text node is visible.
2836
+ */
2837
+ function isTextNodeVisible(textNode) {
2838
+ try {
2839
+ // Special case: when viewportExpansion is -1, consider all text nodes as visible
2840
+ if (viewportExpansion === -1) {
2841
+ // Still check parent visibility for basic filtering
2842
+ const parentElement = textNode.parentElement;
2843
+ if (!parentElement)
2844
+ return false;
2845
+ try {
2846
+ return parentElement.checkVisibility({
2847
+ checkOpacity: true,
2848
+ checkVisibilityCSS: true,
2849
+ });
2850
+ }
2851
+ catch (e) {
2852
+ // Fallback if checkVisibility is not supported
2853
+ const style = window.getComputedStyle(parentElement);
2854
+ return style.display !== 'none' &&
2855
+ style.visibility !== 'hidden' &&
2856
+ style.opacity !== '0';
2857
+ }
2858
+ }
2859
+ const range = document.createRange();
2860
+ range.selectNodeContents(textNode);
2861
+ const rects = range.getClientRects(); // Use getClientRects for Range
2862
+ if (!rects || rects.length === 0) {
2863
+ return false;
2864
+ }
2865
+ let isAnyRectVisible = false;
2866
+ let isAnyRectInViewport = false;
2867
+ for (const rect of rects) {
2868
+ // Check size
2869
+ if (rect.width > 0 && rect.height > 0) {
2870
+ isAnyRectVisible = true;
2871
+ // Viewport check for this rect
2872
+ if (!(rect.bottom < -viewportExpansion ||
2873
+ rect.top > window.innerHeight + viewportExpansion ||
2874
+ rect.right < -viewportExpansion ||
2875
+ rect.left > window.innerWidth + viewportExpansion)) {
2876
+ isAnyRectInViewport = true;
2877
+ break; // Found a visible rect in viewport, no need to check others
2878
+ }
2879
+ }
2880
+ }
2881
+ if (!isAnyRectVisible || !isAnyRectInViewport) {
2882
+ return false;
2883
+ }
2884
+ // Check parent visibility
2885
+ const parentElement = textNode.parentElement;
2886
+ if (!parentElement)
2887
+ return false;
2888
+ try {
2889
+ return parentElement.checkVisibility({
2890
+ checkOpacity: true,
2891
+ checkVisibilityCSS: true,
2892
+ });
2893
+ }
2894
+ catch (e) {
2895
+ // Fallback if checkVisibility is not supported
2896
+ const style = window.getComputedStyle(parentElement);
2897
+ return style.display !== 'none' &&
2898
+ style.visibility !== 'hidden' &&
2899
+ style.opacity !== '0';
2900
+ }
2901
+ }
2902
+ catch (e) {
2903
+ console.warn('Error checking text node visibility:', e);
2904
+ return false;
2905
+ }
2906
+ }
2907
+ /**
2908
+ * Checks if an element is accepted.
2909
+ */
2910
+ function isElementAccepted(element) {
2911
+ if (!element || !element.tagName)
2912
+ return false;
2913
+ // Always accept body and common container elements
2914
+ const alwaysAccept = new Set([
2915
+ "body", "div", "main", "article", "section", "nav", "header", "footer"
2916
+ ]);
2917
+ const tagName = element.tagName.toLowerCase();
2918
+ if (alwaysAccept.has(tagName))
2919
+ return true;
2920
+ const leafElementDenyList = new Set([
2921
+ "svg",
2922
+ "script",
2923
+ "style",
2924
+ "link",
2925
+ "meta",
2926
+ "noscript",
2927
+ "template",
2928
+ ]);
2929
+ return !leafElementDenyList.has(tagName);
2930
+ }
2931
+ /**
2932
+ * Checks if an element is visible.
2933
+ */
2934
+ function isElementVisible(element) {
2935
+ if (alwaysHighlightFileInput && element.tagName.toLowerCase() === "input" && element.type === "file")
2936
+ return true;
2937
+ // SVG elements need special handling for visibility
2938
+ if (element.tagName.toLowerCase() === "svg") {
2939
+ const rect = getCachedBoundingRect(element);
2940
+ const style = getCachedComputedStyle(element);
2941
+ return (rect !== null &&
2942
+ rect.width > 0 &&
2943
+ rect.height > 0 &&
2944
+ style?.visibility !== "hidden" &&
2945
+ style?.display !== "none");
2946
+ }
2947
+ const style = getCachedComputedStyle(element);
2948
+ return (element.offsetWidth > 0 &&
2949
+ element.offsetHeight > 0 &&
2950
+ style?.visibility !== "hidden" &&
2951
+ style?.display !== "none");
2952
+ }
2953
+ /**
2954
+ * Checks if an element is clickable (responds to click events).
2955
+ */
2956
+ function shouldMarkAsClickable(element) {
2957
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) {
2958
+ return false;
2959
+ }
2960
+ const tagName = element.tagName.toLowerCase();
2961
+ // Primarily clickable elements
2962
+ const primaryClickableElements = new Set([
2963
+ "a", // Links
2964
+ "button", // Buttons
2965
+ "summary", // Summary element (clickable part of details)
2966
+ "label", // Form labels (often clickable)
2967
+ "option", // Select options
2968
+ "optgroup", // Option groups
2969
+ ]);
2970
+ if (primaryClickableElements.has(tagName)) {
2971
+ return false;
2972
+ }
2973
+ const role = element.getAttribute("role");
2974
+ // Clickable roles
2975
+ const clickableRoles = new Set([
2976
+ 'button', // Directly clickable element
2977
+ 'link', // Clickable link
2978
+ 'menuitem', // Clickable menu item
2979
+ 'menuitemradio', // Radio-style menu item (selectable)
2980
+ 'menuitemcheckbox', // Checkbox-style menu item (toggleable)
2981
+ 'radio', // Radio button (selectable)
2982
+ 'checkbox', // Checkbox (toggleable)
2983
+ 'tab', // Tab (clickable to switch content)
2984
+ 'switch', // Toggle switch (clickable to change state)
2985
+ 'option', // Selectable option in a list
2986
+ ]);
2987
+ if (role && clickableRoles.has(role)) {
2988
+ return true;
2989
+ }
2990
+ // Check for dropdown indicators
2991
+ if (hasAnyClassName(element, buttonClassNames)) {
2992
+ return true; // Return true for dropdown elements
2993
+ }
2994
+ if (element.getAttribute('data-toggle') === 'dropdown' ||
2995
+ element.getAttribute('aria-haspopup')) {
2996
+ return true;
2997
+ }
2998
+ const clickEvents = ['click', 'mousedown', 'mouseup', 'dblclick'];
2999
+ const listenedEvents = getCachedNodeEventListeners(element);
3000
+ if (listenedEvents && listenedEvents.length > 0) {
3001
+ for (const eventType of clickEvents) {
3002
+ if (listenedEvents.includes(eventType)) {
3003
+ return true;
3004
+ }
3005
+ }
3006
+ }
3007
+ return false;
3008
+ }
3009
+ /**
3010
+ * Checks if an element is interactive.
3011
+ *
3012
+ * lots of comments, and uncommented code - to show the logic of what we already tried
3013
+ *
3014
+ * One of the things we tried at the beginning was also to use event listeners, and other fancy class, style stuff -> what actually worked best was just combining most things with computed cursor style :)
3015
+ */
3016
+ function isInteractiveElement(element) {
3017
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) {
3018
+ return false;
3019
+ }
3020
+ // Cache the tagName and style lookups
3021
+ const tagName = element.tagName.toLowerCase();
3022
+ const style = getCachedComputedStyle(element);
3023
+ // Define interactive cursors
3024
+ const interactiveCursors = new Set([
3025
+ 'pointer', // Link/clickable elements
3026
+ 'move', // Movable elements
3027
+ 'text', // Text selection
3028
+ 'grab', // Grabbable elements
3029
+ 'grabbing', // Currently grabbing
3030
+ 'cell', // Table cell selection
3031
+ 'copy', // Copy operation
3032
+ 'alias', // Alias creation
3033
+ 'all-scroll', // Scrollable content
3034
+ 'col-resize', // Column resize
3035
+ 'context-menu', // Context menu available
3036
+ 'crosshair', // Precise selection
3037
+ 'e-resize', // East resize
3038
+ 'ew-resize', // East-west resize
3039
+ 'help', // Help available
3040
+ 'n-resize', // North resize
3041
+ 'ne-resize', // Northeast resize
3042
+ 'nesw-resize', // Northeast-southwest resize
3043
+ 'ns-resize', // North-south resize
3044
+ 'nw-resize', // Northwest resize
3045
+ 'nwse-resize', // Northwest-southeast resize
3046
+ 'row-resize', // Row resize
3047
+ 's-resize', // South resize
3048
+ 'se-resize', // Southeast resize
3049
+ 'sw-resize', // Southwest resize
3050
+ 'vertical-text', // Vertical text selection
3051
+ 'w-resize', // West resize
3052
+ 'zoom-in', // Zoom in
3053
+ 'zoom-out' // Zoom out
3054
+ ]);
3055
+ // Define non-interactive cursors
3056
+ const nonInteractiveCursors = new Set([
3057
+ 'not-allowed', // Action not allowed
3058
+ 'no-drop', // Drop not allowed
3059
+ 'wait', // Processing
3060
+ 'progress', // In progress
3061
+ 'initial', // Initial value
3062
+ 'inherit' // Inherited value
3063
+ //? Let's just include all potentially clickable elements that are not specifically blocked
3064
+ // 'none', // No cursor
3065
+ // 'default', // Default cursor
3066
+ // 'auto', // Browser default
3067
+ ]);
3068
+ /**
3069
+ * Checks if an element has an interactive pointer.
3070
+ */
3071
+ function doesElementHaveInteractivePointer(element) {
3072
+ if (element.tagName.toLowerCase() === "html")
3073
+ return false;
3074
+ if (style?.cursor && interactiveCursors.has(style.cursor))
3075
+ return true;
3076
+ return false;
3077
+ }
3078
+ // Disabled for now, since it adds too many false positives
3079
+ // let isInteractiveCursor = doesElementHaveInteractivePointer(element);
3080
+ // // Genius fix for almost all interactive elements
3081
+ // if (isInteractiveCursor) {
3082
+ // return true;
3083
+ // }
3084
+ const interactiveElements = new Set([
3085
+ "a", // Links
3086
+ "button", // Buttons
3087
+ "input", // All input types (text, checkbox, radio, etc.)
3088
+ "select", // Dropdown menus
3089
+ "textarea", // Text areas
3090
+ "summary", // Summary element (clickable part of details)
3091
+ "label", // Form labels (often clickable)
3092
+ "option", // Select options
3093
+ "optgroup", // Option groups
3094
+ "fieldset", // Form fieldsets (can be interactive with legend)
3095
+ "legend", // Fieldset legends
3096
+ ]);
3097
+ // Define explicit disable attributes and properties
3098
+ const explicitDisableTags = new Set([
3099
+ 'disabled', // Standard disabled attribute
3100
+ // 'aria-disabled', // ARIA disabled state
3101
+ // 'readonly', // Read-only state
3102
+ // 'aria-readonly', // ARIA read-only state
3103
+ // 'aria-hidden', // Hidden from accessibility
3104
+ // 'hidden', // Hidden attribute
3105
+ // 'inert', // Inert attribute
3106
+ // 'aria-inert', // ARIA inert state
3107
+ // 'tabindex="-1"', // Removed from tab order
3108
+ // 'aria-hidden="true"' // Hidden from screen readers
3109
+ ]);
3110
+ // Check for non-interactive cursor
3111
+ if (style?.cursor && nonInteractiveCursors.has(style.cursor)) {
3112
+ return false;
3113
+ }
3114
+ // handle inputs, select, checkbox, radio, textarea, button and make sure they are not cursor style disabled/not-allowed
3115
+ if (interactiveElements.has(tagName)) {
3116
+ // Check for explicit disable attributes
3117
+ for (const disableTag of explicitDisableTags) {
3118
+ if (element.hasAttribute(disableTag) ||
3119
+ element.getAttribute(disableTag) === 'true' ||
3120
+ element.getAttribute(disableTag) === '') {
3121
+ return false;
3122
+ }
3123
+ }
3124
+ // Check for disabled property on form elements
3125
+ if (element.disabled) {
3126
+ return false;
3127
+ }
3128
+ // Don't mark as non-interactive yet
3129
+ // Check for readonly property on form elements
3130
+ if (element.readOnly) {
3131
+ // return false;
3132
+ }
3133
+ // Check for inert property
3134
+ if (element.inert) {
3135
+ return false;
3136
+ }
3137
+ return true;
3138
+ }
3139
+ const role = element.getAttribute("role");
3140
+ const ariaRole = element.getAttribute("aria-role");
3141
+ // Check for contenteditable attribute
3142
+ if (element.getAttribute("contenteditable") === "true" || element.isContentEditable) {
3143
+ return true;
3144
+ }
3145
+ // Added enhancement to capture dropdown interactive elements
3146
+ if (hasAnyClassName(element, buttonClassNames) ||
3147
+ hasAnyClassName(element, interactiveClassNames) ||
3148
+ hasAnyClassName(element, cursorPointerClassNames) ||
3149
+ element.getAttribute('data-index') ||
3150
+ element.getAttribute('data-toggle') === 'dropdown' ||
3151
+ element.getAttribute('aria-haspopup')) {
3152
+ return true;
3153
+ }
3154
+ const interactiveRoles = new Set([
3155
+ 'button', // Directly clickable element
3156
+ 'link', // Clickable link
3157
+ 'menuitem', // Clickable menu item
3158
+ 'menuitemradio', // Radio-style menu item (selectable)
3159
+ 'menuitemcheckbox', // Checkbox-style menu item (toggleable)
3160
+ 'radio', // Radio button (selectable)
3161
+ 'checkbox', // Checkbox (toggleable)
3162
+ 'tab', // Tab (clickable to switch content)
3163
+ 'switch', // Toggle switch (clickable to change state)
3164
+ 'slider', // Slider control (draggable)
3165
+ 'spinbutton', // Number input with up/down controls
3166
+ 'combobox', // Dropdown with text input
3167
+ 'searchbox', // Search input field
3168
+ 'textbox', // Text input field
3169
+ 'listbox', // Selectable list
3170
+ 'option', // Selectable option in a list
3171
+ 'scrollbar' // Scrollable control
3172
+ ]);
3173
+ // Basic role/attribute checks
3174
+ const hasInteractiveRole = (role && interactiveRoles.has(role)) ||
3175
+ (ariaRole && interactiveRoles.has(ariaRole));
3176
+ if (hasInteractiveRole)
3177
+ return true;
3178
+ const listenedEvents = getCachedNodeEventListeners(element);
3179
+ if (listenedEvents && listenedEvents.length > 0) {
3180
+ for (const eventType of INTERACTION_EVENTS) {
3181
+ if (listenedEvents.includes(eventType)) {
3182
+ return true;
3183
+ }
3184
+ }
3185
+ }
3186
+ return false;
3187
+ }
3188
+ /**
3189
+ * Checks if an element is the topmost element at its position.
3190
+ */
3191
+ function isTopElement(element) {
3192
+ // Special case: when viewportExpansion is -1, consider all elements as "top" elements
3193
+ if (viewportExpansion === -1) {
3194
+ return true;
3195
+ }
3196
+ const rects = getCachedClientRects(element);
3197
+ if (!rects || rects.length === 0) {
3198
+ return false; // No geometry, cannot be top
3199
+ }
3200
+ let isAnyRectInViewport = false;
3201
+ for (const rect of rects) {
3202
+ // Use the same logic as isInExpandedViewport check
3203
+ if (rect.width > 0 && rect.height > 0 && !( // Only check non-empty rects
3204
+ rect.bottom < -viewportExpansion ||
3205
+ rect.top > window.innerHeight + viewportExpansion ||
3206
+ rect.right < -viewportExpansion ||
3207
+ rect.left > window.innerWidth + viewportExpansion)) {
3208
+ isAnyRectInViewport = true;
3209
+ break;
3210
+ }
3211
+ }
3212
+ if (!isAnyRectInViewport) {
3213
+ return false; // All rects are outside the viewport area
3214
+ }
3215
+ // Find the correct document context and root element
3216
+ let doc = element.ownerDocument;
3217
+ // If we're in an iframe, elements are considered top by default
3218
+ if (doc !== window.document) {
3219
+ return true;
3220
+ }
3221
+ // For shadow DOM, we need to check within its own root context
3222
+ const shadowRoot = element.getRootNode();
3223
+ if (shadowRoot instanceof ShadowRoot) {
3224
+ const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
3225
+ const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
3226
+ try {
3227
+ const topEl = shadowRoot.elementFromPoint(centerX, centerY);
3228
+ if (!topEl)
3229
+ return false;
3230
+ let current = topEl;
3231
+ while (current && current !== shadowRoot) {
3232
+ if (current === element)
3233
+ return true;
3234
+ current = current.parentElement;
3235
+ }
3236
+ return false;
3237
+ }
3238
+ catch (e) {
3239
+ return true;
3240
+ }
3241
+ }
3242
+ // For elements in viewport, check if they're topmost
3243
+ const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
3244
+ const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
3245
+ try {
3246
+ const topEl = document.elementFromPoint(centerX, centerY);
3247
+ if (!topEl)
3248
+ return false;
3249
+ let current = topEl;
3250
+ while (current && current !== document.documentElement) {
3251
+ if (current === element)
3252
+ return true;
3253
+ current = current.parentElement;
3254
+ }
3255
+ return false;
3256
+ }
3257
+ catch (e) {
3258
+ return true;
3259
+ }
3260
+ }
3261
+ /**
3262
+ * Checks if an element is within the expanded viewport.
3263
+ */
3264
+ function isInExpandedViewport(element, viewportExpansion) {
3265
+ if (viewportExpansion === -1) {
3266
+ return true;
3267
+ }
3268
+ const rects = element.getClientRects();
3269
+ if (!rects || rects.length === 0) {
3270
+ // Fallback to getBoundingClientRect if getClientRects is empty,
3271
+ // useful for elements like <svg> that might not have client rects but have a bounding box.
3272
+ const boundingRect = getCachedBoundingRect(element);
3273
+ if (!boundingRect || boundingRect.width === 0 || boundingRect.height === 0) {
3274
+ return false;
3275
+ }
3276
+ return !(boundingRect.bottom < -viewportExpansion ||
3277
+ boundingRect.top > window.innerHeight + viewportExpansion ||
3278
+ boundingRect.right < -viewportExpansion ||
3279
+ boundingRect.left > window.innerWidth + viewportExpansion);
3280
+ }
3281
+ // Check if *any* client rect is within the viewport
3282
+ for (const rect of rects) {
3283
+ if (rect.width === 0 || rect.height === 0)
3284
+ continue; // Skip empty rects
3285
+ if (!(rect.bottom < -viewportExpansion ||
3286
+ rect.top > window.innerHeight + viewportExpansion ||
3287
+ rect.right < -viewportExpansion ||
3288
+ rect.left > window.innerWidth + viewportExpansion)) {
3289
+ return true; // Found at least one rect in the viewport
3290
+ }
3291
+ }
3292
+ return false; // No rects were found in the viewport
3293
+ }
3294
+ // /**
3295
+ // * Gets the effective scroll of an element.
3296
+ // *
3297
+ // * @param {HTMLElement} element - The element to get the effective scroll for.
3298
+ // * @returns {Object} The effective scroll of the element.
3299
+ // */
3300
+ // function getEffectiveScroll(element) {
3301
+ // let currentEl = element;
3302
+ // let scrollX = 0;
3303
+ // let scrollY = 0;
3304
+ // while (currentEl && currentEl !== document.documentElement) {
3305
+ // if (currentEl.scrollLeft || currentEl.scrollTop) {
3306
+ // scrollX += currentEl.scrollLeft;
3307
+ // scrollY += currentEl.scrollTop;
3308
+ // }
3309
+ // currentEl = currentEl.parentElement;
3310
+ // }
3311
+ // scrollX += window.scrollX;
3312
+ // scrollY += window.scrollY;
3313
+ // return { scrollX, scrollY };
3314
+ // }
3315
+ /**
3316
+ * Checks if an element is an interactive candidate.
3317
+ */
3318
+ function isInteractiveCandidate(element) {
3319
+ if (!element || element.nodeType !== Node.ELEMENT_NODE)
3320
+ return false;
3321
+ const tagName = element.tagName.toLowerCase();
3322
+ // Fast-path for common interactive elements
3323
+ const interactiveElements = new Set([
3324
+ "a", "button", "input", "select", "textarea", "summary", "label"
3325
+ ]);
3326
+ if (interactiveElements.has(tagName))
3327
+ return true;
3328
+ // Quick attribute checks without getting full lists
3329
+ const hasQuickInteractiveAttr = element.hasAttribute("onclick") ||
3330
+ element.hasAttribute("role") ||
3331
+ element.hasAttribute("tabindex") ||
3332
+ element.hasAttribute("aria-") ||
3333
+ element.hasAttribute("data-action") ||
3334
+ element.getAttribute("contenteditable") === "true";
3335
+ return hasQuickInteractiveAttr;
3336
+ }
3337
+ // --- Define constants for distinct interaction check ---
3338
+ const DISTINCT_INTERACTIVE_TAGS = new Set([
3339
+ 'a', 'button', 'input', 'select', 'textarea', 'summary', 'label', 'option'
3340
+ ]);
3341
+ const INTERACTIVE_ROLES = new Set([
3342
+ 'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox',
3343
+ 'radio', 'checkbox', 'tab', 'switch', 'slider', 'spinbutton',
3344
+ 'combobox', 'searchbox', 'textbox', 'listbox', 'option', 'scrollbar'
3345
+ ]);
3346
+ /**
3347
+ * Heuristically determines if an element should be considered as independently interactive,
3348
+ * even if it's nested inside another interactive container.
3349
+ *
3350
+ * This function helps detect deeply nested actionable elements (e.g., menu items within a button)
3351
+ * that may not be picked up by strict interactivity checks.
3352
+ */
3353
+ function isHeuristicallyInteractive(element) {
3354
+ if (!element || element.nodeType !== Node.ELEMENT_NODE)
3355
+ return false;
3356
+ // Skip non-visible elements early for performance
3357
+ if (!isElementVisible(element))
3358
+ return false;
3359
+ // Check for common attributes that often indicate interactivity
3360
+ const hasInteractiveAttributes = element.hasAttribute('role') ||
3361
+ element.hasAttribute('tabindex') ||
3362
+ element.hasAttribute('onclick') ||
3363
+ typeof element.onclick === 'function';
3364
+ // Check for semantic class names suggesting interactivity
3365
+ const hasInteractiveClass = heuristicClassPattern.test(element.className || '');
3366
+ // Determine whether the element is inside a known interactive container
3367
+ const isInKnownContainer = Boolean(element.closest(containerSelectors));
3368
+ // Ensure the element has at least one visible child (to avoid marking empty wrappers)
3369
+ const hasVisibleChildren = [...element.children].some(child => isElementVisible(child));
3370
+ // Avoid highlighting elements whose parent is <body> (top-level wrappers)
3371
+ const isParentBody = element.parentElement && element.parentElement.isSameNode(document.body);
3372
+ return ((isInteractiveElement(element) || hasInteractiveAttributes || hasInteractiveClass) &&
3373
+ hasVisibleChildren &&
3374
+ isInKnownContainer &&
3375
+ !isParentBody);
3376
+ }
3377
+ /**
3378
+ * Checks if an element likely represents a distinct interaction
3379
+ * separate from its parent (if the parent is also interactive).
3380
+ */
3381
+ function isElementDistinctInteraction(element, nodeData) {
3382
+ if (nodeData.isScrollable) {
3383
+ return true;
3384
+ }
3385
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) {
3386
+ return false;
3387
+ }
3388
+ const tagName = element.tagName.toLowerCase();
3389
+ const role = element.getAttribute('role');
3390
+ // Check if it's an iframe - always distinct boundary
3391
+ if (tagName === 'iframe') {
3392
+ return true;
3393
+ }
3394
+ // Check tag name
3395
+ if (DISTINCT_INTERACTIVE_TAGS.has(tagName)) {
3396
+ return true;
3397
+ }
3398
+ // Check interactive roles
3399
+ if (role && INTERACTIVE_ROLES.has(role)) {
3400
+ return true;
3401
+ }
3402
+ // Check contenteditable
3403
+ if (element.isContentEditable || element.getAttribute('contenteditable') === 'true') {
3404
+ return true;
3405
+ }
3406
+ // Check for common testing/automation attributes
3407
+ if (element.hasAttribute('data-testid') || element.hasAttribute('data-cy') || element.hasAttribute('data-test')) {
3408
+ return true;
3409
+ }
3410
+ // Check for explicit onclick handler (attribute or property)
3411
+ if (element.hasAttribute('onclick') || typeof element.onclick === 'function') {
3412
+ return true;
3413
+ }
3414
+ if (element.hasAttribute('aria-haspopup')) {
3415
+ return true;
3416
+ }
3417
+ if (hasAnyClassName(element, interactiveClassNames)) {
3418
+ return true;
3419
+ }
3420
+ // Check for cursor-pointer class names that indicate clickability
3421
+ if (hasAnyClassName(element, cursorPointerClassNames)) {
3422
+ return true;
3423
+ }
3424
+ // return false
3425
+ // Check for other common interaction event listeners
3426
+ try {
3427
+ const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode;
3428
+ if (typeof getEventListenersForNode === 'function') {
3429
+ const listeners = getEventListenersForNode(element);
3430
+ const interactionEvents = ['click', 'mousedown', 'mouseup', 'dblclick', 'input', 'mouseenter', 'mouseleave', 'keydown', 'keyup', 'submit', 'change', 'focus', 'blur'];
3431
+ for (const eventType of interactionEvents) {
3432
+ for (const listener of listeners) {
3433
+ if (listener.type === eventType) {
3434
+ return true; // Found a common interaction listener
3435
+ }
3436
+ }
3437
+ }
3438
+ }
3439
+ // Fallback: Check common event attributes if getEventListeners is not available (getEventListenersForNode doesn't work in page.evaluate context)
3440
+ const commonEventAttrs = ['onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', 'onsubmit', 'onmouseenter', 'onmouseleave', 'onchange', 'oninput', 'onfocus', 'onblur'];
3441
+ if (commonEventAttrs.some(attr => element.hasAttribute(attr))) {
3442
+ return true;
3443
+ }
3444
+ }
3445
+ catch (e) {
3446
+ // console.warn(\`Could not check event listeners for \${element.tagName}:\`, e);
3447
+ // If checking listeners fails, rely on other checks
3448
+ }
3449
+ // if the element is not strictly interactive but appears clickable based on heuristic signals
3450
+ if (isHeuristicallyInteractive(element)) {
3451
+ return true;
3452
+ }
3453
+ // Default to false: if it's interactive but doesn't match above,
3454
+ // assume it triggers the same action as the parent.
3455
+ return false;
3456
+ }
3457
+ // --- End distinct interaction check ---
3458
+ /**
3459
+ * Calculates Intersection over Union (IoU) for two rectangles.
3460
+ * Returns a value between 0 (no overlap) and 1 (identical).
3461
+ */
3462
+ function calculateIoU(rect1, rect2) {
3463
+ // Calculate intersection
3464
+ const xOverlap = Math.max(0, Math.min(rect1.right, rect2.right) - Math.max(rect1.left, rect2.left));
3465
+ const yOverlap = Math.max(0, Math.min(rect1.bottom, rect2.bottom) - Math.max(rect1.top, rect2.top));
3466
+ const intersectionArea = xOverlap * yOverlap;
3467
+ // Calculate union
3468
+ const area1 = rect1.width * rect1.height;
3469
+ const area2 = rect2.width * rect2.height;
3470
+ const unionArea = area1 + area2 - intersectionArea;
3471
+ // Avoid division by zero
3472
+ if (unionArea === 0)
3473
+ return 0;
3474
+ return intersectionArea / unionArea;
3475
+ }
3476
+ /**
3477
+ * Checks if two rects are effectively the same using IoU threshold.
3478
+ */
3479
+ function areRectsEqual(rect1, rect2) {
3480
+ return calculateIoU(rect1, rect2) >= sameRectIoUThreshold;
3481
+ }
3482
+ /**
3483
+ * Checks if an element can actually receive pointer events.
3484
+ * Returns false if the element is hidden, disabled, or has pointer-events: none.
3485
+ */
3486
+ function canReceivePointerEvents(element) {
3487
+ const style = getCachedComputedStyle(element);
3488
+ // Check CSS properties that prevent events
3489
+ if (style?.pointerEvents === 'none')
3490
+ return false;
3491
+ if (style?.visibility === 'hidden')
3492
+ return false;
3493
+ if (style?.display === 'none')
3494
+ return false;
3495
+ // Check disabled attribute for form elements
3496
+ if (element.disabled)
3497
+ return false;
3498
+ return true;
3499
+ }
3500
+ /**
3501
+ * Checks if an element has an interactive descendant with the same bounding rect.
3502
+ * If so, the descendant should be highlighted instead of this element (innermost wins).
3503
+ */
3504
+ function hasInteractiveDescendantWithSameRect(element, rect) {
3505
+ for (const child of element.children) {
3506
+ if (!(child instanceof HTMLElement))
3507
+ continue;
3508
+ if (!canReceivePointerEvents(child))
3509
+ continue;
3510
+ const childRect = child.getBoundingClientRect();
3511
+ if (!areRectsEqual(rect, childRect))
3512
+ continue;
3513
+ // Child has same rect - check if it's interactive
3514
+ if (isInteractiveElement(child) || isElementScrollable(child)) {
3515
+ return true;
3516
+ }
3517
+ // Child has same rect but isn't interactive - check its descendants
3518
+ if (hasInteractiveDescendantWithSameRect(child, rect)) {
3519
+ return true;
3520
+ }
3521
+ }
3522
+ return false;
3523
+ }
3524
+ /**
3525
+ * Handles the logic for deciding whether to highlight an element and performing the highlight.
3526
+ */
3527
+ function handleHighlighting(nodeData, node, parentIframe, parentHighlightedRect) {
3528
+ if (!nodeData.isInteractive)
3529
+ return { highlighted: false, rect: null }; // Not interactive, definitely don't highlight
3530
+ // Apply action intent filter - skip elements that don't match the specified intent
3531
+ const actionIntent = args.actionIntent || 'all';
3532
+ if (actionIntent !== 'all' && !matchesActionIntent(node, actionIntent)) {
3533
+ return { highlighted: false, rect: null };
3534
+ }
3535
+ // Check if element can actually receive pointer events
3536
+ if (!canReceivePointerEvents(node)) {
3537
+ return { highlighted: false, rect: null };
3538
+ }
3539
+ const currentRect = node.getBoundingClientRect();
3540
+ // Check if there's an interactive descendant with the same rect
3541
+ // If so, skip this element - let the innermost element be highlighted
3542
+ if (hasInteractiveDescendantWithSameRect(node, currentRect)) {
3543
+ return { highlighted: false, rect: null };
3544
+ }
3545
+ let shouldHighlight = false;
3546
+ if (!parentHighlightedRect) {
3547
+ // Parent wasn't highlighted, this interactive node can be highlighted.
3548
+ shouldHighlight = true;
3549
+ }
3550
+ else {
3551
+ // Parent *was* highlighted. Only highlight this node if it represents a distinct interaction.
3552
+ if (areRectsEqual(currentRect, parentHighlightedRect)) {
3553
+ // Same rect as parent - this is the innermost element, should be highlighted
3554
+ // (parent should have been skipped by hasInteractiveDescendantWithSameRect)
3555
+ shouldHighlight = true;
3556
+ }
3557
+ else if (isElementDistinctInteraction(node, nodeData)) {
3558
+ shouldHighlight = true;
3559
+ }
3560
+ else {
3561
+ // console.log(\`Skipping highlight for \${nodeData.tagName} (parent highlighted)\`);
3562
+ shouldHighlight = false;
3563
+ }
3564
+ }
3565
+ if (shouldHighlight) {
3566
+ const attributeNames = node.getAttributeNames?.() || [];
3567
+ for (const name of attributeNames) {
3568
+ const value = node.getAttribute(name);
3569
+ nodeData.attributes[name] = value;
3570
+ }
3571
+ // Check viewport status before assigning index and highlighting
3572
+ if (nodeData.isInViewport === undefined) {
3573
+ nodeData.isInViewport = isInExpandedViewport(node, viewportExpansion);
3574
+ }
3575
+ // When viewportExpansion is -1, all interactive elements should get a highlight index
3576
+ // regardless of viewport status
3577
+ if (nodeData.isInViewport || viewportExpansion === -1) {
3578
+ nodeData.highlightIndex = highlightIndex++;
3579
+ if (doHighlightElements) {
3580
+ // Collect elements for deferred highlighting (after tree-based flow detection)
3581
+ if (focusHighlightIndex >= 0) {
3582
+ if (focusHighlightIndex === nodeData.highlightIndex) {
3583
+ elementsToHighlight.push({ element: node, index: nodeData.highlightIndex, parentIframe });
3584
+ }
3585
+ }
3586
+ else {
3587
+ elementsToHighlight.push({ element: node, index: nodeData.highlightIndex, parentIframe });
3588
+ }
3589
+ return { highlighted: true, rect: currentRect }; // Will be highlighted after traversal
3590
+ }
3591
+ // Even if not drawing highlights, we still "highlighted" for tracking purposes
3592
+ return { highlighted: true, rect: currentRect };
3593
+ }
3594
+ else {
3595
+ // console.log(\`Skipping highlight for \${nodeData.tagName} (outside viewport)\`);
3596
+ }
3597
+ }
3598
+ return { highlighted: false, rect: null }; // Did not highlight
3599
+ }
3600
+ function isElementScrollable(element) {
3601
+ const listenedEvents = getCachedNodeEventListeners(element);
3602
+ if (listenedEvents && listenedEvents.includes('scroll')) {
3603
+ const hasScrollableX = element.scrollWidth > element.clientWidth;
3604
+ const hasScrollableY = element.scrollHeight > element.clientHeight;
3605
+ return hasScrollableX || hasScrollableY;
3606
+ }
3607
+ const style = getCachedComputedStyle(element);
3608
+ const hasScrollableX = ['auto', 'scroll'].includes(style?.overflowX || '') &&
3609
+ element.scrollWidth > element.clientWidth;
3610
+ const hasScrollableY = ['auto', 'scroll'].includes(style?.overflowY || '') &&
3611
+ element.scrollHeight > element.clientHeight;
3612
+ return hasScrollableX || hasScrollableY;
3613
+ }
3614
+ /**
3615
+ * Creates a node data object for a given node and its descendants.
3616
+ */
3617
+ function buildDomTree(node, parentIframe = null, parentHighlightedRect = null) {
3618
+ // Fast rejection checks first
3619
+ if (!node || node.id === HIGHLIGHT_CONTAINER_ID) {
3620
+ return null;
3621
+ }
3622
+ if (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE) {
3623
+ return null;
3624
+ }
3625
+ // Special handling for root node (body)
3626
+ if (node === document.body) {
3627
+ const nodeData = {
3628
+ tagName: 'body',
3629
+ attributes: {},
3630
+ xpath: '/body',
3631
+ children: [],
3632
+ };
3633
+ // Process children of body
3634
+ for (const child of node.childNodes) {
3635
+ const domElement = buildDomTree(child, parentIframe, null); // Body's children have no highlighted parent initially
3636
+ if (domElement)
3637
+ nodeData.children.push(domElement);
3638
+ }
3639
+ const id = \`\${ID.current++}\`;
3640
+ DOM_HASH_MAP[id] = nodeData;
3641
+ return id;
3642
+ }
3643
+ // Process text nodes
3644
+ if (node.nodeType === Node.TEXT_NODE) {
3645
+ const textContent = node.textContent?.trim();
3646
+ if (!textContent) {
3647
+ return null;
3648
+ }
3649
+ // Only check visibility for text nodes that might be visible
3650
+ const parentElement = node.parentElement;
3651
+ if (!parentElement || parentElement.tagName.toLowerCase() === 'script') {
3652
+ return null;
3653
+ }
3654
+ const id = \`\${ID.current++}\`;
3655
+ DOM_HASH_MAP[id] = {
3656
+ type: "TEXT_NODE",
3657
+ text: textContent,
3658
+ isVisible: isTextNodeVisible(node),
3659
+ };
3660
+ return id;
3661
+ }
3662
+ // Quick checks for element nodes
3663
+ if (node.nodeType === Node.ELEMENT_NODE && !isElementAccepted(node)) {
3664
+ return null;
3665
+ }
3666
+ const element = node;
3667
+ const nodeData = {
3668
+ tagName: element.tagName.toLowerCase(),
3669
+ attributes: {},
3670
+ xpath: getXPathTree(element, true),
3671
+ children: [],
3672
+ };
3673
+ // Get attributes for interactive elements or potential text containers
3674
+ if (element.tagName.toLowerCase() === 'iframe' || element.tagName.toLowerCase() === 'body') {
3675
+ const attributeNames = element.getAttributeNames?.() || [];
3676
+ for (const name of attributeNames) {
3677
+ const value = element.getAttribute(name);
3678
+ nodeData.attributes[name] = value;
3679
+ }
3680
+ }
3681
+ let highlightResult = { highlighted: false, rect: null };
3682
+ // Perform visibility, interactivity, and highlighting checks
3683
+ if (node.nodeType === Node.ELEMENT_NODE) {
3684
+ if (alwaysHighlightFileInput && element.tagName.toLowerCase() === 'input' && element.type === 'file') {
3685
+ nodeData.isTopElement = true;
3686
+ if (nodeData.isTopElement) {
3687
+ nodeData.isInteractive = true;
3688
+ nodeData.isInViewport = true; // File inputs should always be considered in viewport
3689
+ // Call the dedicated highlighting function
3690
+ highlightResult = handleHighlighting(nodeData, element, parentIframe, parentHighlightedRect);
3691
+ }
3692
+ }
3693
+ else {
3694
+ nodeData.isVisible = isElementVisible(element); // isElementVisible uses offsetWidth/Height, which is fine
3695
+ if (nodeData.isVisible) {
3696
+ nodeData.isTopElement = isTopElement(element);
3697
+ if (nodeData.isTopElement) {
3698
+ let isScrollable = isElementScrollable(element);
3699
+ nodeData.isInteractive = isInteractiveElement(element) || isScrollable;
3700
+ nodeData.isScrollable = isScrollable;
3701
+ nodeData.markAsClickable = shouldMarkAsClickable(element);
3702
+ // Call the dedicated highlighting function
3703
+ highlightResult = handleHighlighting(nodeData, element, parentIframe, parentHighlightedRect);
3704
+ }
3705
+ }
3706
+ }
3707
+ }
3708
+ // Determine the rect to pass to children: use this element's rect if highlighted, otherwise parent's
3709
+ const rectForChildren = highlightResult.highlighted ? highlightResult.rect : parentHighlightedRect;
3710
+ // Process children, with special handling for iframes and rich text editors
3711
+ if (element.tagName) {
3712
+ const tagName = element.tagName.toLowerCase();
3713
+ // Handle iframes
3714
+ if (tagName === "iframe") {
3715
+ try {
3716
+ const iframeEl = element;
3717
+ const iframeDoc = iframeEl.contentDocument || iframeEl.contentWindow?.document;
3718
+ if (iframeDoc) {
3719
+ for (const child of iframeDoc.childNodes) {
3720
+ const domElement = buildDomTree(child, iframeEl, null); // iframes start fresh
3721
+ if (domElement)
3722
+ nodeData.children.push(domElement);
3723
+ }
3724
+ }
3725
+ }
3726
+ catch (e) {
3727
+ console.warn("Unable to access iframe:", e);
3728
+ }
3729
+ }
3730
+ // Handle rich text editors and contenteditable elements
3731
+ else if (element.isContentEditable ||
3732
+ element.getAttribute("contenteditable") === "true" ||
3733
+ element.id === "tinymce" ||
3734
+ element.classList.contains("mce-content-body") ||
3735
+ (tagName === "body" && element.getAttribute("data-id")?.startsWith("mce_"))) {
3736
+ // Process all child nodes to capture formatted text
3737
+ for (const child of element.childNodes) {
3738
+ const domElement = buildDomTree(child, parentIframe, rectForChildren);
3739
+ if (domElement)
3740
+ nodeData.children.push(domElement);
3741
+ }
3742
+ }
3743
+ else {
3744
+ // Handle shadow DOM
3745
+ if (element.shadowRoot) {
3746
+ nodeData.shadowRoot = true;
3747
+ for (const child of element.shadowRoot.childNodes) {
3748
+ const domElement = buildDomTree(child, parentIframe, rectForChildren);
3749
+ if (domElement)
3750
+ nodeData.children.push(domElement);
3751
+ }
3752
+ }
3753
+ // Handle regular elements
3754
+ for (const child of element.childNodes) {
3755
+ const domElement = buildDomTree(child, parentIframe, rectForChildren);
3756
+ if (domElement)
3757
+ nodeData.children.push(domElement);
3758
+ }
3759
+ }
3760
+ }
3761
+ const id = \`\${ID.current++}\`;
3762
+ DOM_HASH_MAP[id] = nodeData;
3763
+ return id;
3764
+ }
3765
+ const rootId = buildDomTree(document.body);
3766
+ // After traversal, render all highlights
3767
+ let treeEntries = [];
3768
+ if (doHighlightElements) {
3769
+ if (phase === 'labels') {
3770
+ // Labels phase: use provided element data to place labels only
3771
+ processLabelsPhase();
3772
+ }
3773
+ else if (elementsToHighlight.length > 0) {
3774
+ // Boxes phase or legacy mode: traverse tree to draw boxes and optionally labels
3775
+ treeEntries = buildTreeEntries();
3776
+ // Use recursive tree processing for correct hot map behavior:
3777
+ // - Pre-order: Mark bounding boxes in hot map (so children avoid parent borders)
3778
+ // - Post-order: Place labels (so parent labels avoid children's labels, skipped in boxes phase)
3779
+ processElementTreeRecursively(elementsToHighlight);
3780
+ }
3781
+ }
3782
+ // Clear the cache before returning
3783
+ DOM_CACHE.clearCache();
3784
+ return {
3785
+ rootId,
3786
+ map: DOM_HASH_MAP,
3787
+ debugLogs,
3788
+ treeEntries,
3789
+ highlightCount: highlightIndex,
3790
+ // Return the final grayscale image state (after all boxes and labels marked) for debugging
3791
+ finalGrayscaleImage: grayscaleImage ?? undefined,
3792
+ // Note: labelSnapshots are now streamed via onSnapshot callback instead of returned
3793
+ // Return element data from boxes phase for use in labels phase
3794
+ elementData: phase === 'boxes' ? collectedElementData : undefined,
3795
+ };
3796
+ })`;function te(e){return e==="about:blank"||e==="chrome://newtab/"||e==="edge://newtab/"||e==="about:newtab"}var Me=class{constructor(e={}){h.debug("\u{1F333} Initializing DomService with options:",e),this.useDomTreeTs=e.useDomTreeTs??!1,this.jsCode=this.useDomTreeTs?ee:Z}async getClickableElements(e,i={}){let{highlightElements:o=!0,focusElement:l=-1,viewportExpansion:t=0,interactiveClassNames:r=[],alwaysHighlightFileInput:n=!1,sameRectIoUThreshold:d,actionIntent:c="all"}=i,[u,a]=await this.buildDomTree(e,o,l,t,r,n,d,c);return{elementTree:u,selectorMap:a}}async getClickableElementsWithScreenshot(e,i={}){if(i.useAccessibilityTree)return this.getClickableElementsWithAXTree(e,i);let o;i.useCleanScreenshot&&(o=await e.screenshot({type:"png",fullPage:!1}));let l=await this.getClickableElements(e,i);await e.waitForTimeout(100),i.useCleanScreenshot||(o=await e.screenshot({type:"png",fullPage:!1})),await this.removeHighlights(e);let t=o.toString("base64"),r;if(i.useSlicedScreenshots)try{r=(await D(o,{resize:i.resizeSlicedScreenshots})).map(d=>d.toString("base64"))}catch(n){h.warn("Failed to slice screenshot:",n)}return{domState:l,screenshotBase64:t,slicedScreenshotsBase64:r}}async getClickableElementsWithAXTree(e,i={}){h.debug("\u{1F333} Using CDP Accessibility Tree for element detection");let o;i.useCleanScreenshot&&(o=await e.screenshot({type:"png",fullPage:!1}));let l=await e.context().newCDPSession(e),{nodes:t}=await l.send("Accessibility.getFullAXTree",{depth:-1});h.debug(`\u{1F4CA} Got ${t.length} AXNodes from accessibility tree`);let r=t.filter(s=>{if(s.ignored)return!1;let f=s.role?.value;return!(!f||!X.has(f)||s.properties?.find(y=>y.name==="disabled")?.value?.value||!s.backendDOMNodeId)});h.debug(`\u2705 Found ${r.length} interactive elements from AXTree`);let n=t.filter(s=>s.role?.value==="button");h.debug(`\u{1F518} Total buttons in AXTree: ${n.length}`);for(let s of n){let f=[];s.ignored&&f.push("ignored"),s.backendDOMNodeId||f.push("no-backendDOMNodeId"),s.properties?.find(y=>y.name==="disabled")?.value?.value&&f.push("disabled"),h.debug(` - "${s.name?.value||"(no name)"}" ${f.length>0?`[SKIPPED: ${f.join(", ")}]`:"[INCLUDED]"}`)}let d=new Set(r.map(s=>s.backendDOMNodeId)),c=await this.getElementsWithEventListeners(l,i.eventListenerLimit??500),u=0;for(let s of c)d.has(s.backendNodeId)||(r.push({nodeId:`synthetic-${s.backendNodeId}`,ignored:!1,backendDOMNodeId:s.backendNodeId,role:{type:"role",value:"generic"},name:{type:"string",value:""},properties:[{name:"eventListeners",value:{type:"string",value:s.eventTypes.join(",")}}]}),d.add(s.backendNodeId),u++);h.debug(`\u{1F3AF} Added ${u} elements from event listeners (total: ${r.length})`);let a=await this.resolveAXNodesToDOM(l,r),m=a.filter(s=>s.isVisible&&s.isInViewport&&s.isTopElement&&s.boundingRect),w=a.filter(s=>!(s.isVisible&&s.isInViewport&&s.isTopElement&&s.boundingRect));if(w.length>0){h.debug(`\u{1F6AB} Filtered out ${w.length} elements:`);for(let s of w){let f=[];s.isVisible||f.push("not-visible"),s.isInViewport||f.push("not-in-viewport"),s.isTopElement||f.push("not-top-element"),s.boundingRect||f.push("no-bounding-rect"),h.debug(` - <${s.tagName}> "${s.axNode.name?.value||""}" [${f.join(", ")}]`)}}h.debug(`\u{1F441}\uFE0F ${m.length} elements are visible and in viewport`);let{domState:E,highlightIndex:p}=await this.buildDomStateFromAXTree(m);i.highlightElements!==!1&&p>0&&(await this.renderHighlightsForAXElements(e,m.slice(0,p)),await e.waitForTimeout(100)),i.useCleanScreenshot||(o=await e.screenshot({type:"png",fullPage:!1})),await this.removeHighlights(e);try{await l.detach()}catch{}let b=o.toString("base64"),g;if(i.useSlicedScreenshots)try{g=(await D(o,{resize:i.resizeSlicedScreenshots})).map(f=>f.toString("base64"))}catch(s){h.warn("Failed to slice screenshot:",s)}return{domState:E,screenshotBase64:b,slicedScreenshotsBase64:g}}async resolveAXNodesToDOM(e,i){let o=[],l=i.map(r=>r.backendDOMNodeId).filter(r=>r!==void 0);if(l.length===0)return o;let t=[];for(let r of l)try{let{object:n}=await e.send("DOM.resolveNode",{backendNodeId:r});t.push(n.objectId||null)}catch{t.push(null)}for(let r=0;r<i.length;r++){let n=i[r],d=t[r];if(d)try{let{result:c}=await e.send("Runtime.callFunctionOn",{objectId:d,functionDeclaration:`function() {
3797
+ const el = this;
3798
+ const rect = el.getBoundingClientRect();
3799
+ const rects = el.getClientRects();
3800
+ const style = window.getComputedStyle(el);
3801
+
3802
+ // Check visibility
3803
+ const isVisible =
3804
+ el.offsetWidth > 0 &&
3805
+ el.offsetHeight > 0 &&
3806
+ style.visibility !== 'hidden' &&
3807
+ style.display !== 'none' &&
3808
+ style.opacity !== '0';
3809
+
3810
+ // Check if in viewport
3811
+ const viewportWidth = window.innerWidth;
3812
+ const viewportHeight = window.innerHeight;
3813
+ const isInViewport = !(
3814
+ rect.bottom < 0 ||
3815
+ rect.top > viewportHeight ||
3816
+ rect.right < 0 ||
3817
+ rect.left > viewportWidth
3818
+ );
3819
+
3820
+ // Check if topmost element
3821
+ let isTopElement = false;
3822
+ if (isVisible && isInViewport) {
3823
+ const centerX = rect.left + rect.width / 2;
3824
+ const centerY = rect.top + rect.height / 2;
3825
+ const topEl = document.elementFromPoint(centerX, centerY);
3826
+ if (topEl) {
3827
+ let current = topEl;
3828
+ while (current && current !== document.documentElement) {
3829
+ if (current === el) {
3830
+ isTopElement = true;
3831
+ break;
3832
+ }
3833
+ current = current.parentElement;
3834
+ }
3835
+ }
3836
+ }
3837
+
3838
+ // Get XPath
3839
+ function getXPath(element) {
3840
+ const segments = [];
3841
+ let current = element;
3842
+ while (current && current.nodeType === Node.ELEMENT_NODE) {
3843
+ const tagName = current.nodeName.toLowerCase();
3844
+ const siblings = current.parentElement
3845
+ ? Array.from(current.parentElement.children).filter(c => c.nodeName.toLowerCase() === tagName)
3846
+ : [];
3847
+ const index = siblings.length > 1 ? siblings.indexOf(current) + 1 : 0;
3848
+ segments.unshift(index > 0 ? tagName + '[' + index + ']' : tagName);
3849
+ current = current.parentNode;
3850
+ }
3851
+ return segments.join('/');
3852
+ }
3853
+
3854
+ // Get attributes
3855
+ const attributes = {};
3856
+ for (const attr of el.attributes) {
3857
+ attributes[attr.name] = attr.value;
3858
+ }
3859
+
3860
+ // Convert client rects to plain objects
3861
+ const clientRectsArray = [];
3862
+ for (const r of rects) {
3863
+ clientRectsArray.push({
3864
+ x: r.x, y: r.y, width: r.width, height: r.height
3865
+ });
3866
+ }
3867
+
3868
+ return {
3869
+ tagName: el.tagName.toLowerCase(),
3870
+ xpath: getXPath(el),
3871
+ attributes: attributes,
3872
+ isVisible: isVisible,
3873
+ isInViewport: isInViewport,
3874
+ isTopElement: isTopElement,
3875
+ boundingRect: rect.width > 0 && rect.height > 0 ? {
3876
+ x: rect.x, y: rect.y, width: rect.width, height: rect.height
3877
+ } : null,
3878
+ clientRects: clientRectsArray
3879
+ };
3880
+ }`,returnByValue:!0});c.value&&o.push({axNode:n,...c.value})}catch(c){h.debug(`Failed to resolve element: ${c}`)}}return o}async getElementsWithEventListeners(e,i=G){let o=[];try{let{root:l}=await e.send("DOM.getDocument",{depth:0}),t=B.join(","),{nodeIds:r}=await e.send("DOM.querySelectorAll",{nodeId:l.nodeId,selector:t});h.debug(`\u{1F50D} Checking ${Math.min(r.length,i)} elements for event listeners`);for(let n of r.slice(0,i))try{let{object:d}=await e.send("DOM.resolveNode",{nodeId:n});if(!d.objectId)continue;let{listeners:c}=await e.send("DOMDebugger.getEventListeners",{objectId:d.objectId}),u=c.filter(a=>$.has(a.type));if(u.length>0){let{node:a}=await e.send("DOM.describeNode",{nodeId:n});o.push({backendNodeId:a.backendNodeId,eventTypes:u.map(m=>m.type)})}await e.send("Runtime.releaseObject",{objectId:d.objectId})}catch{}h.debug(`\u2705 Found ${o.length} elements with interaction event listeners`)}catch(l){h.warn("Failed to get elements with event listeners:",l)}return o}async buildDomStateFromAXTree(e){let i=new Map,o=new C("body","/body",{},[],!0,!1,!1,!1,!0,!0,!1,null),l=0;for(let t of e){let r=t.axNode.role?.value||"",n=t.axNode.name?.value||"",d=["button","link","menuitem","tab","switch"].includes(r),c=new C(t.tagName,t.xpath,t.attributes,[],t.isVisible,!0,r==="scrollbar",d,t.isTopElement,t.isInViewport,!1,l,t.boundingRect?{topLeft:{x:t.boundingRect.x,y:t.boundingRect.y},topRight:{x:t.boundingRect.x+t.boundingRect.width,y:t.boundingRect.y},bottomLeft:{x:t.boundingRect.x,y:t.boundingRect.y+t.boundingRect.height},bottomRight:{x:t.boundingRect.x+t.boundingRect.width,y:t.boundingRect.y+t.boundingRect.height},center:{x:t.boundingRect.x+t.boundingRect.width/2,y:t.boundingRect.y+t.boundingRect.height/2},width:t.boundingRect.width,height:t.boundingRect.height}:null,null,null,o);if(n){let u=new N(n,!0,c);c.children.push(u)}o.children.push(c),i.set(l,c),l++}return{domState:{elementTree:o,selectorMap:i},highlightIndex:l}}async renderHighlightsForAXElements(e,i){let o=["#FF0000","#00FF00","#0000FF","#FFA500","#800080","#008080","#FF69B4","#4B0082","#FF4500","#2E8B57","#DC143C","#4682B4"];await e.evaluate(({elements:l,colors:t})=>{let r="playwright-highlight-container",n=document.getElementById(r);n||(n=document.createElement("div"),n.id=r,n.style.position="fixed",n.style.pointerEvents="none",n.style.top="0",n.style.left="0",n.style.width="100%",n.style.height="100%",n.style.zIndex="2147483647",n.style.backgroundColor="transparent",document.body.appendChild(n)),l.forEach((d,c)=>{if(!d.boundingRect)return;let u=t[c%t.length],a=d.boundingRect,m=document.createElement("div");m.style.position="fixed",m.style.border=`1px solid ${u}`,m.style.backgroundColor="transparent",m.style.pointerEvents="none",m.style.boxSizing="border-box",m.style.top=`${a.y}px`,m.style.left=`${a.x}px`,m.style.width=`${a.width}px`,m.style.height=`${a.height}px`,n.appendChild(m);let w=document.createElement("div");w.style.position="fixed",w.style.background=u,w.style.color="white",w.style.padding="1px 4px",w.style.borderRadius="4px",w.style.fontSize=c>=100?"8px":"12px",w.textContent=String(c);let E=Math.max(0,a.y-16),p=Math.max(0,Math.min(a.x+a.width-20,window.innerWidth-25));w.style.top=`${E}px`,w.style.left=`${p}px`,n.appendChild(w)})},{elements:i,colors:o})}async removeHighlights(e){try{await e.evaluate(()=>{let i=document.getElementById("playwright-highlight-container");i&&i.remove(),window._highlightCleanupFunctions&&(window._highlightCleanupFunctions.forEach(o=>o()),window._highlightCleanupFunctions=[])}),h.debug("\u2705 Highlights removed from page")}catch(i){h.warn("Failed to remove highlights:",i.message)}}async getCrossOriginIframes(e){let i=await e.locator("iframe").filter({hasNot:e.locator(":visible")}).evaluateAll(d=>d.map(c=>c.src)),o=d=>{try{let c=new URL(d);return["doubleclick.net","adroll.com","googletagmanager.com"].some(a=>c.hostname.includes(a))}catch{return!1}},l=e.url(),t=new URL(l).hostname,r=e.frames(),n=[];for(let d of r){let c=d.url();try{let u=new URL(c).hostname;u&&u!==t&&!i.includes(c)&&!o(c)&&n.push(c)}catch{continue}}return n}async buildDomTree(e,i,o,l,t,r,n,d="all"){if(await e.evaluate("1+1")!==2)throw new Error("The page cannot evaluate javascript code properly");if(te(e.url()))return[new C("body","",{},[],!1,!1,!1,!1,!1,!1,!1,null),new Map];let u={doHighlightElements:i,focusHighlightIndex:o,viewportExpansion:l,debugMode:!1,interactiveClassNames:t,alwaysHighlightFileInput:r,sameRectIoUThreshold:n,actionIntent:d};h.debug(`\u{1F527} Starting JavaScript DOM analysis for ${e.url().slice(0,50)}...`);let a,m=null;if(this.useDomTreeTs&&i)try{let p={...u,phase:"boxes",grayscaleImage:null},b=await e.evaluate(({code:v,argsObj:x})=>new Function("return "+v)()(x),{code:this.jsCode,argsObj:p});h.debug(`\u{1F4E6} Phase 1: Drew ${b.elementData?.length||0} bounding boxes`);let g=await e.screenshot({type:"png",fullPage:!1});h.debug("\u{1F4F8} Captured screenshot with bounding boxes");let s=performance.now(),f=await V(g);m=f.pixels;let T=Math.round(performance.now()-s);h.debug(`\u{1F5BC}\uFE0F Generated grayscale image (${f.width}x${f.height}) in ${T}ms`);let y={...u,phase:"labels",grayscaleImage:m,elementData:b.elementData};a=await e.evaluate(({code:v,argsObj:x})=>new Function("return "+v)()(x),{code:this.jsCode,argsObj:y}),a.map=b.map,a.rootId=b.rootId,a.highlightCount=b.highlightCount,a.perfMetrics=b.perfMetrics,h.debug("\u2705 Phase 2: Labels placed using grayscale-based positioning")}catch(p){h.warn("Two-phase rendering failed, falling back to legacy mode:",p.message),m=null;let b={...u,grayscaleImage:null};a=await e.evaluate(({code:g,argsObj:s})=>new Function("return "+g)()(s),{code:this.jsCode,argsObj:b}),h.debug("\u2705 JavaScript DOM analysis completed (legacy mode)")}else try{a=await e.evaluate(({code:p,argsObj:b})=>new Function("return "+p)()(b),{code:this.jsCode,argsObj:u}),h.debug("\u2705 JavaScript DOM analysis completed")}catch(p){throw h.error("Error evaluating JavaScript:",p.message),p}if(!a||typeof a!="object")throw h.error("JavaScript returned invalid result:",a),new Error("JavaScript DOM analysis returned invalid result");if(!a.map||!a.rootId)throw h.error("JavaScript result missing map or rootId:",JSON.stringify(a,null,2)),new Error("JavaScript result missing required fields (map or rootId)");if(d!=="all"&&a.highlightCount===0){h.debug(`\u26A0\uFE0F No elements matched intent '${d}', falling back to 'all'`);let p={...u,actionIntent:"all"};a=await e.evaluate(({code:b,argsObj:g})=>new Function("return "+b)()(g),{code:this.jsCode,argsObj:p})}if(a&&a.perfMetrics){let b=a.perfMetrics.nodeMetrics?.totalNodes??0,g=0;if(a.map)for(let f of Object.values(a.map))typeof f=="object"&&f!==null&&f.isInteractive&&g++;let s=e.url().length>50?e.url().slice(0,50)+"...":e.url();h.debug(`\u{1F50E} Ran buildDOMTree.js interactive element detection on: ${s} interactive=${g}/${b}`)}h.debug("\u{1F504} Starting TypeScript DOM tree construction...");let E=await this.constructDomTree(a);return h.debug("\u2705 TypeScript DOM tree construction completed"),E}async constructDomTree(e){let i=e.map,o=e.rootId,l=new Map,t=new Map;for(let[n,d]of Object.entries(i)){let[c,u]=this.parseNode(d);if(c!==null&&(t.set(n,c),c instanceof C&&c.highlightIndex!==null&&l.set(c.highlightIndex,c),c instanceof C))for(let a of u){let m=t.get(a);m&&(m.parent=c,c.children.push(m))}}let r=t.get(o);if(!r||!(r instanceof C))throw new Error("Failed to parse HTML to dictionary");return[r,l]}parseNode(e){if(!e)return[null,[]];if(e.type==="TEXT_NODE")return[new N(e.text,e.isVisible,null),[]];let i=null;e.viewport&&(i={width:e.viewport.width,height:e.viewport.height,scrollX:e.viewport.scrollX,scrollY:e.viewport.scrollY});let o=new C(e.tagName,e.xpath,e.attributes||{},[],e.isVisible??!1,e.isInteractive??!1,e.isScrollable??!1,e.markAsClickable??!1,e.isTopElement??!1,e.isInViewport??!1,e.shadowRoot??!1,e.highlightIndex??null,e.viewportCoordinates??null,e.pageCoordinates??null,i,null),l=e.children||[];return[o,l]}};export{ie as a,be as b,ye as c,Me as d};