donobu 5.56.0 → 5.57.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/dist/apis/GptConfigsApi.d.ts +5 -5
  2. package/dist/apis/GptConfigsApi.js +14 -14
  3. package/dist/bindings/PageInteractionTracker.d.ts +1 -1
  4. package/dist/bindings/PageInteractionTracker.js +3 -3
  5. package/dist/bindings/SetDonobuAnnotations.d.ts +1 -1
  6. package/dist/bindings/SetDonobuAnnotations.js +3 -3
  7. package/dist/clients/AnthropicGptClient.d.ts +2 -2
  8. package/dist/clients/AnthropicGptClient.js +77 -77
  9. package/dist/clients/OpenAiGptClient.d.ts +14 -14
  10. package/dist/clients/OpenAiGptClient.js +183 -183
  11. package/dist/esm/apis/GptConfigsApi.d.ts +5 -5
  12. package/dist/esm/apis/GptConfigsApi.js +14 -14
  13. package/dist/esm/bindings/PageInteractionTracker.d.ts +1 -1
  14. package/dist/esm/bindings/PageInteractionTracker.js +3 -3
  15. package/dist/esm/bindings/SetDonobuAnnotations.d.ts +1 -1
  16. package/dist/esm/bindings/SetDonobuAnnotations.js +3 -3
  17. package/dist/esm/clients/AnthropicGptClient.d.ts +2 -2
  18. package/dist/esm/clients/AnthropicGptClient.js +77 -77
  19. package/dist/esm/clients/OpenAiGptClient.d.ts +14 -14
  20. package/dist/esm/clients/OpenAiGptClient.js +183 -183
  21. package/dist/esm/lib/ai/PageAi.js +2 -1
  22. package/dist/esm/lib/page/extendPage.js +2 -1
  23. package/dist/esm/lib/test/utils/TestFileUpdater.d.ts +9 -9
  24. package/dist/esm/lib/test/utils/TestFileUpdater.js +49 -49
  25. package/dist/esm/main.d.ts +2 -0
  26. package/dist/esm/managers/AdminApiController.d.ts +16 -16
  27. package/dist/esm/managers/AdminApiController.js +35 -35
  28. package/dist/esm/managers/DonobuFlow.d.ts +41 -33
  29. package/dist/esm/managers/DonobuFlow.js +362 -532
  30. package/dist/esm/managers/DonobuFlowsManager.js +2 -10
  31. package/dist/esm/managers/FlowDependencyAnalyzer.d.ts +12 -12
  32. package/dist/esm/managers/FlowDependencyAnalyzer.js +77 -77
  33. package/dist/esm/managers/PageInspector.d.ts +38 -38
  34. package/dist/esm/managers/PageInspector.js +745 -745
  35. package/dist/esm/managers/TargetInspector.d.ts +28 -33
  36. package/dist/esm/managers/TestsManager.d.ts +25 -25
  37. package/dist/esm/managers/TestsManager.js +74 -74
  38. package/dist/esm/managers/ToolManager.js +7 -5
  39. package/dist/esm/managers/ToolRegistry.d.ts +5 -1
  40. package/dist/esm/managers/WebTargetInspector.d.ts +9 -5
  41. package/dist/esm/managers/WebTargetInspector.js +45 -47
  42. package/dist/esm/models/AiQuery.d.ts +29 -15
  43. package/dist/esm/models/AiQuery.js +31 -0
  44. package/dist/esm/models/InteractableElement.d.ts +6 -0
  45. package/dist/esm/models/InteractableElement.js +7 -1
  46. package/dist/esm/models/Observation.d.ts +38 -0
  47. package/dist/esm/models/Observation.js +3 -0
  48. package/dist/esm/models/ToolCallContext.d.ts +3 -2
  49. package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.d.ts +2 -2
  50. package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.js +19 -18
  51. package/dist/esm/persistence/flows/FlowsPersistenceSqlite.js +2 -1
  52. package/dist/esm/targets/TargetProvider.d.ts +110 -0
  53. package/dist/esm/targets/TargetProvider.js +25 -0
  54. package/dist/esm/targets/TargetRuntime.d.ts +6 -3
  55. package/dist/esm/targets/WebDialogHandler.d.ts +14 -0
  56. package/dist/esm/targets/WebDialogHandler.js +198 -0
  57. package/dist/esm/targets/WebTargetProvider.d.ts +32 -0
  58. package/dist/esm/targets/WebTargetProvider.js +136 -0
  59. package/dist/esm/targets/WebTargetRuntime.d.ts +2 -2
  60. package/dist/esm/targets/WebTargetRuntime.js +2 -1
  61. package/dist/esm/tools/AssertPageTool.d.ts +1 -1
  62. package/dist/esm/tools/AssertPageTool.js +3 -3
  63. package/dist/esm/tools/DetectBrokenLinksTool.d.ts +2 -2
  64. package/dist/esm/tools/DetectBrokenLinksTool.js +44 -44
  65. package/dist/esm/tools/InputFakerTool.d.ts +4 -4
  66. package/dist/esm/tools/InputFakerTool.js +10 -10
  67. package/dist/esm/tools/InputTextTool.d.ts +4 -4
  68. package/dist/esm/tools/InputTextTool.js +7 -7
  69. package/dist/esm/tools/ReplayableInteraction.d.ts +34 -34
  70. package/dist/esm/tools/ReplayableInteraction.js +245 -245
  71. package/dist/esm/utils/BrowserUtils.d.ts +19 -19
  72. package/dist/esm/utils/BrowserUtils.js +57 -57
  73. package/dist/esm/utils/MiscUtils.d.ts +2 -2
  74. package/dist/esm/utils/MiscUtils.js +16 -16
  75. package/dist/esm/utils/PlaywrightUtils.d.ts +1 -1
  76. package/dist/esm/utils/TargetUtils.d.ts +1 -1
  77. package/dist/esm/utils/TargetUtils.js +15 -13
  78. package/dist/lib/ai/PageAi.js +2 -1
  79. package/dist/lib/page/extendPage.js +2 -1
  80. package/dist/lib/test/utils/TestFileUpdater.d.ts +9 -9
  81. package/dist/lib/test/utils/TestFileUpdater.js +49 -49
  82. package/dist/main.d.ts +2 -0
  83. package/dist/managers/AdminApiController.d.ts +16 -16
  84. package/dist/managers/AdminApiController.js +35 -35
  85. package/dist/managers/DonobuFlow.d.ts +41 -33
  86. package/dist/managers/DonobuFlow.js +362 -532
  87. package/dist/managers/DonobuFlowsManager.js +2 -10
  88. package/dist/managers/FlowDependencyAnalyzer.d.ts +12 -12
  89. package/dist/managers/FlowDependencyAnalyzer.js +77 -77
  90. package/dist/managers/PageInspector.d.ts +38 -38
  91. package/dist/managers/PageInspector.js +745 -745
  92. package/dist/managers/TargetInspector.d.ts +28 -33
  93. package/dist/managers/TestsManager.d.ts +25 -25
  94. package/dist/managers/TestsManager.js +74 -74
  95. package/dist/managers/ToolManager.js +7 -5
  96. package/dist/managers/ToolRegistry.d.ts +5 -1
  97. package/dist/managers/WebTargetInspector.d.ts +9 -5
  98. package/dist/managers/WebTargetInspector.js +45 -47
  99. package/dist/models/AiQuery.d.ts +29 -15
  100. package/dist/models/AiQuery.js +31 -0
  101. package/dist/models/InteractableElement.d.ts +6 -0
  102. package/dist/models/InteractableElement.js +7 -1
  103. package/dist/models/Observation.d.ts +38 -0
  104. package/dist/models/Observation.js +3 -0
  105. package/dist/models/ToolCallContext.d.ts +3 -2
  106. package/dist/persistence/flows/FlowsPersistenceDonobuApi.d.ts +2 -2
  107. package/dist/persistence/flows/FlowsPersistenceDonobuApi.js +19 -18
  108. package/dist/persistence/flows/FlowsPersistenceSqlite.js +2 -1
  109. package/dist/targets/TargetProvider.d.ts +110 -0
  110. package/dist/targets/TargetProvider.js +25 -0
  111. package/dist/targets/TargetRuntime.d.ts +6 -3
  112. package/dist/targets/WebDialogHandler.d.ts +14 -0
  113. package/dist/targets/WebDialogHandler.js +198 -0
  114. package/dist/targets/WebTargetProvider.d.ts +32 -0
  115. package/dist/targets/WebTargetProvider.js +136 -0
  116. package/dist/targets/WebTargetRuntime.d.ts +2 -2
  117. package/dist/targets/WebTargetRuntime.js +2 -1
  118. package/dist/tools/AssertPageTool.d.ts +1 -1
  119. package/dist/tools/AssertPageTool.js +3 -3
  120. package/dist/tools/DetectBrokenLinksTool.d.ts +2 -2
  121. package/dist/tools/DetectBrokenLinksTool.js +44 -44
  122. package/dist/tools/InputFakerTool.d.ts +4 -4
  123. package/dist/tools/InputFakerTool.js +10 -10
  124. package/dist/tools/InputTextTool.d.ts +4 -4
  125. package/dist/tools/InputTextTool.js +7 -7
  126. package/dist/tools/ReplayableInteraction.d.ts +34 -34
  127. package/dist/tools/ReplayableInteraction.js +245 -245
  128. package/dist/utils/BrowserUtils.d.ts +19 -19
  129. package/dist/utils/BrowserUtils.js +57 -57
  130. package/dist/utils/MiscUtils.d.ts +2 -2
  131. package/dist/utils/MiscUtils.js +16 -16
  132. package/dist/utils/PlaywrightUtils.d.ts +1 -1
  133. package/dist/utils/TargetUtils.d.ts +1 -1
  134. package/dist/utils/TargetUtils.js +15 -13
  135. package/package.json +2 -1
@@ -65,695 +65,120 @@ class PageInspector {
65
65
  this.interactableAnnotationAttribute = interactableAnnotationAttribute;
66
66
  }
67
67
  /**
68
- * Assigns a globally unique attribute to all visible and interactable elements in the page.
69
- *
70
- * This method performs the following steps:
71
- * 1. Removes any pre-existing interactable element attributes from the page
72
- * 2. Assigns sequential numeric values as attributes to interactable elements in the main frame
73
- * 3. Processes child frames that are visible in the viewport and assigns attributes to their interactable elements
74
- *
75
- * The method identifies "interactable" elements based on tag names, ARIA roles, CSS classes, and other heuristics.
76
- * Only elements that are:
77
- * - Visible (non-zero dimensions and not hidden via CSS)
78
- * - More than 50% in the viewport
79
- * - Not disabled or inert
80
- * - Actually reachable at their coordinates (topmost in z-index)
81
- * will receive the attribute.
82
- *
83
- * @param page - The Playwright Page object to process
84
- * @throws {PageClosedException} If the page is closed during processing
85
- * @returns {Promise<void>} A promise that resolves when all elements have been attributed
68
+ * Converts an HTML attribute to a JavaScript attribute. For example,
69
+ * "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
70
+ * prefix, and the conversion from kebab-case to camelCase.
86
71
  */
87
- async attributeInteractableElements(page) {
88
- try {
89
- // Remove any preexisting attributes
90
- await this.deattributeInteractableElements(page);
91
- // Get viewport dimensions and scroll position properly
92
- const viewportInfo = await page.evaluate(() => {
93
- return {
94
- viewportWidth: window.innerWidth,
95
- viewportHeight: window.innerHeight,
96
- scrollX: window.scrollX || window.pageXOffset,
97
- scrollY: window.scrollY || window.pageYOffset,
98
- };
99
- });
100
- // 1) Attribute elements in the main page
101
- let annotationOffset = await page.evaluate(PageInspector.attributeElementsInContext, [0, this.interactableElementAttribute]);
102
- // 2) Check child frames, attributing elements if the frame is (partially) in view
103
- const frames = page
104
- .frames()
105
- .filter((frame) => PageInspector.frameFilter(frame) && frame !== page.mainFrame());
106
- for (const frame of frames) {
107
- const elementHandle = await frame.frameElement();
108
- if (!elementHandle) {
109
- continue;
110
- }
111
- const boundingBox = await elementHandle.boundingBox();
112
- if (!boundingBox) {
113
- continue;
114
- }
115
- // boundingBox coordinates are already in viewport space, so we need to account for scroll
116
- const isInViewport = boundingBox.x + boundingBox.width > 0 &&
117
- boundingBox.x < viewportInfo.viewportWidth &&
118
- boundingBox.y + boundingBox.height > 0 &&
119
- boundingBox.y < viewportInfo.viewportHeight;
120
- if (isInViewport) {
121
- annotationOffset = await frame.evaluate(PageInspector.attributeElementsInContext, [annotationOffset, this.interactableElementAttribute]);
122
- }
123
- }
124
- }
125
- catch (error) {
126
- if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
127
- throw new PageClosedException_1.PageClosedException();
128
- }
129
- else {
130
- throw error;
131
- }
72
+ static convertToJsAttribute(htmlAttribute) {
73
+ if (htmlAttribute.startsWith('data-')) {
74
+ htmlAttribute = htmlAttribute.substring(5);
132
75
  }
76
+ const parts = htmlAttribute.split('-');
77
+ const jsAttribute = parts[0] +
78
+ parts
79
+ .slice(1)
80
+ .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase())
81
+ .join('');
82
+ return jsAttribute;
133
83
  }
134
84
  /**
135
- * Retrieves all elements that have been previously attributed with the interactable element attribute.
85
+ * An internal method that is injected into page/frame contexts to find and attribute interactable elements.
136
86
  *
137
87
  * This method:
138
- * 1. Searches all frames in the page (including the main frame and child frames)
139
- * 2. Collects elements with the {@link interactableElementAttribute} attribute
140
- * 3. Creates an {@link InteractableElement} object for each attributed element
88
+ * 1. Identifies potentially interactable elements using a comprehensive selector
89
+ * 2. Filters elements based on visibility, position in viewport, and interactability
90
+ * 3. Assigns unique sequential numeric values to the interactable attribute
141
91
  *
142
- * For each interactable element, it extracts:
143
- * - The attribute value (serving as a unique identifier)
144
- * - A simplified HTML snippet representation of the element
145
- * * For 'select' elements, the complete HTML (including options) is preserved
146
- * * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
147
- * * For all other elements, only the opening tag without children is captured
148
- * * For the main scrolling element (document.scrollingElement), adds special decoration indicating it's the page's main scrolling element
92
+ * The method uses several criteria to determine if an element is truly interactable:
93
+ * - Element must be visible (non-zero dimensions, not hidden via CSS)
94
+ * - Element must have at least 50% of its area within the viewport
95
+ * - Element must not be disabled, inert, or have pointer-events:none
96
+ * - Element must be the topmost element at its coordinates (using point sampling)
149
97
  *
150
- * Note: This method only finds elements that have been previously attributed using
151
- * the {@link attributeInteractableElements} method.
98
+ * Special handling is provided for label elements, which will attribute their
99
+ * associated form controls as well.
152
100
  *
153
- * @param page - The Playwright Page object to process
154
- * @returns {Promise<InteractableElement[]>} A promise that resolves to an array of
155
- * interactable elements with their attribute values and HTML snippets
156
- * @throws {PageClosedException} If the page is closed during processing
101
+ * This method can process both standard DOM elements and elements within shadow roots,
102
+ * ensuring thorough coverage of modern web applications.
157
103
  *
158
- * @example
159
- * const inspector = new PageInspector();
160
- * await inspector.attributeInteractableElements(page);
161
- * const elements = await inspector.getAttributedInteractableElements(page);
162
- * // elements = [{ donobuAttributeValue: "0", htmlSnippet: "<button id=\"submit\">Submit</button>"}]
104
+ * @param arg - A tuple containing [offset: number, interactableAttribute: string]
105
+ * where offset is the starting value for sequential numbering and
106
+ * interactableAttribute is the attribute name to assign
107
+ * @returns The updated offset after assigning attributes (for sequential numbering across frames)
108
+ * @private
109
+ *
110
+ * @remarks
111
+ * This method is designed to be injected into the page context using page.evaluate()
112
+ * and should not be called directly from Node.js code.
163
113
  */
164
- async getAttributedInteractableElements(page) {
165
- try {
166
- const frames = page.frames().filter(PageInspector.frameFilter);
167
- const aggregate = {};
168
- for (const frame of frames) {
169
- const frameMap = await frame.evaluate((interactableAttr) => {
170
- /* --- helpers running in the browser context --- */
171
- function stripDonobuAttrs(el) {
172
- const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
173
- let node = el;
174
- while (node) {
175
- Array.from(node.attributes).forEach((attr) => {
176
- // Strip out Donobu attributes since those are not a part of the
177
- // original HTML.
178
- if (attr.name.startsWith('data-donobu')) {
179
- node.removeAttribute(attr.name);
180
- }
181
- });
182
- node = walker.nextNode();
183
- }
184
- }
185
- /** helper to compute live scroll directions for el */
186
- function getScrollDirections(el) {
187
- // Special case for when the document body is not the scrollingElement
188
- // element. This may happen if the scrollingElement is the
189
- // root <html> element. In this case, it makes no sense to report
190
- // scrollability on <body> and on scrollingElement, since we should
191
- // use the scrollingElement instead.
192
- if (el === document.body &&
193
- document.scrollingElement !== document.body) {
194
- return [];
195
- }
196
- const dirs = [];
197
- const isRoot = el === document.scrollingElement;
198
- const style = getComputedStyle(el);
199
- // Add a small margin so we do not waste time reporting scrollability
200
- // for an element that is not materially scrollable.
201
- const marginPx = 1;
202
- const canY = el.scrollHeight > el.clientHeight + marginPx &&
203
- (isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowY));
204
- const canX = el.scrollWidth > el.clientWidth + marginPx &&
205
- (isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowX));
206
- if (canY) {
207
- if (el.scrollTop > 0) {
208
- dirs.push('UP');
209
- }
210
- if (el.scrollTop < el.scrollHeight - el.clientHeight) {
211
- dirs.push('DOWN');
212
- }
213
- }
214
- if (canX) {
215
- if (el.scrollLeft > 0) {
216
- dirs.push('LEFT');
217
- }
218
- if (el.scrollLeft < el.scrollWidth - el.clientWidth) {
219
- dirs.push('RIGHT');
220
- }
221
- }
222
- return dirs;
223
- }
224
- function serialise(el) {
225
- const deepClone = (el.tagName.toLowerCase() === 'select'
226
- ? el.cloneNode(true)
227
- : el.cloneNode(false));
228
- stripDonobuAttrs(deepClone);
229
- if (el.tagName.toLowerCase() === 'select') {
230
- const scrollComment = el === document.scrollingElement
231
- ? '<!-- This is the main page scrolling element -->'
232
- : '';
233
- return scrollComment + deepClone.outerHTML; // full markup incl. <option>s
234
- }
235
- // Get the text content of the original element
236
- const textContent = el.textContent?.trim() || '';
237
- if (textContent) {
238
- // Truncate text if longer than 32 characters
239
- const displayText = textContent.length > 32
240
- ? textContent.substring(0, 32) + '...'
241
- : textContent;
242
- // Return opening tag + text + closing tag
243
- const fullTag = deepClone.outerHTML;
244
- const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
245
- const tagName = el.tagName.toLowerCase();
246
- const scrollComment = el === document.scrollingElement
247
- ? '<!-- This is the main page scrolling element -->'
248
- : '';
249
- return `${scrollComment}${openingTag}${displayText}</${tagName}>`;
250
- }
251
- else {
252
- // opening tag only
253
- const html = deepClone.outerHTML;
254
- const scrollComment = el === document.scrollingElement
255
- ? '<!-- This is the main page scrolling element -->'
256
- : '';
257
- return scrollComment + html.slice(0, html.indexOf('>') + 1);
258
- }
259
- }
260
- const out = {};
261
- // Recursively process document and all shadow roots
262
- const processNode = (root) => {
263
- // Find elements with the interactable attribute
264
- root.querySelectorAll(`[${interactableAttr}]`).forEach((el) => {
265
- const val = el.getAttribute(interactableAttr);
266
- if (!val) {
267
- return;
268
- }
269
- out[val] = {
270
- htmlSnippet: serialise(el),
271
- scrollable: getScrollDirections(el),
272
- };
273
- });
274
- // Recursively process any child shadow roots
275
- root.querySelectorAll('*').forEach((el) => {
276
- if (el.shadowRoot) {
277
- processNode(el.shadowRoot);
278
- }
279
- });
280
- };
281
- // Start processing from the document root
282
- processNode(document);
283
- return out;
284
- }, this.interactableElementAttribute);
285
- Object.assign(aggregate, frameMap);
286
- }
287
- return Object.keys(aggregate)
288
- .sort((a, b) => Number(a) - Number(b))
289
- .map((key) => ({
290
- donobuAttributeValue: key,
291
- htmlSnippet: aggregate[key].htmlSnippet,
292
- scrollable: aggregate[key].scrollable,
293
- }));
294
- }
295
- catch (error) {
296
- if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
297
- throw new PageClosedException_1.PageClosedException();
298
- }
299
- else {
300
- throw error;
301
- }
114
+ static attributeElementsInContext(arg) {
115
+ let offset = arg[0];
116
+ const interactableAttribute = arg[1];
117
+ // --- Utility Functions ---
118
+ function isElementVisible(rect, style) {
119
+ return (rect.width > 0 &&
120
+ rect.height > 0 &&
121
+ style.display !== 'none' &&
122
+ style.visibility !== 'hidden');
302
123
  }
303
- }
304
- /**
305
- * Visually annotates all interactable elements with numbered indicators on the page.
306
- *
307
- * This method:
308
- * 1. Processes all accessible frames in the page
309
- * 2. Creates (or reuses) a shadow DOM container to isolate annotation styling
310
- * 3. Places circular numbered indicators over each element that has the
311
- * {@link interactableElementAttribute} attribute
312
- *
313
- * The annotations:
314
- * - Are positioned at the center of each interactable element
315
- * - Have the same numeric value as the element's attribute
316
- * - Are styled as black circles with red borders and white text
317
- * - Are placed in a shadow DOM to avoid style conflicts with the page
318
- * - Have the {@link interactableAnnotationAttribute} for identification
319
- * - Are non-interactive (pointer-events: none)
320
- *
321
- * Note: This method requires elements to be previously attributed using the
322
- * {@link attributeInteractableElements} method to find the elements to annotate.
323
- *
324
- * @param page - The Playwright Page object to process
325
- * @returns {Promise<void>} A promise that resolves when all elements have been annotated
326
- * @throws {PageClosedException} If the page is closed during processing
327
- *
328
- * @example
329
- * const inspector = new PageInspector();
330
- * await inspector.attributeInteractableElements(page);
331
- * await inspector.annotateInteractableElements(page);
332
- */
333
- async annotateInteractableElements(page) {
334
- try {
335
- // Filter frames as needed
336
- const frames = page
337
- .frames()
338
- .filter((frame) => PageInspector.frameFilter(frame));
339
- for (const frame of frames) {
340
- await frame.evaluate(([interactableAttr, annotationAttr]) => {
341
- // 1) Ensure we have a shadow container in the main document
342
- let container = document.getElementById('annotation-shadow-container');
343
- if (!container) {
344
- container = document.createElement('div');
345
- container.id = 'annotation-shadow-container';
346
- // Position container so child elements can be absolutely placed
347
- Object.assign(container.style, {
348
- position: 'absolute',
349
- top: '0',
350
- left: '0',
351
- width: '100%',
352
- height: '100%',
353
- pointerEvents: 'none', // Let clicks pass through
354
- zIndex: '2147483647', // win every z-index fight
355
- });
356
- // Check if document.body exists before trying to append.
357
- if (document.body) {
358
- document.body.appendChild(container);
359
- }
360
- else if (document.documentElement) {
361
- // Fall back to document.documentElement if body does not exist.
362
- document.documentElement.appendChild(container);
363
- }
364
- else {
365
- // If neither exists, we can't proceed with annotations in this frame.
366
- console.warn(`Cannot create annotation container for ${window.location.href} since the document structure is incomplete`);
367
- return;
368
- }
369
- // Attach a shadow root
370
- const shadowRoot = container.attachShadow({ mode: 'open' });
371
- // Add a <style> element inside the shadow root to reset and define annotation styles
372
- const style = document.createElement('style');
373
- style.textContent = `
374
- :host {
375
- all: initial; /* Reset styles in shadow root */
376
- }
377
- .annotation {
378
- position: absolute;
379
- z-index: 2147483647;
380
- background-color: black;
381
- color: white;
382
- width: 40px;
383
- height: 40px;
384
- border-radius: 50%;
385
- display: flex;
386
- align-items: center;
387
- justify-content: center;
388
- font-size: 14px;
389
- font-weight: bold;
390
- line-height: 20px;
391
- text-align: center;
392
- box-shadow: 0px 2px 4px rgba(0,0,0,0.2);
393
- border: 4px solid #FF4136;
394
- pointer-events: none;
124
+ function isElementMoreThanHalfInViewport(rect) {
125
+ const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
126
+ const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
127
+ const visibleWidth = Math.min(rect.right, viewportWidth) - Math.max(rect.left, 0);
128
+ const visibleHeight = Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0);
129
+ const visibleArea = Math.max(0, visibleWidth) * Math.max(0, visibleHeight);
130
+ const elementArea = rect.width * rect.height;
131
+ return visibleArea >= elementArea / 2;
132
+ }
133
+ function isElementEnabled(element, style) {
134
+ // Check standard disabled attribute (for form controls like button, input, etc.)
135
+ if (element.hasAttribute('disabled')) {
136
+ return false;
137
+ }
138
+ // Check for ARIA attributes that indicate non-interactivity
139
+ if (element.getAttribute('aria-hidden') === 'true') {
140
+ return false;
141
+ }
142
+ // Check for pointer-events: none which prevents interactions
143
+ if (style.pointerEvents === 'none') {
144
+ return false;
145
+ }
146
+ // Check for inert attribute which makes elements non-interactive
147
+ if (element.hasAttribute('inert')) {
148
+ return false;
149
+ }
150
+ // If the element is in a form and the fieldset is disabled, it might be disabled as well
151
+ let parent = element.parentElement;
152
+ while (parent) {
153
+ if (parent.tagName.toLowerCase() === 'fieldset' &&
154
+ parent.hasAttribute('disabled') &&
155
+ element.tagName.toLowerCase() !== 'legend') {
156
+ return false;
395
157
  }
396
- `;
397
- shadowRoot.appendChild(style);
398
- }
399
- // Retrieve the shadow root to place annotation elements
400
- const containerEl = document.getElementById('annotation-shadow-container');
401
- if (!containerEl?.shadowRoot) {
402
- return;
403
- }
404
- const shadowRoot = containerEl.shadowRoot;
405
- // 2) Factory to create a new annotation inside the shadow root
406
- const createAnnotation = (value) => {
407
- const annotation = document.createElement('div');
408
- annotation.classList.add('annotation');
409
- annotation.dataset[annotationAttr] = '1';
410
- annotation.textContent = value;
411
- return annotation;
412
- };
413
- // 3) Position annotation relative to an element
414
- const positionAnnotation = (annotation, element) => {
415
- const rect = element.getBoundingClientRect();
416
- // Center the annotation on the element, adjusting for its size
417
- // Since container is absolute, we need to account for scroll position
418
- const x = rect.left + rect.width / 2 - 20 + window.scrollX;
419
- const y = rect.top + rect.height / 2 - 20 + window.scrollY;
420
- annotation.style.left = `${x}px`;
421
- annotation.style.top = `${y}px`;
422
- };
423
- // 4) Traverse DOM (including any nested shadow roots) to find interactable elements
424
- const processNode = (root) => {
425
- // Find elements with the interactable attribute
426
- const elements = root.querySelectorAll(`[${interactableAttr}]`);
427
- elements.forEach((element) => {
428
- const value = element.getAttribute(interactableAttr);
429
- if (value) {
430
- const annotation = createAnnotation(value);
431
- shadowRoot.appendChild(annotation);
432
- positionAnnotation(annotation, element);
433
- }
434
- });
435
- // Recursively process any child shadow roots
436
- root.querySelectorAll('*').forEach((el) => {
437
- if (el.shadowRoot) {
438
- processNode(el.shadowRoot);
439
- }
440
- });
441
- };
442
- // Start processing from the (frame) document root
443
- processNode(document);
444
- }, [
445
- this.interactableElementAttribute,
446
- PageInspector.convertToJsAttribute(this.interactableAnnotationAttribute),
447
- ]);
158
+ parent = parent.parentElement;
448
159
  }
160
+ return true;
449
161
  }
450
- catch (error) {
451
- if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
452
- throw new PageClosedException_1.PageClosedException();
162
+ function isScrollable(el) {
163
+ const canY = el.scrollHeight > el.clientHeight;
164
+ const canX = el.scrollWidth > el.clientWidth;
165
+ // If nothing overflows, bail early
166
+ if (!canY && !canX) {
167
+ return false;
453
168
  }
454
- else {
455
- throw error;
169
+ // The document’s scrolling element works even when overflow is “visible”
170
+ if (el === document.scrollingElement) {
171
+ return true;
456
172
  }
457
- }
458
- }
459
- /**
460
- * Removes all visual annotations from the page that were created by
461
- * the {@link annotateInteractableElements} method.
462
- *
463
- * This method:
464
- * 1. Processes all accessible frames in the page
465
- * 2. Finds and removes the shadow DOM container with ID 'annotation-shadow-container'
466
- * that contains all the annotations
467
- *
468
- * This effectively removes all numbered indicators that were previously placed
469
- * over interactable elements, leaving the page in its original visual state.
470
- * Note that this only removes the visual annotations, not the
471
- * {@link interactableElementAttribute} attributes on the elements themselves.
472
- *
473
- * @param page - The Playwright Page object to process
474
- * @returns {Promise<void>} A promise that resolves when all annotations have been removed
475
- * @throws {PageClosedException} If the page is closed during processing
476
- *
477
- * @example
478
- * const inspector = new PageInspector();
479
- * await inspector.attributeInteractableElements(page);
480
- * await inspector.annotateInteractableElements(page);
481
- * // ... do some operations with the annotations visible ...
482
- * await inspector.removeDonobuAnnotations(page);
483
- * // All visual annotations are now removed from the page
484
- */
485
- async removeDonobuAnnotations(page) {
486
- try {
487
- const frames = page
488
- .frames()
489
- .filter((frame) => PageInspector.frameFilter(frame));
490
- for (const frame of frames) {
491
- await frame.evaluate(() => {
492
- const containerId = 'annotation-shadow-container';
493
- const container = document.getElementById(containerId);
494
- if (container) {
495
- container.remove();
496
- }
497
- });
173
+ const s = getComputedStyle(el);
174
+ const rect = el.getBoundingClientRect();
175
+ const visible = rect.width > 0 && rect.height > 0;
176
+ if (!visible) {
177
+ return false;
498
178
  }
499
- }
500
- catch (error) {
501
- if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
502
- throw new PageClosedException_1.PageClosedException();
503
- }
504
- throw error;
505
- }
506
- }
507
- /**
508
- * Removes all interactable element attributes that were previously added to elements in the page.
509
- *
510
- * This method:
511
- * 1. Processes all accessible frames in the page
512
- * 2. Finds all elements with the {@link interactableElementAttribute} attribute
513
- * 3. Removes this attribute from each element
514
- *
515
- * This effectively undoes the changes made by the {@link attributeInteractableElements} method,
516
- * returning the page's DOM to its original state without the custom attributes.
517
- * Note that this does not affect any visual annotations - to remove those, use
518
- * the {@link removeDonobuAnnotations} method separately.
519
- *
520
- * This method is automatically called at the beginning of {@link attributeInteractableElements}
521
- * to ensure a clean state before adding new attributes, but can also be called
522
- * independently to clean up the DOM.
523
- *
524
- * @param page - The Playwright Page object to process
525
- * @returns {Promise<void>} A promise that resolves when all attributes have been removed
526
- * @throws {PageClosedException} If the page is closed during processing
527
- *
528
- * @example
529
- * const inspector = new PageInspector();
530
- * await inspector.attributeInteractableElements(page);
531
- * // ... perform operations with attributed elements ...
532
- * await inspector.deattributeInteractableElements(page);
533
- * // All interactable element attributes are now removed from the page
534
- */
535
- async deattributeInteractableElements(page) {
536
- try {
537
- const frames = page.frames().filter(PageInspector.frameFilter);
538
- const attr = this.interactableElementAttribute;
539
- for (const frame of frames) {
540
- await frame.evaluate(([a]) => {
541
- /** Depth-first removal inside document & every shadow root */
542
- const removeDeep = (root) => {
543
- root
544
- .querySelectorAll(`[${a}]`)
545
- .forEach((el) => el.removeAttribute(a));
546
- root.querySelectorAll('*').forEach((el) => {
547
- const sr = el.shadowRoot;
548
- if (sr) {
549
- removeDeep(sr);
550
- }
551
- });
552
- };
553
- removeDeep(document);
554
- }, [attr]);
555
- }
556
- }
557
- catch (error) {
558
- if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
559
- throw new PageClosedException_1.PageClosedException();
560
- }
561
- else {
562
- throw error;
563
- }
564
- }
565
- }
566
- /**
567
- * Retrieves the HTML snippet for a single element.
568
- *
569
- * This method:
570
- * 1. Extracts a simplified HTML snippet representation of the element
571
- * * For 'select' elements, the complete HTML (including options) is preserved
572
- * * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
573
- * * For all other elements, only the opening tag without children is captured
574
- * 2. Strips any Donobu-specific attributes from the snippet
575
- *
576
- * @example
577
- * const inspector = new PageInspector();
578
- * const submitButton = page.querySelector('button[type="submit"]');
579
- * const htmlSnippet = await inspector.getHtmlSnippet(submitButton);
580
- * // htmlSnippet = "<button type=\"submit\">Submit</button>"
581
- */
582
- async getHtmlSnippet(elementHandle) {
583
- try {
584
- // Evaluate in the element's context to get the HTML snippet
585
- const htmlSnippet = await elementHandle.evaluate((element) => {
586
- // Helper function to strip Donobu attributes
587
- function stripDonobuAttrs(el) {
588
- const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
589
- let node = el;
590
- while (node) {
591
- Array.from(node.attributes).forEach((attr) => {
592
- // Strip out Donobu attributes since those are not a part of the
593
- // original HTML.
594
- if (attr.name.startsWith('data-donobu')) {
595
- node.removeAttribute(attr.name);
596
- }
597
- });
598
- node = walker.nextNode();
599
- }
600
- }
601
- // Helper function to serialize element
602
- function serialise(el) {
603
- const deepClone = el.tagName.toLowerCase() === 'select'
604
- ? el.cloneNode(true)
605
- : el.cloneNode(false);
606
- stripDonobuAttrs(deepClone);
607
- if (el.tagName.toLowerCase() === 'select') {
608
- return deepClone.outerHTML; // full markup incl. <option>s
609
- }
610
- // Get the text content of the original element
611
- const textContent = el.textContent?.trim() || '';
612
- if (textContent) {
613
- // Truncate text if longer than 32 characters
614
- const displayText = textContent.length > 32
615
- ? textContent.substring(0, 32) + '...'
616
- : textContent;
617
- // Return opening tag + text + closing tag
618
- const fullTag = deepClone.outerHTML;
619
- const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
620
- const tagName = el.tagName.toLowerCase();
621
- return `${openingTag}${displayText}</${tagName}>`;
622
- }
623
- else {
624
- // opening tag only
625
- const html = deepClone.outerHTML;
626
- return html.slice(0, html.indexOf('>') + 1);
627
- }
628
- }
629
- return serialise(element);
630
- });
631
- return htmlSnippet;
632
- }
633
- catch (error) {
634
- if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
635
- throw new PageClosedException_1.PageClosedException();
636
- }
637
- else {
638
- throw error;
639
- }
640
- }
641
- }
642
- /**
643
- * Converts an HTML attribute to a JavaScript attribute. For example,
644
- * "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
645
- * prefix, and the conversion from kebab-case to camelCase.
646
- */
647
- static convertToJsAttribute(htmlAttribute) {
648
- if (htmlAttribute.startsWith('data-')) {
649
- htmlAttribute = htmlAttribute.substring(5);
650
- }
651
- const parts = htmlAttribute.split('-');
652
- const jsAttribute = parts[0] +
653
- parts
654
- .slice(1)
655
- .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase())
656
- .join('');
657
- return jsAttribute;
658
- }
659
- /**
660
- * An internal method that is injected into page/frame contexts to find and attribute interactable elements.
661
- *
662
- * This method:
663
- * 1. Identifies potentially interactable elements using a comprehensive selector
664
- * 2. Filters elements based on visibility, position in viewport, and interactability
665
- * 3. Assigns unique sequential numeric values to the interactable attribute
666
- *
667
- * The method uses several criteria to determine if an element is truly interactable:
668
- * - Element must be visible (non-zero dimensions, not hidden via CSS)
669
- * - Element must have at least 50% of its area within the viewport
670
- * - Element must not be disabled, inert, or have pointer-events:none
671
- * - Element must be the topmost element at its coordinates (using point sampling)
672
- *
673
- * Special handling is provided for label elements, which will attribute their
674
- * associated form controls as well.
675
- *
676
- * This method can process both standard DOM elements and elements within shadow roots,
677
- * ensuring thorough coverage of modern web applications.
678
- *
679
- * @param arg - A tuple containing [offset: number, interactableAttribute: string]
680
- * where offset is the starting value for sequential numbering and
681
- * interactableAttribute is the attribute name to assign
682
- * @returns The updated offset after assigning attributes (for sequential numbering across frames)
683
- * @private
684
- *
685
- * @remarks
686
- * This method is designed to be injected into the page context using page.evaluate()
687
- * and should not be called directly from Node.js code.
688
- */
689
- static attributeElementsInContext(arg) {
690
- let offset = arg[0];
691
- const interactableAttribute = arg[1];
692
- // --- Utility Functions ---
693
- function isElementVisible(rect, style) {
694
- return (rect.width > 0 &&
695
- rect.height > 0 &&
696
- style.display !== 'none' &&
697
- style.visibility !== 'hidden');
698
- }
699
- function isElementMoreThanHalfInViewport(rect) {
700
- const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
701
- const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
702
- const visibleWidth = Math.min(rect.right, viewportWidth) - Math.max(rect.left, 0);
703
- const visibleHeight = Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0);
704
- const visibleArea = Math.max(0, visibleWidth) * Math.max(0, visibleHeight);
705
- const elementArea = rect.width * rect.height;
706
- return visibleArea >= elementArea / 2;
707
- }
708
- function isElementEnabled(element, style) {
709
- // Check standard disabled attribute (for form controls like button, input, etc.)
710
- if (element.hasAttribute('disabled')) {
711
- return false;
712
- }
713
- // Check for ARIA attributes that indicate non-interactivity
714
- if (element.getAttribute('aria-hidden') === 'true') {
715
- return false;
716
- }
717
- // Check for pointer-events: none which prevents interactions
718
- if (style.pointerEvents === 'none') {
719
- return false;
720
- }
721
- // Check for inert attribute which makes elements non-interactive
722
- if (element.hasAttribute('inert')) {
723
- return false;
724
- }
725
- // If the element is in a form and the fieldset is disabled, it might be disabled as well
726
- let parent = element.parentElement;
727
- while (parent) {
728
- if (parent.tagName.toLowerCase() === 'fieldset' &&
729
- parent.hasAttribute('disabled') &&
730
- element.tagName.toLowerCase() !== 'legend') {
731
- return false;
732
- }
733
- parent = parent.parentElement;
734
- }
735
- return true;
736
- }
737
- function isScrollable(el) {
738
- const canY = el.scrollHeight > el.clientHeight;
739
- const canX = el.scrollWidth > el.clientWidth;
740
- // If nothing overflows, bail early
741
- if (!canY && !canX) {
742
- return false;
743
- }
744
- // The document’s scrolling element works even when overflow is “visible”
745
- if (el === document.scrollingElement) {
746
- return true;
747
- }
748
- const s = getComputedStyle(el);
749
- const rect = el.getBoundingClientRect();
750
- const visible = rect.width > 0 && rect.height > 0;
751
- if (!visible) {
752
- return false;
753
- }
754
- const yOK = canY && /(auto|scroll|overlay)/.test(s.overflowY);
755
- const xOK = canX && /(auto|scroll|overlay)/.test(s.overflowX);
756
- return yOK || xOK;
179
+ const yOK = canY && /(auto|scroll|overlay)/.test(s.overflowY);
180
+ const xOK = canX && /(auto|scroll|overlay)/.test(s.overflowX);
181
+ return yOK || xOK;
757
182
  }
758
183
  /**
759
184
  * Generate a few test points on the element's bounding box. We only need
@@ -928,91 +353,666 @@ class PageInspector {
928
353
  if (isScrollable(el) && !uniqueElements.has(el)) {
929
354
  uniqueElements.add(el);
930
355
  }
931
- };
932
- document.querySelectorAll('*').forEach(maybeAddScrollable);
933
- if (document.scrollingElement) {
934
- maybeAddScrollable(document.scrollingElement);
356
+ };
357
+ document.querySelectorAll('*').forEach(maybeAddScrollable);
358
+ if (document.scrollingElement) {
359
+ maybeAddScrollable(document.scrollingElement);
360
+ }
361
+ /**
362
+ * Run the visibility / enabled / top-most checks on a single element and,
363
+ * if they pass, assign it the next interactable number. Returns `true` if
364
+ * the element (or, via the <label htmlFor> mapping, its associated control)
365
+ * was attributed.
366
+ */
367
+ function tryAttributeElement(element) {
368
+ if (element.hasAttribute(interactableAttribute)) {
369
+ return false;
370
+ }
371
+ const rect = element.getBoundingClientRect();
372
+ const style = window.getComputedStyle(element);
373
+ const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
374
+ const enabled = isElementEnabled(element, style);
375
+ if (!visible || !enabled) {
376
+ return false;
377
+ }
378
+ // Check a few probe points to make sure the element is top-most
379
+ for (const pt of getPointsToCheck(rect)) {
380
+ let elToCheck = getDeepElementFromPoint(pt.x, pt.y);
381
+ while (elToCheck) {
382
+ if (elToCheck === element) {
383
+ element.setAttribute(interactableAttribute, offset.toString());
384
+ offset++;
385
+ return true; // this element done
386
+ }
387
+ // Handle <label> -> control mapping (explicit `for`/`htmlFor`)
388
+ if (elToCheck.tagName.toLowerCase() === 'label' &&
389
+ elToCheck.htmlFor) {
390
+ const forId = elToCheck.htmlFor;
391
+ const control = document.getElementById(forId);
392
+ if (control &&
393
+ !control.hasAttribute(interactableAttribute) // prevent double number
394
+ ) {
395
+ control.setAttribute(interactableAttribute, offset.toString());
396
+ offset++;
397
+ }
398
+ return true;
399
+ }
400
+ elToCheck = elToCheck.parentElement;
401
+ }
402
+ }
403
+ return false;
404
+ }
405
+ // 2) Iterate and assign numbers
406
+ uniqueElements.forEach((element) => {
407
+ if (element === document.scrollingElement) {
408
+ // Special-case: always keep the root scrolling element
409
+ element.setAttribute(interactableAttribute, offset.toString());
410
+ offset++;
411
+ return; // skip the usual checks
412
+ }
413
+ else if (element.hasAttribute(interactableAttribute)) {
414
+ // Skip if this element already carries a value (e.g. assigned via <label>)
415
+ return;
416
+ }
417
+ if (tryAttributeElement(element)) {
418
+ return;
419
+ }
420
+ // Fallback: the element is a visually-hidden native control (e.g. a 0x0,
421
+ // opacity-0, pointer-events:none <input>) wrapped in a styled <label>.
422
+ // This is the standard pattern for Ant Design Segmented/Radio/Checkbox/
423
+ // Switch and many other component libraries: the native input is hidden
424
+ // and the surrounding <label> is the real clickable surface. Since the
425
+ // hidden control fails the visibility/enabled checks above, attribute the
426
+ // wrapping <label> instead so the toggle is still annotated.
427
+ const wrappingLabel = element.closest('label');
428
+ if (wrappingLabel &&
429
+ wrappingLabel !== element &&
430
+ !wrappingLabel.hasAttribute(interactableAttribute)) {
431
+ tryAttributeElement(wrappingLabel);
432
+ }
433
+ });
434
+ return offset;
435
+ }
436
+ static frameFilter(frame) {
437
+ return (!frame.isDetached() &&
438
+ !frame.url().startsWith('about:') &&
439
+ !frame.url().startsWith('chrome:') &&
440
+ !frame.url().startsWith('edge:'));
441
+ }
442
+ /**
443
+ * Assigns a globally unique attribute to all visible and interactable elements in the page.
444
+ *
445
+ * This method performs the following steps:
446
+ * 1. Removes any pre-existing interactable element attributes from the page
447
+ * 2. Assigns sequential numeric values as attributes to interactable elements in the main frame
448
+ * 3. Processes child frames that are visible in the viewport and assigns attributes to their interactable elements
449
+ *
450
+ * The method identifies "interactable" elements based on tag names, ARIA roles, CSS classes, and other heuristics.
451
+ * Only elements that are:
452
+ * - Visible (non-zero dimensions and not hidden via CSS)
453
+ * - More than 50% in the viewport
454
+ * - Not disabled or inert
455
+ * - Actually reachable at their coordinates (topmost in z-index)
456
+ * will receive the attribute.
457
+ *
458
+ * @param page - The Playwright Page object to process
459
+ * @throws {PageClosedException} If the page is closed during processing
460
+ * @returns {Promise<void>} A promise that resolves when all elements have been attributed
461
+ */
462
+ async attributeInteractableElements(page) {
463
+ try {
464
+ // Remove any preexisting attributes
465
+ await this.deattributeInteractableElements(page);
466
+ // Get viewport dimensions and scroll position properly
467
+ const viewportInfo = await page.evaluate(() => {
468
+ return {
469
+ viewportWidth: window.innerWidth,
470
+ viewportHeight: window.innerHeight,
471
+ scrollX: window.scrollX || window.pageXOffset,
472
+ scrollY: window.scrollY || window.pageYOffset,
473
+ };
474
+ });
475
+ // 1) Attribute elements in the main page
476
+ let annotationOffset = await page.evaluate(PageInspector.attributeElementsInContext, [0, this.interactableElementAttribute]);
477
+ // 2) Check child frames, attributing elements if the frame is (partially) in view
478
+ const frames = page
479
+ .frames()
480
+ .filter((frame) => PageInspector.frameFilter(frame) && frame !== page.mainFrame());
481
+ for (const frame of frames) {
482
+ const elementHandle = await frame.frameElement();
483
+ if (!elementHandle) {
484
+ continue;
485
+ }
486
+ const boundingBox = await elementHandle.boundingBox();
487
+ if (!boundingBox) {
488
+ continue;
489
+ }
490
+ // boundingBox coordinates are already in viewport space, so we need to account for scroll
491
+ const isInViewport = boundingBox.x + boundingBox.width > 0 &&
492
+ boundingBox.x < viewportInfo.viewportWidth &&
493
+ boundingBox.y + boundingBox.height > 0 &&
494
+ boundingBox.y < viewportInfo.viewportHeight;
495
+ if (isInViewport) {
496
+ annotationOffset = await frame.evaluate(PageInspector.attributeElementsInContext, [annotationOffset, this.interactableElementAttribute]);
497
+ }
498
+ }
499
+ }
500
+ catch (error) {
501
+ if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
502
+ throw new PageClosedException_1.PageClosedException();
503
+ }
504
+ else {
505
+ throw error;
506
+ }
507
+ }
508
+ }
509
+ /**
510
+ * Retrieves all elements that have been previously attributed with the interactable element attribute.
511
+ *
512
+ * This method:
513
+ * 1. Searches all frames in the page (including the main frame and child frames)
514
+ * 2. Collects elements with the {@link interactableElementAttribute} attribute
515
+ * 3. Creates an {@link InteractableElement} object for each attributed element
516
+ *
517
+ * For each interactable element, it extracts:
518
+ * - The attribute value (serving as a unique identifier)
519
+ * - A simplified HTML snippet representation of the element
520
+ * * For 'select' elements, the complete HTML (including options) is preserved
521
+ * * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
522
+ * * For all other elements, only the opening tag without children is captured
523
+ * * For the main scrolling element (document.scrollingElement), adds special decoration indicating it's the page's main scrolling element
524
+ *
525
+ * Note: This method only finds elements that have been previously attributed using
526
+ * the {@link attributeInteractableElements} method.
527
+ *
528
+ * @param page - The Playwright Page object to process
529
+ * @returns {Promise<InteractableElement[]>} A promise that resolves to an array of
530
+ * interactable elements with their attribute values and HTML snippets
531
+ * @throws {PageClosedException} If the page is closed during processing
532
+ *
533
+ * @example
534
+ * const inspector = new PageInspector();
535
+ * await inspector.attributeInteractableElements(page);
536
+ * const elements = await inspector.getAttributedInteractableElements(page);
537
+ * // elements = [{ donobuAttributeValue: "0", htmlSnippet: "<button id=\"submit\">Submit</button>"}]
538
+ */
539
+ async getAttributedInteractableElements(page) {
540
+ try {
541
+ const frames = page.frames().filter(PageInspector.frameFilter);
542
+ const aggregate = {};
543
+ for (const frame of frames) {
544
+ const frameMap = await frame.evaluate((interactableAttr) => {
545
+ /* --- helpers running in the browser context --- */
546
+ function stripDonobuAttrs(el) {
547
+ const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
548
+ let node = el;
549
+ while (node) {
550
+ Array.from(node.attributes).forEach((attr) => {
551
+ // Strip out Donobu attributes since those are not a part of the
552
+ // original HTML.
553
+ if (attr.name.startsWith('data-donobu')) {
554
+ node.removeAttribute(attr.name);
555
+ }
556
+ });
557
+ node = walker.nextNode();
558
+ }
559
+ }
560
+ /** helper to compute live scroll directions for el */
561
+ function getScrollDirections(el) {
562
+ // Special case for when the document body is not the scrollingElement
563
+ // element. This may happen if the scrollingElement is the
564
+ // root <html> element. In this case, it makes no sense to report
565
+ // scrollability on <body> and on scrollingElement, since we should
566
+ // use the scrollingElement instead.
567
+ if (el === document.body &&
568
+ document.scrollingElement !== document.body) {
569
+ return [];
570
+ }
571
+ const dirs = [];
572
+ const isRoot = el === document.scrollingElement;
573
+ const style = getComputedStyle(el);
574
+ // Add a small margin so we do not waste time reporting scrollability
575
+ // for an element that is not materially scrollable.
576
+ const marginPx = 1;
577
+ const canY = el.scrollHeight > el.clientHeight + marginPx &&
578
+ (isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowY));
579
+ const canX = el.scrollWidth > el.clientWidth + marginPx &&
580
+ (isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowX));
581
+ if (canY) {
582
+ if (el.scrollTop > 0) {
583
+ dirs.push('UP');
584
+ }
585
+ if (el.scrollTop < el.scrollHeight - el.clientHeight) {
586
+ dirs.push('DOWN');
587
+ }
588
+ }
589
+ if (canX) {
590
+ if (el.scrollLeft > 0) {
591
+ dirs.push('LEFT');
592
+ }
593
+ if (el.scrollLeft < el.scrollWidth - el.clientWidth) {
594
+ dirs.push('RIGHT');
595
+ }
596
+ }
597
+ return dirs;
598
+ }
599
+ function serialise(el) {
600
+ const deepClone = (el.tagName.toLowerCase() === 'select'
601
+ ? el.cloneNode(true)
602
+ : el.cloneNode(false));
603
+ stripDonobuAttrs(deepClone);
604
+ if (el.tagName.toLowerCase() === 'select') {
605
+ const scrollComment = el === document.scrollingElement
606
+ ? '<!-- This is the main page scrolling element -->'
607
+ : '';
608
+ return scrollComment + deepClone.outerHTML; // full markup incl. <option>s
609
+ }
610
+ // Get the text content of the original element
611
+ const textContent = el.textContent?.trim() || '';
612
+ if (textContent) {
613
+ // Truncate text if longer than 32 characters
614
+ const displayText = textContent.length > 32
615
+ ? textContent.substring(0, 32) + '...'
616
+ : textContent;
617
+ // Return opening tag + text + closing tag
618
+ const fullTag = deepClone.outerHTML;
619
+ const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
620
+ const tagName = el.tagName.toLowerCase();
621
+ const scrollComment = el === document.scrollingElement
622
+ ? '<!-- This is the main page scrolling element -->'
623
+ : '';
624
+ return `${scrollComment}${openingTag}${displayText}</${tagName}>`;
625
+ }
626
+ else {
627
+ // opening tag only
628
+ const html = deepClone.outerHTML;
629
+ const scrollComment = el === document.scrollingElement
630
+ ? '<!-- This is the main page scrolling element -->'
631
+ : '';
632
+ return scrollComment + html.slice(0, html.indexOf('>') + 1);
633
+ }
634
+ }
635
+ const out = {};
636
+ // Recursively process document and all shadow roots
637
+ const processNode = (root) => {
638
+ // Find elements with the interactable attribute
639
+ root.querySelectorAll(`[${interactableAttr}]`).forEach((el) => {
640
+ const val = el.getAttribute(interactableAttr);
641
+ if (!val) {
642
+ return;
643
+ }
644
+ out[val] = {
645
+ htmlSnippet: serialise(el),
646
+ scrollable: getScrollDirections(el),
647
+ };
648
+ });
649
+ // Recursively process any child shadow roots
650
+ root.querySelectorAll('*').forEach((el) => {
651
+ if (el.shadowRoot) {
652
+ processNode(el.shadowRoot);
653
+ }
654
+ });
655
+ };
656
+ // Start processing from the document root
657
+ processNode(document);
658
+ return out;
659
+ }, this.interactableElementAttribute);
660
+ Object.assign(aggregate, frameMap);
661
+ }
662
+ return Object.keys(aggregate)
663
+ .sort((a, b) => Number(a) - Number(b))
664
+ .map((key) => ({
665
+ donobuAttributeValue: key,
666
+ htmlSnippet: aggregate[key].htmlSnippet,
667
+ scrollable: aggregate[key].scrollable,
668
+ }));
669
+ }
670
+ catch (error) {
671
+ if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
672
+ throw new PageClosedException_1.PageClosedException();
673
+ }
674
+ else {
675
+ throw error;
676
+ }
677
+ }
678
+ }
679
+ /**
680
+ * Visually annotates all interactable elements with numbered indicators on the page.
681
+ *
682
+ * This method:
683
+ * 1. Processes all accessible frames in the page
684
+ * 2. Creates (or reuses) a shadow DOM container to isolate annotation styling
685
+ * 3. Places circular numbered indicators over each element that has the
686
+ * {@link interactableElementAttribute} attribute
687
+ *
688
+ * The annotations:
689
+ * - Are positioned at the center of each interactable element
690
+ * - Have the same numeric value as the element's attribute
691
+ * - Are styled as black circles with red borders and white text
692
+ * - Are placed in a shadow DOM to avoid style conflicts with the page
693
+ * - Have the {@link interactableAnnotationAttribute} for identification
694
+ * - Are non-interactive (pointer-events: none)
695
+ *
696
+ * Note: This method requires elements to be previously attributed using the
697
+ * {@link attributeInteractableElements} method to find the elements to annotate.
698
+ *
699
+ * @param page - The Playwright Page object to process
700
+ * @returns {Promise<void>} A promise that resolves when all elements have been annotated
701
+ * @throws {PageClosedException} If the page is closed during processing
702
+ *
703
+ * @example
704
+ * const inspector = new PageInspector();
705
+ * await inspector.attributeInteractableElements(page);
706
+ * await inspector.annotateInteractableElements(page);
707
+ */
708
+ async annotateInteractableElements(page) {
709
+ try {
710
+ // Filter frames as needed
711
+ const frames = page
712
+ .frames()
713
+ .filter((frame) => PageInspector.frameFilter(frame));
714
+ for (const frame of frames) {
715
+ await frame.evaluate(([interactableAttr, annotationAttr]) => {
716
+ // 1) Ensure we have a shadow container in the main document
717
+ let container = document.getElementById('annotation-shadow-container');
718
+ if (!container) {
719
+ container = document.createElement('div');
720
+ container.id = 'annotation-shadow-container';
721
+ // Position container so child elements can be absolutely placed
722
+ Object.assign(container.style, {
723
+ position: 'absolute',
724
+ top: '0',
725
+ left: '0',
726
+ width: '100%',
727
+ height: '100%',
728
+ pointerEvents: 'none', // Let clicks pass through
729
+ zIndex: '2147483647', // win every z-index fight
730
+ });
731
+ // Check if document.body exists before trying to append.
732
+ if (document.body) {
733
+ document.body.appendChild(container);
734
+ }
735
+ else if (document.documentElement) {
736
+ // Fall back to document.documentElement if body does not exist.
737
+ document.documentElement.appendChild(container);
738
+ }
739
+ else {
740
+ // If neither exists, we can't proceed with annotations in this frame.
741
+ console.warn(`Cannot create annotation container for ${window.location.href} since the document structure is incomplete`);
742
+ return;
743
+ }
744
+ // Attach a shadow root
745
+ const shadowRoot = container.attachShadow({ mode: 'open' });
746
+ // Add a <style> element inside the shadow root to reset and define annotation styles
747
+ const style = document.createElement('style');
748
+ style.textContent = `
749
+ :host {
750
+ all: initial; /* Reset styles in shadow root */
751
+ }
752
+ .annotation {
753
+ position: absolute;
754
+ z-index: 2147483647;
755
+ background-color: black;
756
+ color: white;
757
+ width: 40px;
758
+ height: 40px;
759
+ border-radius: 50%;
760
+ display: flex;
761
+ align-items: center;
762
+ justify-content: center;
763
+ font-size: 14px;
764
+ font-weight: bold;
765
+ line-height: 20px;
766
+ text-align: center;
767
+ box-shadow: 0px 2px 4px rgba(0,0,0,0.2);
768
+ border: 4px solid #FF4136;
769
+ pointer-events: none;
770
+ }
771
+ `;
772
+ shadowRoot.appendChild(style);
773
+ }
774
+ // Retrieve the shadow root to place annotation elements
775
+ const containerEl = document.getElementById('annotation-shadow-container');
776
+ if (!containerEl?.shadowRoot) {
777
+ return;
778
+ }
779
+ const shadowRoot = containerEl.shadowRoot;
780
+ // 2) Factory to create a new annotation inside the shadow root
781
+ const createAnnotation = (value) => {
782
+ const annotation = document.createElement('div');
783
+ annotation.classList.add('annotation');
784
+ annotation.dataset[annotationAttr] = '1';
785
+ annotation.textContent = value;
786
+ return annotation;
787
+ };
788
+ // 3) Position annotation relative to an element
789
+ const positionAnnotation = (annotation, element) => {
790
+ const rect = element.getBoundingClientRect();
791
+ // Center the annotation on the element, adjusting for its size
792
+ // Since container is absolute, we need to account for scroll position
793
+ const x = rect.left + rect.width / 2 - 20 + window.scrollX;
794
+ const y = rect.top + rect.height / 2 - 20 + window.scrollY;
795
+ annotation.style.left = `${x}px`;
796
+ annotation.style.top = `${y}px`;
797
+ };
798
+ // 4) Traverse DOM (including any nested shadow roots) to find interactable elements
799
+ const processNode = (root) => {
800
+ // Find elements with the interactable attribute
801
+ const elements = root.querySelectorAll(`[${interactableAttr}]`);
802
+ elements.forEach((element) => {
803
+ const value = element.getAttribute(interactableAttr);
804
+ if (value) {
805
+ const annotation = createAnnotation(value);
806
+ shadowRoot.appendChild(annotation);
807
+ positionAnnotation(annotation, element);
808
+ }
809
+ });
810
+ // Recursively process any child shadow roots
811
+ root.querySelectorAll('*').forEach((el) => {
812
+ if (el.shadowRoot) {
813
+ processNode(el.shadowRoot);
814
+ }
815
+ });
816
+ };
817
+ // Start processing from the (frame) document root
818
+ processNode(document);
819
+ }, [
820
+ this.interactableElementAttribute,
821
+ PageInspector.convertToJsAttribute(this.interactableAnnotationAttribute),
822
+ ]);
823
+ }
935
824
  }
936
- /**
937
- * Run the visibility / enabled / top-most checks on a single element and,
938
- * if they pass, assign it the next interactable number. Returns `true` if
939
- * the element (or, via the <label htmlFor> mapping, its associated control)
940
- * was attributed.
941
- */
942
- function tryAttributeElement(element) {
943
- if (element.hasAttribute(interactableAttribute)) {
944
- return false;
825
+ catch (error) {
826
+ if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
827
+ throw new PageClosedException_1.PageClosedException();
945
828
  }
946
- const rect = element.getBoundingClientRect();
947
- const style = window.getComputedStyle(element);
948
- const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
949
- const enabled = isElementEnabled(element, style);
950
- if (!visible || !enabled) {
951
- return false;
829
+ else {
830
+ throw error;
952
831
  }
953
- // Check a few probe points to make sure the element is top-most
954
- for (const pt of getPointsToCheck(rect)) {
955
- let elToCheck = getDeepElementFromPoint(pt.x, pt.y);
956
- while (elToCheck) {
957
- if (elToCheck === element) {
958
- element.setAttribute(interactableAttribute, offset.toString());
959
- offset++;
960
- return true; // this element done
961
- }
962
- // Handle <label> -> control mapping (explicit `for`/`htmlFor`)
963
- if (elToCheck.tagName.toLowerCase() === 'label' &&
964
- elToCheck.htmlFor) {
965
- const forId = elToCheck.htmlFor;
966
- const control = document.getElementById(forId);
967
- if (control &&
968
- !control.hasAttribute(interactableAttribute) // prevent double number
969
- ) {
970
- control.setAttribute(interactableAttribute, offset.toString());
971
- offset++;
972
- }
973
- return true;
832
+ }
833
+ }
834
+ /**
835
+ * Removes all visual annotations from the page that were created by
836
+ * the {@link annotateInteractableElements} method.
837
+ *
838
+ * This method:
839
+ * 1. Processes all accessible frames in the page
840
+ * 2. Finds and removes the shadow DOM container with ID 'annotation-shadow-container'
841
+ * that contains all the annotations
842
+ *
843
+ * This effectively removes all numbered indicators that were previously placed
844
+ * over interactable elements, leaving the page in its original visual state.
845
+ * Note that this only removes the visual annotations, not the
846
+ * {@link interactableElementAttribute} attributes on the elements themselves.
847
+ *
848
+ * @param page - The Playwright Page object to process
849
+ * @returns {Promise<void>} A promise that resolves when all annotations have been removed
850
+ * @throws {PageClosedException} If the page is closed during processing
851
+ *
852
+ * @example
853
+ * const inspector = new PageInspector();
854
+ * await inspector.attributeInteractableElements(page);
855
+ * await inspector.annotateInteractableElements(page);
856
+ * // ... do some operations with the annotations visible ...
857
+ * await inspector.removeDonobuAnnotations(page);
858
+ * // All visual annotations are now removed from the page
859
+ */
860
+ async removeDonobuAnnotations(page) {
861
+ try {
862
+ const frames = page
863
+ .frames()
864
+ .filter((frame) => PageInspector.frameFilter(frame));
865
+ for (const frame of frames) {
866
+ await frame.evaluate(() => {
867
+ const containerId = 'annotation-shadow-container';
868
+ const container = document.getElementById(containerId);
869
+ if (container) {
870
+ container.remove();
974
871
  }
975
- elToCheck = elToCheck.parentElement;
976
- }
872
+ });
977
873
  }
978
- return false;
979
874
  }
980
- // 2) Iterate and assign numbers
981
- uniqueElements.forEach((element) => {
982
- if (element === document.scrollingElement) {
983
- // Special-case: always keep the root scrolling element
984
- element.setAttribute(interactableAttribute, offset.toString());
985
- offset++;
986
- return; // skip the usual checks
875
+ catch (error) {
876
+ if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
877
+ throw new PageClosedException_1.PageClosedException();
987
878
  }
988
- else if (element.hasAttribute(interactableAttribute)) {
989
- // Skip if this element already carries a value (e.g. assigned via <label>)
990
- return;
879
+ throw error;
880
+ }
881
+ }
882
+ /**
883
+ * Removes all interactable element attributes that were previously added to elements in the page.
884
+ *
885
+ * This method:
886
+ * 1. Processes all accessible frames in the page
887
+ * 2. Finds all elements with the {@link interactableElementAttribute} attribute
888
+ * 3. Removes this attribute from each element
889
+ *
890
+ * This effectively undoes the changes made by the {@link attributeInteractableElements} method,
891
+ * returning the page's DOM to its original state without the custom attributes.
892
+ * Note that this does not affect any visual annotations - to remove those, use
893
+ * the {@link removeDonobuAnnotations} method separately.
894
+ *
895
+ * This method is automatically called at the beginning of {@link attributeInteractableElements}
896
+ * to ensure a clean state before adding new attributes, but can also be called
897
+ * independently to clean up the DOM.
898
+ *
899
+ * @param page - The Playwright Page object to process
900
+ * @returns {Promise<void>} A promise that resolves when all attributes have been removed
901
+ * @throws {PageClosedException} If the page is closed during processing
902
+ *
903
+ * @example
904
+ * const inspector = new PageInspector();
905
+ * await inspector.attributeInteractableElements(page);
906
+ * // ... perform operations with attributed elements ...
907
+ * await inspector.deattributeInteractableElements(page);
908
+ * // All interactable element attributes are now removed from the page
909
+ */
910
+ async deattributeInteractableElements(page) {
911
+ try {
912
+ const frames = page.frames().filter(PageInspector.frameFilter);
913
+ const attr = this.interactableElementAttribute;
914
+ for (const frame of frames) {
915
+ await frame.evaluate(([a]) => {
916
+ /** Depth-first removal inside document & every shadow root */
917
+ const removeDeep = (root) => {
918
+ root
919
+ .querySelectorAll(`[${a}]`)
920
+ .forEach((el) => el.removeAttribute(a));
921
+ root.querySelectorAll('*').forEach((el) => {
922
+ const sr = el.shadowRoot;
923
+ if (sr) {
924
+ removeDeep(sr);
925
+ }
926
+ });
927
+ };
928
+ removeDeep(document);
929
+ }, [attr]);
991
930
  }
992
- if (tryAttributeElement(element)) {
993
- return;
931
+ }
932
+ catch (error) {
933
+ if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
934
+ throw new PageClosedException_1.PageClosedException();
994
935
  }
995
- // Fallback: the element is a visually-hidden native control (e.g. a 0x0,
996
- // opacity-0, pointer-events:none <input>) wrapped in a styled <label>.
997
- // This is the standard pattern for Ant Design Segmented/Radio/Checkbox/
998
- // Switch and many other component libraries: the native input is hidden
999
- // and the surrounding <label> is the real clickable surface. Since the
1000
- // hidden control fails the visibility/enabled checks above, attribute the
1001
- // wrapping <label> instead so the toggle is still annotated.
1002
- const wrappingLabel = element.closest('label');
1003
- if (wrappingLabel &&
1004
- wrappingLabel !== element &&
1005
- !wrappingLabel.hasAttribute(interactableAttribute)) {
1006
- tryAttributeElement(wrappingLabel);
936
+ else {
937
+ throw error;
1007
938
  }
1008
- });
1009
- return offset;
939
+ }
1010
940
  }
1011
- static frameFilter(frame) {
1012
- return (!frame.isDetached() &&
1013
- !frame.url().startsWith('about:') &&
1014
- !frame.url().startsWith('chrome:') &&
1015
- !frame.url().startsWith('edge:'));
941
+ /**
942
+ * Retrieves the HTML snippet for a single element.
943
+ *
944
+ * This method:
945
+ * 1. Extracts a simplified HTML snippet representation of the element
946
+ * * For 'select' elements, the complete HTML (including options) is preserved
947
+ * * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
948
+ * * For all other elements, only the opening tag without children is captured
949
+ * 2. Strips any Donobu-specific attributes from the snippet
950
+ *
951
+ * @example
952
+ * const inspector = new PageInspector();
953
+ * const submitButton = page.querySelector('button[type="submit"]');
954
+ * const htmlSnippet = await inspector.getHtmlSnippet(submitButton);
955
+ * // htmlSnippet = "<button type=\"submit\">Submit</button>"
956
+ */
957
+ async getHtmlSnippet(elementHandle) {
958
+ try {
959
+ // Evaluate in the element's context to get the HTML snippet
960
+ const htmlSnippet = await elementHandle.evaluate((element) => {
961
+ // Helper function to strip Donobu attributes
962
+ function stripDonobuAttrs(el) {
963
+ const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
964
+ let node = el;
965
+ while (node) {
966
+ Array.from(node.attributes).forEach((attr) => {
967
+ // Strip out Donobu attributes since those are not a part of the
968
+ // original HTML.
969
+ if (attr.name.startsWith('data-donobu')) {
970
+ node.removeAttribute(attr.name);
971
+ }
972
+ });
973
+ node = walker.nextNode();
974
+ }
975
+ }
976
+ // Helper function to serialize element
977
+ function serialise(el) {
978
+ const deepClone = el.tagName.toLowerCase() === 'select'
979
+ ? el.cloneNode(true)
980
+ : el.cloneNode(false);
981
+ stripDonobuAttrs(deepClone);
982
+ if (el.tagName.toLowerCase() === 'select') {
983
+ return deepClone.outerHTML; // full markup incl. <option>s
984
+ }
985
+ // Get the text content of the original element
986
+ const textContent = el.textContent?.trim() || '';
987
+ if (textContent) {
988
+ // Truncate text if longer than 32 characters
989
+ const displayText = textContent.length > 32
990
+ ? textContent.substring(0, 32) + '...'
991
+ : textContent;
992
+ // Return opening tag + text + closing tag
993
+ const fullTag = deepClone.outerHTML;
994
+ const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
995
+ const tagName = el.tagName.toLowerCase();
996
+ return `${openingTag}${displayText}</${tagName}>`;
997
+ }
998
+ else {
999
+ // opening tag only
1000
+ const html = deepClone.outerHTML;
1001
+ return html.slice(0, html.indexOf('>') + 1);
1002
+ }
1003
+ }
1004
+ return serialise(element);
1005
+ });
1006
+ return htmlSnippet;
1007
+ }
1008
+ catch (error) {
1009
+ if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
1010
+ throw new PageClosedException_1.PageClosedException();
1011
+ }
1012
+ else {
1013
+ throw error;
1014
+ }
1015
+ }
1016
1016
  }
1017
1017
  }
1018
1018
  exports.PageInspector = PageInspector;