donobu 5.55.0 → 5.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/GptConfigsApi.d.ts +5 -5
- package/dist/apis/GptConfigsApi.js +14 -14
- package/dist/bindings/PageInteractionTracker.d.ts +1 -1
- package/dist/bindings/PageInteractionTracker.js +3 -3
- package/dist/bindings/SetDonobuAnnotations.d.ts +1 -1
- package/dist/bindings/SetDonobuAnnotations.js +3 -3
- package/dist/clients/AnthropicGptClient.d.ts +2 -2
- package/dist/clients/AnthropicGptClient.js +77 -77
- package/dist/clients/OpenAiGptClient.d.ts +14 -14
- package/dist/clients/OpenAiGptClient.js +183 -183
- package/dist/esm/apis/GptConfigsApi.d.ts +5 -5
- package/dist/esm/apis/GptConfigsApi.js +14 -14
- package/dist/esm/bindings/PageInteractionTracker.d.ts +1 -1
- package/dist/esm/bindings/PageInteractionTracker.js +3 -3
- package/dist/esm/bindings/SetDonobuAnnotations.d.ts +1 -1
- package/dist/esm/bindings/SetDonobuAnnotations.js +3 -3
- package/dist/esm/clients/AnthropicGptClient.d.ts +2 -2
- package/dist/esm/clients/AnthropicGptClient.js +77 -77
- package/dist/esm/clients/OpenAiGptClient.d.ts +14 -14
- package/dist/esm/clients/OpenAiGptClient.js +183 -183
- package/dist/esm/lib/ai/PageAi.js +2 -1
- package/dist/esm/lib/page/extendPage.js +2 -1
- package/dist/esm/lib/test/utils/TestFileUpdater.d.ts +9 -9
- package/dist/esm/lib/test/utils/TestFileUpdater.js +49 -49
- package/dist/esm/main.d.ts +2 -0
- package/dist/esm/managers/AdminApiController.d.ts +16 -16
- package/dist/esm/managers/AdminApiController.js +35 -35
- package/dist/esm/managers/DonobuFlow.d.ts +57 -36
- package/dist/esm/managers/DonobuFlow.js +489 -564
- package/dist/esm/managers/DonobuFlowsManager.js +13 -17
- package/dist/esm/managers/FlowDependencyAnalyzer.d.ts +12 -12
- package/dist/esm/managers/FlowDependencyAnalyzer.js +77 -77
- package/dist/esm/managers/PageInspector.d.ts +38 -38
- package/dist/esm/managers/PageInspector.js +745 -745
- package/dist/esm/managers/TargetInspector.d.ts +28 -33
- package/dist/esm/managers/TestsManager.d.ts +25 -25
- package/dist/esm/managers/TestsManager.js +74 -74
- package/dist/esm/managers/ToolManager.js +7 -5
- package/dist/esm/managers/ToolRegistry.d.ts +5 -1
- package/dist/esm/managers/WebTargetInspector.d.ts +9 -5
- package/dist/esm/managers/WebTargetInspector.js +45 -47
- package/dist/esm/models/AiQuery.d.ts +29 -15
- package/dist/esm/models/AiQuery.js +31 -0
- package/dist/esm/models/ControlPanel.d.ts +18 -13
- package/dist/esm/models/InteractableElement.d.ts +6 -0
- package/dist/esm/models/InteractableElement.js +7 -1
- package/dist/esm/models/Observation.d.ts +38 -0
- package/dist/esm/models/Observation.js +3 -0
- package/dist/esm/models/ToolCallContext.d.ts +3 -2
- package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.d.ts +2 -2
- package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.js +19 -18
- package/dist/esm/persistence/flows/FlowsPersistenceSqlite.js +2 -1
- package/dist/esm/targets/TargetProvider.d.ts +110 -0
- package/dist/esm/targets/TargetProvider.js +25 -0
- package/dist/esm/targets/TargetRuntime.d.ts +6 -3
- package/dist/esm/targets/WebDialogHandler.d.ts +14 -0
- package/dist/esm/targets/WebDialogHandler.js +198 -0
- package/dist/esm/targets/WebTargetProvider.d.ts +32 -0
- package/dist/esm/targets/WebTargetProvider.js +136 -0
- package/dist/esm/targets/WebTargetRuntime.d.ts +2 -2
- package/dist/esm/targets/WebTargetRuntime.js +2 -1
- package/dist/esm/tools/AcknowledgeUserInstruction.d.ts +6 -0
- package/dist/esm/tools/AcknowledgeUserInstruction.js +7 -0
- package/dist/esm/tools/AssertPageTool.d.ts +1 -1
- package/dist/esm/tools/AssertPageTool.js +3 -3
- package/dist/esm/tools/DetectBrokenLinksTool.d.ts +2 -2
- package/dist/esm/tools/DetectBrokenLinksTool.js +44 -44
- package/dist/esm/tools/InputFakerTool.d.ts +4 -4
- package/dist/esm/tools/InputFakerTool.js +10 -10
- package/dist/esm/tools/InputTextTool.d.ts +4 -4
- package/dist/esm/tools/InputTextTool.js +7 -7
- package/dist/esm/tools/ReplayableInteraction.d.ts +34 -34
- package/dist/esm/tools/ReplayableInteraction.js +245 -245
- package/dist/esm/tools/Tool.d.ts +6 -3
- package/dist/esm/tools/Tool.js +5 -2
- package/dist/esm/utils/BrowserUtils.d.ts +19 -19
- package/dist/esm/utils/BrowserUtils.js +57 -57
- package/dist/esm/utils/MiscUtils.d.ts +2 -2
- package/dist/esm/utils/MiscUtils.js +16 -16
- package/dist/esm/utils/PlaywrightUtils.d.ts +1 -1
- package/dist/esm/utils/TargetUtils.d.ts +1 -1
- package/dist/esm/utils/TargetUtils.js +15 -13
- package/dist/lib/ai/PageAi.js +2 -1
- package/dist/lib/page/extendPage.js +2 -1
- package/dist/lib/test/utils/TestFileUpdater.d.ts +9 -9
- package/dist/lib/test/utils/TestFileUpdater.js +49 -49
- package/dist/main.d.ts +2 -0
- package/dist/managers/AdminApiController.d.ts +16 -16
- package/dist/managers/AdminApiController.js +35 -35
- package/dist/managers/DonobuFlow.d.ts +57 -36
- package/dist/managers/DonobuFlow.js +489 -564
- package/dist/managers/DonobuFlowsManager.js +13 -17
- package/dist/managers/FlowDependencyAnalyzer.d.ts +12 -12
- package/dist/managers/FlowDependencyAnalyzer.js +77 -77
- package/dist/managers/PageInspector.d.ts +38 -38
- package/dist/managers/PageInspector.js +745 -745
- package/dist/managers/TargetInspector.d.ts +28 -33
- package/dist/managers/TestsManager.d.ts +25 -25
- package/dist/managers/TestsManager.js +74 -74
- package/dist/managers/ToolManager.js +7 -5
- package/dist/managers/ToolRegistry.d.ts +5 -1
- package/dist/managers/WebTargetInspector.d.ts +9 -5
- package/dist/managers/WebTargetInspector.js +45 -47
- package/dist/models/AiQuery.d.ts +29 -15
- package/dist/models/AiQuery.js +31 -0
- package/dist/models/ControlPanel.d.ts +18 -13
- package/dist/models/InteractableElement.d.ts +6 -0
- package/dist/models/InteractableElement.js +7 -1
- package/dist/models/Observation.d.ts +38 -0
- package/dist/models/Observation.js +3 -0
- package/dist/models/ToolCallContext.d.ts +3 -2
- package/dist/persistence/flows/FlowsPersistenceDonobuApi.d.ts +2 -2
- package/dist/persistence/flows/FlowsPersistenceDonobuApi.js +19 -18
- package/dist/persistence/flows/FlowsPersistenceSqlite.js +2 -1
- package/dist/targets/TargetProvider.d.ts +110 -0
- package/dist/targets/TargetProvider.js +25 -0
- package/dist/targets/TargetRuntime.d.ts +6 -3
- package/dist/targets/WebDialogHandler.d.ts +14 -0
- package/dist/targets/WebDialogHandler.js +198 -0
- package/dist/targets/WebTargetProvider.d.ts +32 -0
- package/dist/targets/WebTargetProvider.js +136 -0
- package/dist/targets/WebTargetRuntime.d.ts +2 -2
- package/dist/targets/WebTargetRuntime.js +2 -1
- package/dist/tools/AcknowledgeUserInstruction.d.ts +6 -0
- package/dist/tools/AcknowledgeUserInstruction.js +7 -0
- package/dist/tools/AssertPageTool.d.ts +1 -1
- package/dist/tools/AssertPageTool.js +3 -3
- package/dist/tools/DetectBrokenLinksTool.d.ts +2 -2
- package/dist/tools/DetectBrokenLinksTool.js +44 -44
- package/dist/tools/InputFakerTool.d.ts +4 -4
- package/dist/tools/InputFakerTool.js +10 -10
- package/dist/tools/InputTextTool.d.ts +4 -4
- package/dist/tools/InputTextTool.js +7 -7
- package/dist/tools/ReplayableInteraction.d.ts +34 -34
- package/dist/tools/ReplayableInteraction.js +245 -245
- package/dist/tools/Tool.d.ts +6 -3
- package/dist/tools/Tool.js +5 -2
- package/dist/utils/BrowserUtils.d.ts +19 -19
- package/dist/utils/BrowserUtils.js +57 -57
- package/dist/utils/MiscUtils.d.ts +2 -2
- package/dist/utils/MiscUtils.js +16 -16
- package/dist/utils/PlaywrightUtils.d.ts +1 -1
- package/dist/utils/TargetUtils.d.ts +1 -1
- package/dist/utils/TargetUtils.js +15 -13
- package/package.json +2 -1
|
@@ -65,695 +65,120 @@ class PageInspector {
|
|
|
65
65
|
this.interactableAnnotationAttribute = interactableAnnotationAttribute;
|
|
66
66
|
}
|
|
67
67
|
/**
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* 1. Removes any pre-existing interactable element attributes from the page
|
|
72
|
-
* 2. Assigns sequential numeric values as attributes to interactable elements in the main frame
|
|
73
|
-
* 3. Processes child frames that are visible in the viewport and assigns attributes to their interactable elements
|
|
74
|
-
*
|
|
75
|
-
* The method identifies "interactable" elements based on tag names, ARIA roles, CSS classes, and other heuristics.
|
|
76
|
-
* Only elements that are:
|
|
77
|
-
* - Visible (non-zero dimensions and not hidden via CSS)
|
|
78
|
-
* - More than 50% in the viewport
|
|
79
|
-
* - Not disabled or inert
|
|
80
|
-
* - Actually reachable at their coordinates (topmost in z-index)
|
|
81
|
-
* will receive the attribute.
|
|
82
|
-
*
|
|
83
|
-
* @param page - The Playwright Page object to process
|
|
84
|
-
* @throws {PageClosedException} If the page is closed during processing
|
|
85
|
-
* @returns {Promise<void>} A promise that resolves when all elements have been attributed
|
|
68
|
+
* Converts an HTML attribute to a JavaScript attribute. For example,
|
|
69
|
+
* "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
|
|
70
|
+
* prefix, and the conversion from kebab-case to camelCase.
|
|
86
71
|
*/
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
await this.deattributeInteractableElements(page);
|
|
91
|
-
// Get viewport dimensions and scroll position properly
|
|
92
|
-
const viewportInfo = await page.evaluate(() => {
|
|
93
|
-
return {
|
|
94
|
-
viewportWidth: window.innerWidth,
|
|
95
|
-
viewportHeight: window.innerHeight,
|
|
96
|
-
scrollX: window.scrollX || window.pageXOffset,
|
|
97
|
-
scrollY: window.scrollY || window.pageYOffset,
|
|
98
|
-
};
|
|
99
|
-
});
|
|
100
|
-
// 1) Attribute elements in the main page
|
|
101
|
-
let annotationOffset = await page.evaluate(PageInspector.attributeElementsInContext, [0, this.interactableElementAttribute]);
|
|
102
|
-
// 2) Check child frames, attributing elements if the frame is (partially) in view
|
|
103
|
-
const frames = page
|
|
104
|
-
.frames()
|
|
105
|
-
.filter((frame) => PageInspector.frameFilter(frame) && frame !== page.mainFrame());
|
|
106
|
-
for (const frame of frames) {
|
|
107
|
-
const elementHandle = await frame.frameElement();
|
|
108
|
-
if (!elementHandle) {
|
|
109
|
-
continue;
|
|
110
|
-
}
|
|
111
|
-
const boundingBox = await elementHandle.boundingBox();
|
|
112
|
-
if (!boundingBox) {
|
|
113
|
-
continue;
|
|
114
|
-
}
|
|
115
|
-
// boundingBox coordinates are already in viewport space, so we need to account for scroll
|
|
116
|
-
const isInViewport = boundingBox.x + boundingBox.width > 0 &&
|
|
117
|
-
boundingBox.x < viewportInfo.viewportWidth &&
|
|
118
|
-
boundingBox.y + boundingBox.height > 0 &&
|
|
119
|
-
boundingBox.y < viewportInfo.viewportHeight;
|
|
120
|
-
if (isInViewport) {
|
|
121
|
-
annotationOffset = await frame.evaluate(PageInspector.attributeElementsInContext, [annotationOffset, this.interactableElementAttribute]);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
catch (error) {
|
|
126
|
-
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
127
|
-
throw new PageClosedException_1.PageClosedException();
|
|
128
|
-
}
|
|
129
|
-
else {
|
|
130
|
-
throw error;
|
|
131
|
-
}
|
|
72
|
+
static convertToJsAttribute(htmlAttribute) {
|
|
73
|
+
if (htmlAttribute.startsWith('data-')) {
|
|
74
|
+
htmlAttribute = htmlAttribute.substring(5);
|
|
132
75
|
}
|
|
76
|
+
const parts = htmlAttribute.split('-');
|
|
77
|
+
const jsAttribute = parts[0] +
|
|
78
|
+
parts
|
|
79
|
+
.slice(1)
|
|
80
|
+
.map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase())
|
|
81
|
+
.join('');
|
|
82
|
+
return jsAttribute;
|
|
133
83
|
}
|
|
134
84
|
/**
|
|
135
|
-
*
|
|
85
|
+
* An internal method that is injected into page/frame contexts to find and attribute interactable elements.
|
|
136
86
|
*
|
|
137
87
|
* This method:
|
|
138
|
-
* 1.
|
|
139
|
-
* 2.
|
|
140
|
-
* 3.
|
|
88
|
+
* 1. Identifies potentially interactable elements using a comprehensive selector
|
|
89
|
+
* 2. Filters elements based on visibility, position in viewport, and interactability
|
|
90
|
+
* 3. Assigns unique sequential numeric values to the interactable attribute
|
|
141
91
|
*
|
|
142
|
-
*
|
|
143
|
-
* -
|
|
144
|
-
* -
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
* * For all other elements, only the opening tag without children is captured
|
|
148
|
-
* * For the main scrolling element (document.scrollingElement), adds special decoration indicating it's the page's main scrolling element
|
|
92
|
+
* The method uses several criteria to determine if an element is truly interactable:
|
|
93
|
+
* - Element must be visible (non-zero dimensions, not hidden via CSS)
|
|
94
|
+
* - Element must have at least 50% of its area within the viewport
|
|
95
|
+
* - Element must not be disabled, inert, or have pointer-events:none
|
|
96
|
+
* - Element must be the topmost element at its coordinates (using point sampling)
|
|
149
97
|
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
98
|
+
* Special handling is provided for label elements, which will attribute their
|
|
99
|
+
* associated form controls as well.
|
|
152
100
|
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
* interactable elements with their attribute values and HTML snippets
|
|
156
|
-
* @throws {PageClosedException} If the page is closed during processing
|
|
101
|
+
* This method can process both standard DOM elements and elements within shadow roots,
|
|
102
|
+
* ensuring thorough coverage of modern web applications.
|
|
157
103
|
*
|
|
158
|
-
* @
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
104
|
+
* @param arg - A tuple containing [offset: number, interactableAttribute: string]
|
|
105
|
+
* where offset is the starting value for sequential numbering and
|
|
106
|
+
* interactableAttribute is the attribute name to assign
|
|
107
|
+
* @returns The updated offset after assigning attributes (for sequential numbering across frames)
|
|
108
|
+
* @private
|
|
109
|
+
*
|
|
110
|
+
* @remarks
|
|
111
|
+
* This method is designed to be injected into the page context using page.evaluate()
|
|
112
|
+
* and should not be called directly from Node.js code.
|
|
163
113
|
*/
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
let node = el;
|
|
174
|
-
while (node) {
|
|
175
|
-
Array.from(node.attributes).forEach((attr) => {
|
|
176
|
-
// Strip out Donobu attributes since those are not a part of the
|
|
177
|
-
// original HTML.
|
|
178
|
-
if (attr.name.startsWith('data-donobu')) {
|
|
179
|
-
node.removeAttribute(attr.name);
|
|
180
|
-
}
|
|
181
|
-
});
|
|
182
|
-
node = walker.nextNode();
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
/** helper to compute live scroll directions for el */
|
|
186
|
-
function getScrollDirections(el) {
|
|
187
|
-
// Special case for when the document body is not the scrollingElement
|
|
188
|
-
// element. This may happen if the scrollingElement is the
|
|
189
|
-
// root <html> element. In this case, it makes no sense to report
|
|
190
|
-
// scrollability on <body> and on scrollingElement, since we should
|
|
191
|
-
// use the scrollingElement instead.
|
|
192
|
-
if (el === document.body &&
|
|
193
|
-
document.scrollingElement !== document.body) {
|
|
194
|
-
return [];
|
|
195
|
-
}
|
|
196
|
-
const dirs = [];
|
|
197
|
-
const isRoot = el === document.scrollingElement;
|
|
198
|
-
const style = getComputedStyle(el);
|
|
199
|
-
// Add a small margin so we do not waste time reporting scrollability
|
|
200
|
-
// for an element that is not materially scrollable.
|
|
201
|
-
const marginPx = 1;
|
|
202
|
-
const canY = el.scrollHeight > el.clientHeight + marginPx &&
|
|
203
|
-
(isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowY));
|
|
204
|
-
const canX = el.scrollWidth > el.clientWidth + marginPx &&
|
|
205
|
-
(isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowX));
|
|
206
|
-
if (canY) {
|
|
207
|
-
if (el.scrollTop > 0) {
|
|
208
|
-
dirs.push('UP');
|
|
209
|
-
}
|
|
210
|
-
if (el.scrollTop < el.scrollHeight - el.clientHeight) {
|
|
211
|
-
dirs.push('DOWN');
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
if (canX) {
|
|
215
|
-
if (el.scrollLeft > 0) {
|
|
216
|
-
dirs.push('LEFT');
|
|
217
|
-
}
|
|
218
|
-
if (el.scrollLeft < el.scrollWidth - el.clientWidth) {
|
|
219
|
-
dirs.push('RIGHT');
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
return dirs;
|
|
223
|
-
}
|
|
224
|
-
function serialise(el) {
|
|
225
|
-
const deepClone = (el.tagName.toLowerCase() === 'select'
|
|
226
|
-
? el.cloneNode(true)
|
|
227
|
-
: el.cloneNode(false));
|
|
228
|
-
stripDonobuAttrs(deepClone);
|
|
229
|
-
if (el.tagName.toLowerCase() === 'select') {
|
|
230
|
-
const scrollComment = el === document.scrollingElement
|
|
231
|
-
? '<!-- This is the main page scrolling element -->'
|
|
232
|
-
: '';
|
|
233
|
-
return scrollComment + deepClone.outerHTML; // full markup incl. <option>s
|
|
234
|
-
}
|
|
235
|
-
// Get the text content of the original element
|
|
236
|
-
const textContent = el.textContent?.trim() || '';
|
|
237
|
-
if (textContent) {
|
|
238
|
-
// Truncate text if longer than 32 characters
|
|
239
|
-
const displayText = textContent.length > 32
|
|
240
|
-
? textContent.substring(0, 32) + '...'
|
|
241
|
-
: textContent;
|
|
242
|
-
// Return opening tag + text + closing tag
|
|
243
|
-
const fullTag = deepClone.outerHTML;
|
|
244
|
-
const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
|
|
245
|
-
const tagName = el.tagName.toLowerCase();
|
|
246
|
-
const scrollComment = el === document.scrollingElement
|
|
247
|
-
? '<!-- This is the main page scrolling element -->'
|
|
248
|
-
: '';
|
|
249
|
-
return `${scrollComment}${openingTag}${displayText}</${tagName}>`;
|
|
250
|
-
}
|
|
251
|
-
else {
|
|
252
|
-
// opening tag only
|
|
253
|
-
const html = deepClone.outerHTML;
|
|
254
|
-
const scrollComment = el === document.scrollingElement
|
|
255
|
-
? '<!-- This is the main page scrolling element -->'
|
|
256
|
-
: '';
|
|
257
|
-
return scrollComment + html.slice(0, html.indexOf('>') + 1);
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
const out = {};
|
|
261
|
-
// Recursively process document and all shadow roots
|
|
262
|
-
const processNode = (root) => {
|
|
263
|
-
// Find elements with the interactable attribute
|
|
264
|
-
root.querySelectorAll(`[${interactableAttr}]`).forEach((el) => {
|
|
265
|
-
const val = el.getAttribute(interactableAttr);
|
|
266
|
-
if (!val) {
|
|
267
|
-
return;
|
|
268
|
-
}
|
|
269
|
-
out[val] = {
|
|
270
|
-
htmlSnippet: serialise(el),
|
|
271
|
-
scrollable: getScrollDirections(el),
|
|
272
|
-
};
|
|
273
|
-
});
|
|
274
|
-
// Recursively process any child shadow roots
|
|
275
|
-
root.querySelectorAll('*').forEach((el) => {
|
|
276
|
-
if (el.shadowRoot) {
|
|
277
|
-
processNode(el.shadowRoot);
|
|
278
|
-
}
|
|
279
|
-
});
|
|
280
|
-
};
|
|
281
|
-
// Start processing from the document root
|
|
282
|
-
processNode(document);
|
|
283
|
-
return out;
|
|
284
|
-
}, this.interactableElementAttribute);
|
|
285
|
-
Object.assign(aggregate, frameMap);
|
|
286
|
-
}
|
|
287
|
-
return Object.keys(aggregate)
|
|
288
|
-
.sort((a, b) => Number(a) - Number(b))
|
|
289
|
-
.map((key) => ({
|
|
290
|
-
donobuAttributeValue: key,
|
|
291
|
-
htmlSnippet: aggregate[key].htmlSnippet,
|
|
292
|
-
scrollable: aggregate[key].scrollable,
|
|
293
|
-
}));
|
|
294
|
-
}
|
|
295
|
-
catch (error) {
|
|
296
|
-
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
297
|
-
throw new PageClosedException_1.PageClosedException();
|
|
298
|
-
}
|
|
299
|
-
else {
|
|
300
|
-
throw error;
|
|
301
|
-
}
|
|
114
|
+
static attributeElementsInContext(arg) {
|
|
115
|
+
let offset = arg[0];
|
|
116
|
+
const interactableAttribute = arg[1];
|
|
117
|
+
// --- Utility Functions ---
|
|
118
|
+
function isElementVisible(rect, style) {
|
|
119
|
+
return (rect.width > 0 &&
|
|
120
|
+
rect.height > 0 &&
|
|
121
|
+
style.display !== 'none' &&
|
|
122
|
+
style.visibility !== 'hidden');
|
|
302
123
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
const frames = page
|
|
337
|
-
.frames()
|
|
338
|
-
.filter((frame) => PageInspector.frameFilter(frame));
|
|
339
|
-
for (const frame of frames) {
|
|
340
|
-
await frame.evaluate(([interactableAttr, annotationAttr]) => {
|
|
341
|
-
// 1) Ensure we have a shadow container in the main document
|
|
342
|
-
let container = document.getElementById('annotation-shadow-container');
|
|
343
|
-
if (!container) {
|
|
344
|
-
container = document.createElement('div');
|
|
345
|
-
container.id = 'annotation-shadow-container';
|
|
346
|
-
// Position container so child elements can be absolutely placed
|
|
347
|
-
Object.assign(container.style, {
|
|
348
|
-
position: 'absolute',
|
|
349
|
-
top: '0',
|
|
350
|
-
left: '0',
|
|
351
|
-
width: '100%',
|
|
352
|
-
height: '100%',
|
|
353
|
-
pointerEvents: 'none', // Let clicks pass through
|
|
354
|
-
zIndex: '2147483647', // win every z-index fight
|
|
355
|
-
});
|
|
356
|
-
// Check if document.body exists before trying to append.
|
|
357
|
-
if (document.body) {
|
|
358
|
-
document.body.appendChild(container);
|
|
359
|
-
}
|
|
360
|
-
else if (document.documentElement) {
|
|
361
|
-
// Fall back to document.documentElement if body does not exist.
|
|
362
|
-
document.documentElement.appendChild(container);
|
|
363
|
-
}
|
|
364
|
-
else {
|
|
365
|
-
// If neither exists, we can't proceed with annotations in this frame.
|
|
366
|
-
console.warn(`Cannot create annotation container for ${window.location.href} since the document structure is incomplete`);
|
|
367
|
-
return;
|
|
368
|
-
}
|
|
369
|
-
// Attach a shadow root
|
|
370
|
-
const shadowRoot = container.attachShadow({ mode: 'open' });
|
|
371
|
-
// Add a <style> element inside the shadow root to reset and define annotation styles
|
|
372
|
-
const style = document.createElement('style');
|
|
373
|
-
style.textContent = `
|
|
374
|
-
:host {
|
|
375
|
-
all: initial; /* Reset styles in shadow root */
|
|
376
|
-
}
|
|
377
|
-
.annotation {
|
|
378
|
-
position: absolute;
|
|
379
|
-
z-index: 2147483647;
|
|
380
|
-
background-color: black;
|
|
381
|
-
color: white;
|
|
382
|
-
width: 40px;
|
|
383
|
-
height: 40px;
|
|
384
|
-
border-radius: 50%;
|
|
385
|
-
display: flex;
|
|
386
|
-
align-items: center;
|
|
387
|
-
justify-content: center;
|
|
388
|
-
font-size: 14px;
|
|
389
|
-
font-weight: bold;
|
|
390
|
-
line-height: 20px;
|
|
391
|
-
text-align: center;
|
|
392
|
-
box-shadow: 0px 2px 4px rgba(0,0,0,0.2);
|
|
393
|
-
border: 4px solid #FF4136;
|
|
394
|
-
pointer-events: none;
|
|
124
|
+
function isElementMoreThanHalfInViewport(rect) {
|
|
125
|
+
const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
|
|
126
|
+
const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
|
|
127
|
+
const visibleWidth = Math.min(rect.right, viewportWidth) - Math.max(rect.left, 0);
|
|
128
|
+
const visibleHeight = Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0);
|
|
129
|
+
const visibleArea = Math.max(0, visibleWidth) * Math.max(0, visibleHeight);
|
|
130
|
+
const elementArea = rect.width * rect.height;
|
|
131
|
+
return visibleArea >= elementArea / 2;
|
|
132
|
+
}
|
|
133
|
+
function isElementEnabled(element, style) {
|
|
134
|
+
// Check standard disabled attribute (for form controls like button, input, etc.)
|
|
135
|
+
if (element.hasAttribute('disabled')) {
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
// Check for ARIA attributes that indicate non-interactivity
|
|
139
|
+
if (element.getAttribute('aria-hidden') === 'true') {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
// Check for pointer-events: none which prevents interactions
|
|
143
|
+
if (style.pointerEvents === 'none') {
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
// Check for inert attribute which makes elements non-interactive
|
|
147
|
+
if (element.hasAttribute('inert')) {
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
// If the element is in a form and the fieldset is disabled, it might be disabled as well
|
|
151
|
+
let parent = element.parentElement;
|
|
152
|
+
while (parent) {
|
|
153
|
+
if (parent.tagName.toLowerCase() === 'fieldset' &&
|
|
154
|
+
parent.hasAttribute('disabled') &&
|
|
155
|
+
element.tagName.toLowerCase() !== 'legend') {
|
|
156
|
+
return false;
|
|
395
157
|
}
|
|
396
|
-
|
|
397
|
-
shadowRoot.appendChild(style);
|
|
398
|
-
}
|
|
399
|
-
// Retrieve the shadow root to place annotation elements
|
|
400
|
-
const containerEl = document.getElementById('annotation-shadow-container');
|
|
401
|
-
if (!containerEl?.shadowRoot) {
|
|
402
|
-
return;
|
|
403
|
-
}
|
|
404
|
-
const shadowRoot = containerEl.shadowRoot;
|
|
405
|
-
// 2) Factory to create a new annotation inside the shadow root
|
|
406
|
-
const createAnnotation = (value) => {
|
|
407
|
-
const annotation = document.createElement('div');
|
|
408
|
-
annotation.classList.add('annotation');
|
|
409
|
-
annotation.dataset[annotationAttr] = '1';
|
|
410
|
-
annotation.textContent = value;
|
|
411
|
-
return annotation;
|
|
412
|
-
};
|
|
413
|
-
// 3) Position annotation relative to an element
|
|
414
|
-
const positionAnnotation = (annotation, element) => {
|
|
415
|
-
const rect = element.getBoundingClientRect();
|
|
416
|
-
// Center the annotation on the element, adjusting for its size
|
|
417
|
-
// Since container is absolute, we need to account for scroll position
|
|
418
|
-
const x = rect.left + rect.width / 2 - 20 + window.scrollX;
|
|
419
|
-
const y = rect.top + rect.height / 2 - 20 + window.scrollY;
|
|
420
|
-
annotation.style.left = `${x}px`;
|
|
421
|
-
annotation.style.top = `${y}px`;
|
|
422
|
-
};
|
|
423
|
-
// 4) Traverse DOM (including any nested shadow roots) to find interactable elements
|
|
424
|
-
const processNode = (root) => {
|
|
425
|
-
// Find elements with the interactable attribute
|
|
426
|
-
const elements = root.querySelectorAll(`[${interactableAttr}]`);
|
|
427
|
-
elements.forEach((element) => {
|
|
428
|
-
const value = element.getAttribute(interactableAttr);
|
|
429
|
-
if (value) {
|
|
430
|
-
const annotation = createAnnotation(value);
|
|
431
|
-
shadowRoot.appendChild(annotation);
|
|
432
|
-
positionAnnotation(annotation, element);
|
|
433
|
-
}
|
|
434
|
-
});
|
|
435
|
-
// Recursively process any child shadow roots
|
|
436
|
-
root.querySelectorAll('*').forEach((el) => {
|
|
437
|
-
if (el.shadowRoot) {
|
|
438
|
-
processNode(el.shadowRoot);
|
|
439
|
-
}
|
|
440
|
-
});
|
|
441
|
-
};
|
|
442
|
-
// Start processing from the (frame) document root
|
|
443
|
-
processNode(document);
|
|
444
|
-
}, [
|
|
445
|
-
this.interactableElementAttribute,
|
|
446
|
-
PageInspector.convertToJsAttribute(this.interactableAnnotationAttribute),
|
|
447
|
-
]);
|
|
158
|
+
parent = parent.parentElement;
|
|
448
159
|
}
|
|
160
|
+
return true;
|
|
449
161
|
}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
162
|
+
function isScrollable(el) {
|
|
163
|
+
const canY = el.scrollHeight > el.clientHeight;
|
|
164
|
+
const canX = el.scrollWidth > el.clientWidth;
|
|
165
|
+
// If nothing overflows, bail early
|
|
166
|
+
if (!canY && !canX) {
|
|
167
|
+
return false;
|
|
453
168
|
}
|
|
454
|
-
|
|
455
|
-
|
|
169
|
+
// The document’s scrolling element works even when overflow is “visible”
|
|
170
|
+
if (el === document.scrollingElement) {
|
|
171
|
+
return true;
|
|
456
172
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
*
|
|
463
|
-
* This method:
|
|
464
|
-
* 1. Processes all accessible frames in the page
|
|
465
|
-
* 2. Finds and removes the shadow DOM container with ID 'annotation-shadow-container'
|
|
466
|
-
* that contains all the annotations
|
|
467
|
-
*
|
|
468
|
-
* This effectively removes all numbered indicators that were previously placed
|
|
469
|
-
* over interactable elements, leaving the page in its original visual state.
|
|
470
|
-
* Note that this only removes the visual annotations, not the
|
|
471
|
-
* {@link interactableElementAttribute} attributes on the elements themselves.
|
|
472
|
-
*
|
|
473
|
-
* @param page - The Playwright Page object to process
|
|
474
|
-
* @returns {Promise<void>} A promise that resolves when all annotations have been removed
|
|
475
|
-
* @throws {PageClosedException} If the page is closed during processing
|
|
476
|
-
*
|
|
477
|
-
* @example
|
|
478
|
-
* const inspector = new PageInspector();
|
|
479
|
-
* await inspector.attributeInteractableElements(page);
|
|
480
|
-
* await inspector.annotateInteractableElements(page);
|
|
481
|
-
* // ... do some operations with the annotations visible ...
|
|
482
|
-
* await inspector.removeDonobuAnnotations(page);
|
|
483
|
-
* // All visual annotations are now removed from the page
|
|
484
|
-
*/
|
|
485
|
-
async removeDonobuAnnotations(page) {
|
|
486
|
-
try {
|
|
487
|
-
const frames = page
|
|
488
|
-
.frames()
|
|
489
|
-
.filter((frame) => PageInspector.frameFilter(frame));
|
|
490
|
-
for (const frame of frames) {
|
|
491
|
-
await frame.evaluate(() => {
|
|
492
|
-
const containerId = 'annotation-shadow-container';
|
|
493
|
-
const container = document.getElementById(containerId);
|
|
494
|
-
if (container) {
|
|
495
|
-
container.remove();
|
|
496
|
-
}
|
|
497
|
-
});
|
|
173
|
+
const s = getComputedStyle(el);
|
|
174
|
+
const rect = el.getBoundingClientRect();
|
|
175
|
+
const visible = rect.width > 0 && rect.height > 0;
|
|
176
|
+
if (!visible) {
|
|
177
|
+
return false;
|
|
498
178
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
throw new PageClosedException_1.PageClosedException();
|
|
503
|
-
}
|
|
504
|
-
throw error;
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
/**
|
|
508
|
-
* Removes all interactable element attributes that were previously added to elements in the page.
|
|
509
|
-
*
|
|
510
|
-
* This method:
|
|
511
|
-
* 1. Processes all accessible frames in the page
|
|
512
|
-
* 2. Finds all elements with the {@link interactableElementAttribute} attribute
|
|
513
|
-
* 3. Removes this attribute from each element
|
|
514
|
-
*
|
|
515
|
-
* This effectively undoes the changes made by the {@link attributeInteractableElements} method,
|
|
516
|
-
* returning the page's DOM to its original state without the custom attributes.
|
|
517
|
-
* Note that this does not affect any visual annotations - to remove those, use
|
|
518
|
-
* the {@link removeDonobuAnnotations} method separately.
|
|
519
|
-
*
|
|
520
|
-
* This method is automatically called at the beginning of {@link attributeInteractableElements}
|
|
521
|
-
* to ensure a clean state before adding new attributes, but can also be called
|
|
522
|
-
* independently to clean up the DOM.
|
|
523
|
-
*
|
|
524
|
-
* @param page - The Playwright Page object to process
|
|
525
|
-
* @returns {Promise<void>} A promise that resolves when all attributes have been removed
|
|
526
|
-
* @throws {PageClosedException} If the page is closed during processing
|
|
527
|
-
*
|
|
528
|
-
* @example
|
|
529
|
-
* const inspector = new PageInspector();
|
|
530
|
-
* await inspector.attributeInteractableElements(page);
|
|
531
|
-
* // ... perform operations with attributed elements ...
|
|
532
|
-
* await inspector.deattributeInteractableElements(page);
|
|
533
|
-
* // All interactable element attributes are now removed from the page
|
|
534
|
-
*/
|
|
535
|
-
async deattributeInteractableElements(page) {
|
|
536
|
-
try {
|
|
537
|
-
const frames = page.frames().filter(PageInspector.frameFilter);
|
|
538
|
-
const attr = this.interactableElementAttribute;
|
|
539
|
-
for (const frame of frames) {
|
|
540
|
-
await frame.evaluate(([a]) => {
|
|
541
|
-
/** Depth-first removal inside document & every shadow root */
|
|
542
|
-
const removeDeep = (root) => {
|
|
543
|
-
root
|
|
544
|
-
.querySelectorAll(`[${a}]`)
|
|
545
|
-
.forEach((el) => el.removeAttribute(a));
|
|
546
|
-
root.querySelectorAll('*').forEach((el) => {
|
|
547
|
-
const sr = el.shadowRoot;
|
|
548
|
-
if (sr) {
|
|
549
|
-
removeDeep(sr);
|
|
550
|
-
}
|
|
551
|
-
});
|
|
552
|
-
};
|
|
553
|
-
removeDeep(document);
|
|
554
|
-
}, [attr]);
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
catch (error) {
|
|
558
|
-
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
559
|
-
throw new PageClosedException_1.PageClosedException();
|
|
560
|
-
}
|
|
561
|
-
else {
|
|
562
|
-
throw error;
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
/**
|
|
567
|
-
* Retrieves the HTML snippet for a single element.
|
|
568
|
-
*
|
|
569
|
-
* This method:
|
|
570
|
-
* 1. Extracts a simplified HTML snippet representation of the element
|
|
571
|
-
* * For 'select' elements, the complete HTML (including options) is preserved
|
|
572
|
-
* * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
|
|
573
|
-
* * For all other elements, only the opening tag without children is captured
|
|
574
|
-
* 2. Strips any Donobu-specific attributes from the snippet
|
|
575
|
-
*
|
|
576
|
-
* @example
|
|
577
|
-
* const inspector = new PageInspector();
|
|
578
|
-
* const submitButton = page.querySelector('button[type="submit"]');
|
|
579
|
-
* const htmlSnippet = await inspector.getHtmlSnippet(submitButton);
|
|
580
|
-
* // htmlSnippet = "<button type=\"submit\">Submit</button>"
|
|
581
|
-
*/
|
|
582
|
-
async getHtmlSnippet(elementHandle) {
|
|
583
|
-
try {
|
|
584
|
-
// Evaluate in the element's context to get the HTML snippet
|
|
585
|
-
const htmlSnippet = await elementHandle.evaluate((element) => {
|
|
586
|
-
// Helper function to strip Donobu attributes
|
|
587
|
-
function stripDonobuAttrs(el) {
|
|
588
|
-
const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
|
|
589
|
-
let node = el;
|
|
590
|
-
while (node) {
|
|
591
|
-
Array.from(node.attributes).forEach((attr) => {
|
|
592
|
-
// Strip out Donobu attributes since those are not a part of the
|
|
593
|
-
// original HTML.
|
|
594
|
-
if (attr.name.startsWith('data-donobu')) {
|
|
595
|
-
node.removeAttribute(attr.name);
|
|
596
|
-
}
|
|
597
|
-
});
|
|
598
|
-
node = walker.nextNode();
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
// Helper function to serialize element
|
|
602
|
-
function serialise(el) {
|
|
603
|
-
const deepClone = el.tagName.toLowerCase() === 'select'
|
|
604
|
-
? el.cloneNode(true)
|
|
605
|
-
: el.cloneNode(false);
|
|
606
|
-
stripDonobuAttrs(deepClone);
|
|
607
|
-
if (el.tagName.toLowerCase() === 'select') {
|
|
608
|
-
return deepClone.outerHTML; // full markup incl. <option>s
|
|
609
|
-
}
|
|
610
|
-
// Get the text content of the original element
|
|
611
|
-
const textContent = el.textContent?.trim() || '';
|
|
612
|
-
if (textContent) {
|
|
613
|
-
// Truncate text if longer than 32 characters
|
|
614
|
-
const displayText = textContent.length > 32
|
|
615
|
-
? textContent.substring(0, 32) + '...'
|
|
616
|
-
: textContent;
|
|
617
|
-
// Return opening tag + text + closing tag
|
|
618
|
-
const fullTag = deepClone.outerHTML;
|
|
619
|
-
const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
|
|
620
|
-
const tagName = el.tagName.toLowerCase();
|
|
621
|
-
return `${openingTag}${displayText}</${tagName}>`;
|
|
622
|
-
}
|
|
623
|
-
else {
|
|
624
|
-
// opening tag only
|
|
625
|
-
const html = deepClone.outerHTML;
|
|
626
|
-
return html.slice(0, html.indexOf('>') + 1);
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
|
-
return serialise(element);
|
|
630
|
-
});
|
|
631
|
-
return htmlSnippet;
|
|
632
|
-
}
|
|
633
|
-
catch (error) {
|
|
634
|
-
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
635
|
-
throw new PageClosedException_1.PageClosedException();
|
|
636
|
-
}
|
|
637
|
-
else {
|
|
638
|
-
throw error;
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
/**
|
|
643
|
-
* Converts an HTML attribute to a JavaScript attribute. For example,
|
|
644
|
-
* "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
|
|
645
|
-
* prefix, and the conversion from kebab-case to camelCase.
|
|
646
|
-
*/
|
|
647
|
-
static convertToJsAttribute(htmlAttribute) {
|
|
648
|
-
if (htmlAttribute.startsWith('data-')) {
|
|
649
|
-
htmlAttribute = htmlAttribute.substring(5);
|
|
650
|
-
}
|
|
651
|
-
const parts = htmlAttribute.split('-');
|
|
652
|
-
const jsAttribute = parts[0] +
|
|
653
|
-
parts
|
|
654
|
-
.slice(1)
|
|
655
|
-
.map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase())
|
|
656
|
-
.join('');
|
|
657
|
-
return jsAttribute;
|
|
658
|
-
}
|
|
659
|
-
/**
|
|
660
|
-
* An internal method that is injected into page/frame contexts to find and attribute interactable elements.
|
|
661
|
-
*
|
|
662
|
-
* This method:
|
|
663
|
-
* 1. Identifies potentially interactable elements using a comprehensive selector
|
|
664
|
-
* 2. Filters elements based on visibility, position in viewport, and interactability
|
|
665
|
-
* 3. Assigns unique sequential numeric values to the interactable attribute
|
|
666
|
-
*
|
|
667
|
-
* The method uses several criteria to determine if an element is truly interactable:
|
|
668
|
-
* - Element must be visible (non-zero dimensions, not hidden via CSS)
|
|
669
|
-
* - Element must have at least 50% of its area within the viewport
|
|
670
|
-
* - Element must not be disabled, inert, or have pointer-events:none
|
|
671
|
-
* - Element must be the topmost element at its coordinates (using point sampling)
|
|
672
|
-
*
|
|
673
|
-
* Special handling is provided for label elements, which will attribute their
|
|
674
|
-
* associated form controls as well.
|
|
675
|
-
*
|
|
676
|
-
* This method can process both standard DOM elements and elements within shadow roots,
|
|
677
|
-
* ensuring thorough coverage of modern web applications.
|
|
678
|
-
*
|
|
679
|
-
* @param arg - A tuple containing [offset: number, interactableAttribute: string]
|
|
680
|
-
* where offset is the starting value for sequential numbering and
|
|
681
|
-
* interactableAttribute is the attribute name to assign
|
|
682
|
-
* @returns The updated offset after assigning attributes (for sequential numbering across frames)
|
|
683
|
-
* @private
|
|
684
|
-
*
|
|
685
|
-
* @remarks
|
|
686
|
-
* This method is designed to be injected into the page context using page.evaluate()
|
|
687
|
-
* and should not be called directly from Node.js code.
|
|
688
|
-
*/
|
|
689
|
-
static attributeElementsInContext(arg) {
|
|
690
|
-
let offset = arg[0];
|
|
691
|
-
const interactableAttribute = arg[1];
|
|
692
|
-
// --- Utility Functions ---
|
|
693
|
-
function isElementVisible(rect, style) {
|
|
694
|
-
return (rect.width > 0 &&
|
|
695
|
-
rect.height > 0 &&
|
|
696
|
-
style.display !== 'none' &&
|
|
697
|
-
style.visibility !== 'hidden');
|
|
698
|
-
}
|
|
699
|
-
function isElementMoreThanHalfInViewport(rect) {
|
|
700
|
-
const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
|
|
701
|
-
const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
|
|
702
|
-
const visibleWidth = Math.min(rect.right, viewportWidth) - Math.max(rect.left, 0);
|
|
703
|
-
const visibleHeight = Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0);
|
|
704
|
-
const visibleArea = Math.max(0, visibleWidth) * Math.max(0, visibleHeight);
|
|
705
|
-
const elementArea = rect.width * rect.height;
|
|
706
|
-
return visibleArea >= elementArea / 2;
|
|
707
|
-
}
|
|
708
|
-
function isElementEnabled(element, style) {
|
|
709
|
-
// Check standard disabled attribute (for form controls like button, input, etc.)
|
|
710
|
-
if (element.hasAttribute('disabled')) {
|
|
711
|
-
return false;
|
|
712
|
-
}
|
|
713
|
-
// Check for ARIA attributes that indicate non-interactivity
|
|
714
|
-
if (element.getAttribute('aria-hidden') === 'true') {
|
|
715
|
-
return false;
|
|
716
|
-
}
|
|
717
|
-
// Check for pointer-events: none which prevents interactions
|
|
718
|
-
if (style.pointerEvents === 'none') {
|
|
719
|
-
return false;
|
|
720
|
-
}
|
|
721
|
-
// Check for inert attribute which makes elements non-interactive
|
|
722
|
-
if (element.hasAttribute('inert')) {
|
|
723
|
-
return false;
|
|
724
|
-
}
|
|
725
|
-
// If the element is in a form and the fieldset is disabled, it might be disabled as well
|
|
726
|
-
let parent = element.parentElement;
|
|
727
|
-
while (parent) {
|
|
728
|
-
if (parent.tagName.toLowerCase() === 'fieldset' &&
|
|
729
|
-
parent.hasAttribute('disabled') &&
|
|
730
|
-
element.tagName.toLowerCase() !== 'legend') {
|
|
731
|
-
return false;
|
|
732
|
-
}
|
|
733
|
-
parent = parent.parentElement;
|
|
734
|
-
}
|
|
735
|
-
return true;
|
|
736
|
-
}
|
|
737
|
-
function isScrollable(el) {
|
|
738
|
-
const canY = el.scrollHeight > el.clientHeight;
|
|
739
|
-
const canX = el.scrollWidth > el.clientWidth;
|
|
740
|
-
// If nothing overflows, bail early
|
|
741
|
-
if (!canY && !canX) {
|
|
742
|
-
return false;
|
|
743
|
-
}
|
|
744
|
-
// The document’s scrolling element works even when overflow is “visible”
|
|
745
|
-
if (el === document.scrollingElement) {
|
|
746
|
-
return true;
|
|
747
|
-
}
|
|
748
|
-
const s = getComputedStyle(el);
|
|
749
|
-
const rect = el.getBoundingClientRect();
|
|
750
|
-
const visible = rect.width > 0 && rect.height > 0;
|
|
751
|
-
if (!visible) {
|
|
752
|
-
return false;
|
|
753
|
-
}
|
|
754
|
-
const yOK = canY && /(auto|scroll|overlay)/.test(s.overflowY);
|
|
755
|
-
const xOK = canX && /(auto|scroll|overlay)/.test(s.overflowX);
|
|
756
|
-
return yOK || xOK;
|
|
179
|
+
const yOK = canY && /(auto|scroll|overlay)/.test(s.overflowY);
|
|
180
|
+
const xOK = canX && /(auto|scroll|overlay)/.test(s.overflowX);
|
|
181
|
+
return yOK || xOK;
|
|
757
182
|
}
|
|
758
183
|
/**
|
|
759
184
|
* Generate a few test points on the element's bounding box. We only need
|
|
@@ -928,91 +353,666 @@ class PageInspector {
|
|
|
928
353
|
if (isScrollable(el) && !uniqueElements.has(el)) {
|
|
929
354
|
uniqueElements.add(el);
|
|
930
355
|
}
|
|
931
|
-
};
|
|
932
|
-
document.querySelectorAll('*').forEach(maybeAddScrollable);
|
|
933
|
-
if (document.scrollingElement) {
|
|
934
|
-
maybeAddScrollable(document.scrollingElement);
|
|
356
|
+
};
|
|
357
|
+
document.querySelectorAll('*').forEach(maybeAddScrollable);
|
|
358
|
+
if (document.scrollingElement) {
|
|
359
|
+
maybeAddScrollable(document.scrollingElement);
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Run the visibility / enabled / top-most checks on a single element and,
|
|
363
|
+
* if they pass, assign it the next interactable number. Returns `true` if
|
|
364
|
+
* the element (or, via the <label htmlFor> mapping, its associated control)
|
|
365
|
+
* was attributed.
|
|
366
|
+
*/
|
|
367
|
+
function tryAttributeElement(element) {
|
|
368
|
+
if (element.hasAttribute(interactableAttribute)) {
|
|
369
|
+
return false;
|
|
370
|
+
}
|
|
371
|
+
const rect = element.getBoundingClientRect();
|
|
372
|
+
const style = window.getComputedStyle(element);
|
|
373
|
+
const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
|
|
374
|
+
const enabled = isElementEnabled(element, style);
|
|
375
|
+
if (!visible || !enabled) {
|
|
376
|
+
return false;
|
|
377
|
+
}
|
|
378
|
+
// Check a few probe points to make sure the element is top-most
|
|
379
|
+
for (const pt of getPointsToCheck(rect)) {
|
|
380
|
+
let elToCheck = getDeepElementFromPoint(pt.x, pt.y);
|
|
381
|
+
while (elToCheck) {
|
|
382
|
+
if (elToCheck === element) {
|
|
383
|
+
element.setAttribute(interactableAttribute, offset.toString());
|
|
384
|
+
offset++;
|
|
385
|
+
return true; // this element done
|
|
386
|
+
}
|
|
387
|
+
// Handle <label> -> control mapping (explicit `for`/`htmlFor`)
|
|
388
|
+
if (elToCheck.tagName.toLowerCase() === 'label' &&
|
|
389
|
+
elToCheck.htmlFor) {
|
|
390
|
+
const forId = elToCheck.htmlFor;
|
|
391
|
+
const control = document.getElementById(forId);
|
|
392
|
+
if (control &&
|
|
393
|
+
!control.hasAttribute(interactableAttribute) // prevent double number
|
|
394
|
+
) {
|
|
395
|
+
control.setAttribute(interactableAttribute, offset.toString());
|
|
396
|
+
offset++;
|
|
397
|
+
}
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
elToCheck = elToCheck.parentElement;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
return false;
|
|
404
|
+
}
|
|
405
|
+
// 2) Iterate and assign numbers
|
|
406
|
+
uniqueElements.forEach((element) => {
|
|
407
|
+
if (element === document.scrollingElement) {
|
|
408
|
+
// Special-case: always keep the root scrolling element
|
|
409
|
+
element.setAttribute(interactableAttribute, offset.toString());
|
|
410
|
+
offset++;
|
|
411
|
+
return; // skip the usual checks
|
|
412
|
+
}
|
|
413
|
+
else if (element.hasAttribute(interactableAttribute)) {
|
|
414
|
+
// Skip if this element already carries a value (e.g. assigned via <label>)
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
if (tryAttributeElement(element)) {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
// Fallback: the element is a visually-hidden native control (e.g. a 0x0,
|
|
421
|
+
// opacity-0, pointer-events:none <input>) wrapped in a styled <label>.
|
|
422
|
+
// This is the standard pattern for Ant Design Segmented/Radio/Checkbox/
|
|
423
|
+
// Switch and many other component libraries: the native input is hidden
|
|
424
|
+
// and the surrounding <label> is the real clickable surface. Since the
|
|
425
|
+
// hidden control fails the visibility/enabled checks above, attribute the
|
|
426
|
+
// wrapping <label> instead so the toggle is still annotated.
|
|
427
|
+
const wrappingLabel = element.closest('label');
|
|
428
|
+
if (wrappingLabel &&
|
|
429
|
+
wrappingLabel !== element &&
|
|
430
|
+
!wrappingLabel.hasAttribute(interactableAttribute)) {
|
|
431
|
+
tryAttributeElement(wrappingLabel);
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
return offset;
|
|
435
|
+
}
|
|
436
|
+
static frameFilter(frame) {
|
|
437
|
+
return (!frame.isDetached() &&
|
|
438
|
+
!frame.url().startsWith('about:') &&
|
|
439
|
+
!frame.url().startsWith('chrome:') &&
|
|
440
|
+
!frame.url().startsWith('edge:'));
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* Assigns a globally unique attribute to all visible and interactable elements in the page.
|
|
444
|
+
*
|
|
445
|
+
* This method performs the following steps:
|
|
446
|
+
* 1. Removes any pre-existing interactable element attributes from the page
|
|
447
|
+
* 2. Assigns sequential numeric values as attributes to interactable elements in the main frame
|
|
448
|
+
* 3. Processes child frames that are visible in the viewport and assigns attributes to their interactable elements
|
|
449
|
+
*
|
|
450
|
+
* The method identifies "interactable" elements based on tag names, ARIA roles, CSS classes, and other heuristics.
|
|
451
|
+
* Only elements that are:
|
|
452
|
+
* - Visible (non-zero dimensions and not hidden via CSS)
|
|
453
|
+
* - More than 50% in the viewport
|
|
454
|
+
* - Not disabled or inert
|
|
455
|
+
* - Actually reachable at their coordinates (topmost in z-index)
|
|
456
|
+
* will receive the attribute.
|
|
457
|
+
*
|
|
458
|
+
* @param page - The Playwright Page object to process
|
|
459
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
460
|
+
* @returns {Promise<void>} A promise that resolves when all elements have been attributed
|
|
461
|
+
*/
|
|
462
|
+
async attributeInteractableElements(page) {
|
|
463
|
+
try {
|
|
464
|
+
// Remove any preexisting attributes
|
|
465
|
+
await this.deattributeInteractableElements(page);
|
|
466
|
+
// Get viewport dimensions and scroll position properly
|
|
467
|
+
const viewportInfo = await page.evaluate(() => {
|
|
468
|
+
return {
|
|
469
|
+
viewportWidth: window.innerWidth,
|
|
470
|
+
viewportHeight: window.innerHeight,
|
|
471
|
+
scrollX: window.scrollX || window.pageXOffset,
|
|
472
|
+
scrollY: window.scrollY || window.pageYOffset,
|
|
473
|
+
};
|
|
474
|
+
});
|
|
475
|
+
// 1) Attribute elements in the main page
|
|
476
|
+
let annotationOffset = await page.evaluate(PageInspector.attributeElementsInContext, [0, this.interactableElementAttribute]);
|
|
477
|
+
// 2) Check child frames, attributing elements if the frame is (partially) in view
|
|
478
|
+
const frames = page
|
|
479
|
+
.frames()
|
|
480
|
+
.filter((frame) => PageInspector.frameFilter(frame) && frame !== page.mainFrame());
|
|
481
|
+
for (const frame of frames) {
|
|
482
|
+
const elementHandle = await frame.frameElement();
|
|
483
|
+
if (!elementHandle) {
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
const boundingBox = await elementHandle.boundingBox();
|
|
487
|
+
if (!boundingBox) {
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
// boundingBox coordinates are already in viewport space, so we need to account for scroll
|
|
491
|
+
const isInViewport = boundingBox.x + boundingBox.width > 0 &&
|
|
492
|
+
boundingBox.x < viewportInfo.viewportWidth &&
|
|
493
|
+
boundingBox.y + boundingBox.height > 0 &&
|
|
494
|
+
boundingBox.y < viewportInfo.viewportHeight;
|
|
495
|
+
if (isInViewport) {
|
|
496
|
+
annotationOffset = await frame.evaluate(PageInspector.attributeElementsInContext, [annotationOffset, this.interactableElementAttribute]);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
catch (error) {
|
|
501
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
502
|
+
throw new PageClosedException_1.PageClosedException();
|
|
503
|
+
}
|
|
504
|
+
else {
|
|
505
|
+
throw error;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* Retrieves all elements that have been previously attributed with the interactable element attribute.
|
|
511
|
+
*
|
|
512
|
+
* This method:
|
|
513
|
+
* 1. Searches all frames in the page (including the main frame and child frames)
|
|
514
|
+
* 2. Collects elements with the {@link interactableElementAttribute} attribute
|
|
515
|
+
* 3. Creates an {@link InteractableElement} object for each attributed element
|
|
516
|
+
*
|
|
517
|
+
* For each interactable element, it extracts:
|
|
518
|
+
* - The attribute value (serving as a unique identifier)
|
|
519
|
+
* - A simplified HTML snippet representation of the element
|
|
520
|
+
* * For 'select' elements, the complete HTML (including options) is preserved
|
|
521
|
+
* * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
|
|
522
|
+
* * For all other elements, only the opening tag without children is captured
|
|
523
|
+
* * For the main scrolling element (document.scrollingElement), adds special decoration indicating it's the page's main scrolling element
|
|
524
|
+
*
|
|
525
|
+
* Note: This method only finds elements that have been previously attributed using
|
|
526
|
+
* the {@link attributeInteractableElements} method.
|
|
527
|
+
*
|
|
528
|
+
* @param page - The Playwright Page object to process
|
|
529
|
+
* @returns {Promise<InteractableElement[]>} A promise that resolves to an array of
|
|
530
|
+
* interactable elements with their attribute values and HTML snippets
|
|
531
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
532
|
+
*
|
|
533
|
+
* @example
|
|
534
|
+
* const inspector = new PageInspector();
|
|
535
|
+
* await inspector.attributeInteractableElements(page);
|
|
536
|
+
* const elements = await inspector.getAttributedInteractableElements(page);
|
|
537
|
+
* // elements = [{ donobuAttributeValue: "0", htmlSnippet: "<button id=\"submit\">Submit</button>"}]
|
|
538
|
+
*/
|
|
539
|
+
async getAttributedInteractableElements(page) {
|
|
540
|
+
try {
|
|
541
|
+
const frames = page.frames().filter(PageInspector.frameFilter);
|
|
542
|
+
const aggregate = {};
|
|
543
|
+
for (const frame of frames) {
|
|
544
|
+
const frameMap = await frame.evaluate((interactableAttr) => {
|
|
545
|
+
/* --- helpers running in the browser context --- */
|
|
546
|
+
function stripDonobuAttrs(el) {
|
|
547
|
+
const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
|
|
548
|
+
let node = el;
|
|
549
|
+
while (node) {
|
|
550
|
+
Array.from(node.attributes).forEach((attr) => {
|
|
551
|
+
// Strip out Donobu attributes since those are not a part of the
|
|
552
|
+
// original HTML.
|
|
553
|
+
if (attr.name.startsWith('data-donobu')) {
|
|
554
|
+
node.removeAttribute(attr.name);
|
|
555
|
+
}
|
|
556
|
+
});
|
|
557
|
+
node = walker.nextNode();
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
/** helper to compute live scroll directions for el */
|
|
561
|
+
function getScrollDirections(el) {
|
|
562
|
+
// Special case for when the document body is not the scrollingElement
|
|
563
|
+
// element. This may happen if the scrollingElement is the
|
|
564
|
+
// root <html> element. In this case, it makes no sense to report
|
|
565
|
+
// scrollability on <body> and on scrollingElement, since we should
|
|
566
|
+
// use the scrollingElement instead.
|
|
567
|
+
if (el === document.body &&
|
|
568
|
+
document.scrollingElement !== document.body) {
|
|
569
|
+
return [];
|
|
570
|
+
}
|
|
571
|
+
const dirs = [];
|
|
572
|
+
const isRoot = el === document.scrollingElement;
|
|
573
|
+
const style = getComputedStyle(el);
|
|
574
|
+
// Add a small margin so we do not waste time reporting scrollability
|
|
575
|
+
// for an element that is not materially scrollable.
|
|
576
|
+
const marginPx = 1;
|
|
577
|
+
const canY = el.scrollHeight > el.clientHeight + marginPx &&
|
|
578
|
+
(isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowY));
|
|
579
|
+
const canX = el.scrollWidth > el.clientWidth + marginPx &&
|
|
580
|
+
(isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowX));
|
|
581
|
+
if (canY) {
|
|
582
|
+
if (el.scrollTop > 0) {
|
|
583
|
+
dirs.push('UP');
|
|
584
|
+
}
|
|
585
|
+
if (el.scrollTop < el.scrollHeight - el.clientHeight) {
|
|
586
|
+
dirs.push('DOWN');
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
if (canX) {
|
|
590
|
+
if (el.scrollLeft > 0) {
|
|
591
|
+
dirs.push('LEFT');
|
|
592
|
+
}
|
|
593
|
+
if (el.scrollLeft < el.scrollWidth - el.clientWidth) {
|
|
594
|
+
dirs.push('RIGHT');
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
return dirs;
|
|
598
|
+
}
|
|
599
|
+
function serialise(el) {
|
|
600
|
+
const deepClone = (el.tagName.toLowerCase() === 'select'
|
|
601
|
+
? el.cloneNode(true)
|
|
602
|
+
: el.cloneNode(false));
|
|
603
|
+
stripDonobuAttrs(deepClone);
|
|
604
|
+
if (el.tagName.toLowerCase() === 'select') {
|
|
605
|
+
const scrollComment = el === document.scrollingElement
|
|
606
|
+
? '<!-- This is the main page scrolling element -->'
|
|
607
|
+
: '';
|
|
608
|
+
return scrollComment + deepClone.outerHTML; // full markup incl. <option>s
|
|
609
|
+
}
|
|
610
|
+
// Get the text content of the original element
|
|
611
|
+
const textContent = el.textContent?.trim() || '';
|
|
612
|
+
if (textContent) {
|
|
613
|
+
// Truncate text if longer than 32 characters
|
|
614
|
+
const displayText = textContent.length > 32
|
|
615
|
+
? textContent.substring(0, 32) + '...'
|
|
616
|
+
: textContent;
|
|
617
|
+
// Return opening tag + text + closing tag
|
|
618
|
+
const fullTag = deepClone.outerHTML;
|
|
619
|
+
const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
|
|
620
|
+
const tagName = el.tagName.toLowerCase();
|
|
621
|
+
const scrollComment = el === document.scrollingElement
|
|
622
|
+
? '<!-- This is the main page scrolling element -->'
|
|
623
|
+
: '';
|
|
624
|
+
return `${scrollComment}${openingTag}${displayText}</${tagName}>`;
|
|
625
|
+
}
|
|
626
|
+
else {
|
|
627
|
+
// opening tag only
|
|
628
|
+
const html = deepClone.outerHTML;
|
|
629
|
+
const scrollComment = el === document.scrollingElement
|
|
630
|
+
? '<!-- This is the main page scrolling element -->'
|
|
631
|
+
: '';
|
|
632
|
+
return scrollComment + html.slice(0, html.indexOf('>') + 1);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
const out = {};
|
|
636
|
+
// Recursively process document and all shadow roots
|
|
637
|
+
const processNode = (root) => {
|
|
638
|
+
// Find elements with the interactable attribute
|
|
639
|
+
root.querySelectorAll(`[${interactableAttr}]`).forEach((el) => {
|
|
640
|
+
const val = el.getAttribute(interactableAttr);
|
|
641
|
+
if (!val) {
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
out[val] = {
|
|
645
|
+
htmlSnippet: serialise(el),
|
|
646
|
+
scrollable: getScrollDirections(el),
|
|
647
|
+
};
|
|
648
|
+
});
|
|
649
|
+
// Recursively process any child shadow roots
|
|
650
|
+
root.querySelectorAll('*').forEach((el) => {
|
|
651
|
+
if (el.shadowRoot) {
|
|
652
|
+
processNode(el.shadowRoot);
|
|
653
|
+
}
|
|
654
|
+
});
|
|
655
|
+
};
|
|
656
|
+
// Start processing from the document root
|
|
657
|
+
processNode(document);
|
|
658
|
+
return out;
|
|
659
|
+
}, this.interactableElementAttribute);
|
|
660
|
+
Object.assign(aggregate, frameMap);
|
|
661
|
+
}
|
|
662
|
+
return Object.keys(aggregate)
|
|
663
|
+
.sort((a, b) => Number(a) - Number(b))
|
|
664
|
+
.map((key) => ({
|
|
665
|
+
donobuAttributeValue: key,
|
|
666
|
+
htmlSnippet: aggregate[key].htmlSnippet,
|
|
667
|
+
scrollable: aggregate[key].scrollable,
|
|
668
|
+
}));
|
|
669
|
+
}
|
|
670
|
+
catch (error) {
|
|
671
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
672
|
+
throw new PageClosedException_1.PageClosedException();
|
|
673
|
+
}
|
|
674
|
+
else {
|
|
675
|
+
throw error;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* Visually annotates all interactable elements with numbered indicators on the page.
|
|
681
|
+
*
|
|
682
|
+
* This method:
|
|
683
|
+
* 1. Processes all accessible frames in the page
|
|
684
|
+
* 2. Creates (or reuses) a shadow DOM container to isolate annotation styling
|
|
685
|
+
* 3. Places circular numbered indicators over each element that has the
|
|
686
|
+
* {@link interactableElementAttribute} attribute
|
|
687
|
+
*
|
|
688
|
+
* The annotations:
|
|
689
|
+
* - Are positioned at the center of each interactable element
|
|
690
|
+
* - Have the same numeric value as the element's attribute
|
|
691
|
+
* - Are styled as black circles with red borders and white text
|
|
692
|
+
* - Are placed in a shadow DOM to avoid style conflicts with the page
|
|
693
|
+
* - Have the {@link interactableAnnotationAttribute} for identification
|
|
694
|
+
* - Are non-interactive (pointer-events: none)
|
|
695
|
+
*
|
|
696
|
+
* Note: This method requires elements to be previously attributed using the
|
|
697
|
+
* {@link attributeInteractableElements} method to find the elements to annotate.
|
|
698
|
+
*
|
|
699
|
+
* @param page - The Playwright Page object to process
|
|
700
|
+
* @returns {Promise<void>} A promise that resolves when all elements have been annotated
|
|
701
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
702
|
+
*
|
|
703
|
+
* @example
|
|
704
|
+
* const inspector = new PageInspector();
|
|
705
|
+
* await inspector.attributeInteractableElements(page);
|
|
706
|
+
* await inspector.annotateInteractableElements(page);
|
|
707
|
+
*/
|
|
708
|
+
async annotateInteractableElements(page) {
|
|
709
|
+
try {
|
|
710
|
+
// Filter frames as needed
|
|
711
|
+
const frames = page
|
|
712
|
+
.frames()
|
|
713
|
+
.filter((frame) => PageInspector.frameFilter(frame));
|
|
714
|
+
for (const frame of frames) {
|
|
715
|
+
await frame.evaluate(([interactableAttr, annotationAttr]) => {
|
|
716
|
+
// 1) Ensure we have a shadow container in the main document
|
|
717
|
+
let container = document.getElementById('annotation-shadow-container');
|
|
718
|
+
if (!container) {
|
|
719
|
+
container = document.createElement('div');
|
|
720
|
+
container.id = 'annotation-shadow-container';
|
|
721
|
+
// Position container so child elements can be absolutely placed
|
|
722
|
+
Object.assign(container.style, {
|
|
723
|
+
position: 'absolute',
|
|
724
|
+
top: '0',
|
|
725
|
+
left: '0',
|
|
726
|
+
width: '100%',
|
|
727
|
+
height: '100%',
|
|
728
|
+
pointerEvents: 'none', // Let clicks pass through
|
|
729
|
+
zIndex: '2147483647', // win every z-index fight
|
|
730
|
+
});
|
|
731
|
+
// Check if document.body exists before trying to append.
|
|
732
|
+
if (document.body) {
|
|
733
|
+
document.body.appendChild(container);
|
|
734
|
+
}
|
|
735
|
+
else if (document.documentElement) {
|
|
736
|
+
// Fall back to document.documentElement if body does not exist.
|
|
737
|
+
document.documentElement.appendChild(container);
|
|
738
|
+
}
|
|
739
|
+
else {
|
|
740
|
+
// If neither exists, we can't proceed with annotations in this frame.
|
|
741
|
+
console.warn(`Cannot create annotation container for ${window.location.href} since the document structure is incomplete`);
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
// Attach a shadow root
|
|
745
|
+
const shadowRoot = container.attachShadow({ mode: 'open' });
|
|
746
|
+
// Add a <style> element inside the shadow root to reset and define annotation styles
|
|
747
|
+
const style = document.createElement('style');
|
|
748
|
+
style.textContent = `
|
|
749
|
+
:host {
|
|
750
|
+
all: initial; /* Reset styles in shadow root */
|
|
751
|
+
}
|
|
752
|
+
.annotation {
|
|
753
|
+
position: absolute;
|
|
754
|
+
z-index: 2147483647;
|
|
755
|
+
background-color: black;
|
|
756
|
+
color: white;
|
|
757
|
+
width: 40px;
|
|
758
|
+
height: 40px;
|
|
759
|
+
border-radius: 50%;
|
|
760
|
+
display: flex;
|
|
761
|
+
align-items: center;
|
|
762
|
+
justify-content: center;
|
|
763
|
+
font-size: 14px;
|
|
764
|
+
font-weight: bold;
|
|
765
|
+
line-height: 20px;
|
|
766
|
+
text-align: center;
|
|
767
|
+
box-shadow: 0px 2px 4px rgba(0,0,0,0.2);
|
|
768
|
+
border: 4px solid #FF4136;
|
|
769
|
+
pointer-events: none;
|
|
770
|
+
}
|
|
771
|
+
`;
|
|
772
|
+
shadowRoot.appendChild(style);
|
|
773
|
+
}
|
|
774
|
+
// Retrieve the shadow root to place annotation elements
|
|
775
|
+
const containerEl = document.getElementById('annotation-shadow-container');
|
|
776
|
+
if (!containerEl?.shadowRoot) {
|
|
777
|
+
return;
|
|
778
|
+
}
|
|
779
|
+
const shadowRoot = containerEl.shadowRoot;
|
|
780
|
+
// 2) Factory to create a new annotation inside the shadow root
|
|
781
|
+
const createAnnotation = (value) => {
|
|
782
|
+
const annotation = document.createElement('div');
|
|
783
|
+
annotation.classList.add('annotation');
|
|
784
|
+
annotation.dataset[annotationAttr] = '1';
|
|
785
|
+
annotation.textContent = value;
|
|
786
|
+
return annotation;
|
|
787
|
+
};
|
|
788
|
+
// 3) Position annotation relative to an element
|
|
789
|
+
const positionAnnotation = (annotation, element) => {
|
|
790
|
+
const rect = element.getBoundingClientRect();
|
|
791
|
+
// Center the annotation on the element, adjusting for its size
|
|
792
|
+
// Since container is absolute, we need to account for scroll position
|
|
793
|
+
const x = rect.left + rect.width / 2 - 20 + window.scrollX;
|
|
794
|
+
const y = rect.top + rect.height / 2 - 20 + window.scrollY;
|
|
795
|
+
annotation.style.left = `${x}px`;
|
|
796
|
+
annotation.style.top = `${y}px`;
|
|
797
|
+
};
|
|
798
|
+
// 4) Traverse DOM (including any nested shadow roots) to find interactable elements
|
|
799
|
+
const processNode = (root) => {
|
|
800
|
+
// Find elements with the interactable attribute
|
|
801
|
+
const elements = root.querySelectorAll(`[${interactableAttr}]`);
|
|
802
|
+
elements.forEach((element) => {
|
|
803
|
+
const value = element.getAttribute(interactableAttr);
|
|
804
|
+
if (value) {
|
|
805
|
+
const annotation = createAnnotation(value);
|
|
806
|
+
shadowRoot.appendChild(annotation);
|
|
807
|
+
positionAnnotation(annotation, element);
|
|
808
|
+
}
|
|
809
|
+
});
|
|
810
|
+
// Recursively process any child shadow roots
|
|
811
|
+
root.querySelectorAll('*').forEach((el) => {
|
|
812
|
+
if (el.shadowRoot) {
|
|
813
|
+
processNode(el.shadowRoot);
|
|
814
|
+
}
|
|
815
|
+
});
|
|
816
|
+
};
|
|
817
|
+
// Start processing from the (frame) document root
|
|
818
|
+
processNode(document);
|
|
819
|
+
}, [
|
|
820
|
+
this.interactableElementAttribute,
|
|
821
|
+
PageInspector.convertToJsAttribute(this.interactableAnnotationAttribute),
|
|
822
|
+
]);
|
|
823
|
+
}
|
|
935
824
|
}
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
* the element (or, via the <label htmlFor> mapping, its associated control)
|
|
940
|
-
* was attributed.
|
|
941
|
-
*/
|
|
942
|
-
function tryAttributeElement(element) {
|
|
943
|
-
if (element.hasAttribute(interactableAttribute)) {
|
|
944
|
-
return false;
|
|
825
|
+
catch (error) {
|
|
826
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
827
|
+
throw new PageClosedException_1.PageClosedException();
|
|
945
828
|
}
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
|
|
949
|
-
const enabled = isElementEnabled(element, style);
|
|
950
|
-
if (!visible || !enabled) {
|
|
951
|
-
return false;
|
|
829
|
+
else {
|
|
830
|
+
throw error;
|
|
952
831
|
}
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
/**
|
|
835
|
+
* Removes all visual annotations from the page that were created by
|
|
836
|
+
* the {@link annotateInteractableElements} method.
|
|
837
|
+
*
|
|
838
|
+
* This method:
|
|
839
|
+
* 1. Processes all accessible frames in the page
|
|
840
|
+
* 2. Finds and removes the shadow DOM container with ID 'annotation-shadow-container'
|
|
841
|
+
* that contains all the annotations
|
|
842
|
+
*
|
|
843
|
+
* This effectively removes all numbered indicators that were previously placed
|
|
844
|
+
* over interactable elements, leaving the page in its original visual state.
|
|
845
|
+
* Note that this only removes the visual annotations, not the
|
|
846
|
+
* {@link interactableElementAttribute} attributes on the elements themselves.
|
|
847
|
+
*
|
|
848
|
+
* @param page - The Playwright Page object to process
|
|
849
|
+
* @returns {Promise<void>} A promise that resolves when all annotations have been removed
|
|
850
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
851
|
+
*
|
|
852
|
+
* @example
|
|
853
|
+
* const inspector = new PageInspector();
|
|
854
|
+
* await inspector.attributeInteractableElements(page);
|
|
855
|
+
* await inspector.annotateInteractableElements(page);
|
|
856
|
+
* // ... do some operations with the annotations visible ...
|
|
857
|
+
* await inspector.removeDonobuAnnotations(page);
|
|
858
|
+
* // All visual annotations are now removed from the page
|
|
859
|
+
*/
|
|
860
|
+
async removeDonobuAnnotations(page) {
|
|
861
|
+
try {
|
|
862
|
+
const frames = page
|
|
863
|
+
.frames()
|
|
864
|
+
.filter((frame) => PageInspector.frameFilter(frame));
|
|
865
|
+
for (const frame of frames) {
|
|
866
|
+
await frame.evaluate(() => {
|
|
867
|
+
const containerId = 'annotation-shadow-container';
|
|
868
|
+
const container = document.getElementById(containerId);
|
|
869
|
+
if (container) {
|
|
870
|
+
container.remove();
|
|
974
871
|
}
|
|
975
|
-
|
|
976
|
-
}
|
|
872
|
+
});
|
|
977
873
|
}
|
|
978
|
-
return false;
|
|
979
874
|
}
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
// Special-case: always keep the root scrolling element
|
|
984
|
-
element.setAttribute(interactableAttribute, offset.toString());
|
|
985
|
-
offset++;
|
|
986
|
-
return; // skip the usual checks
|
|
875
|
+
catch (error) {
|
|
876
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
877
|
+
throw new PageClosedException_1.PageClosedException();
|
|
987
878
|
}
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
879
|
+
throw error;
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
/**
|
|
883
|
+
* Removes all interactable element attributes that were previously added to elements in the page.
|
|
884
|
+
*
|
|
885
|
+
* This method:
|
|
886
|
+
* 1. Processes all accessible frames in the page
|
|
887
|
+
* 2. Finds all elements with the {@link interactableElementAttribute} attribute
|
|
888
|
+
* 3. Removes this attribute from each element
|
|
889
|
+
*
|
|
890
|
+
* This effectively undoes the changes made by the {@link attributeInteractableElements} method,
|
|
891
|
+
* returning the page's DOM to its original state without the custom attributes.
|
|
892
|
+
* Note that this does not affect any visual annotations - to remove those, use
|
|
893
|
+
* the {@link removeDonobuAnnotations} method separately.
|
|
894
|
+
*
|
|
895
|
+
* This method is automatically called at the beginning of {@link attributeInteractableElements}
|
|
896
|
+
* to ensure a clean state before adding new attributes, but can also be called
|
|
897
|
+
* independently to clean up the DOM.
|
|
898
|
+
*
|
|
899
|
+
* @param page - The Playwright Page object to process
|
|
900
|
+
* @returns {Promise<void>} A promise that resolves when all attributes have been removed
|
|
901
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
902
|
+
*
|
|
903
|
+
* @example
|
|
904
|
+
* const inspector = new PageInspector();
|
|
905
|
+
* await inspector.attributeInteractableElements(page);
|
|
906
|
+
* // ... perform operations with attributed elements ...
|
|
907
|
+
* await inspector.deattributeInteractableElements(page);
|
|
908
|
+
* // All interactable element attributes are now removed from the page
|
|
909
|
+
*/
|
|
910
|
+
async deattributeInteractableElements(page) {
|
|
911
|
+
try {
|
|
912
|
+
const frames = page.frames().filter(PageInspector.frameFilter);
|
|
913
|
+
const attr = this.interactableElementAttribute;
|
|
914
|
+
for (const frame of frames) {
|
|
915
|
+
await frame.evaluate(([a]) => {
|
|
916
|
+
/** Depth-first removal inside document & every shadow root */
|
|
917
|
+
const removeDeep = (root) => {
|
|
918
|
+
root
|
|
919
|
+
.querySelectorAll(`[${a}]`)
|
|
920
|
+
.forEach((el) => el.removeAttribute(a));
|
|
921
|
+
root.querySelectorAll('*').forEach((el) => {
|
|
922
|
+
const sr = el.shadowRoot;
|
|
923
|
+
if (sr) {
|
|
924
|
+
removeDeep(sr);
|
|
925
|
+
}
|
|
926
|
+
});
|
|
927
|
+
};
|
|
928
|
+
removeDeep(document);
|
|
929
|
+
}, [attr]);
|
|
991
930
|
}
|
|
992
|
-
|
|
993
|
-
|
|
931
|
+
}
|
|
932
|
+
catch (error) {
|
|
933
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
934
|
+
throw new PageClosedException_1.PageClosedException();
|
|
994
935
|
}
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
// This is the standard pattern for Ant Design Segmented/Radio/Checkbox/
|
|
998
|
-
// Switch and many other component libraries: the native input is hidden
|
|
999
|
-
// and the surrounding <label> is the real clickable surface. Since the
|
|
1000
|
-
// hidden control fails the visibility/enabled checks above, attribute the
|
|
1001
|
-
// wrapping <label> instead so the toggle is still annotated.
|
|
1002
|
-
const wrappingLabel = element.closest('label');
|
|
1003
|
-
if (wrappingLabel &&
|
|
1004
|
-
wrappingLabel !== element &&
|
|
1005
|
-
!wrappingLabel.hasAttribute(interactableAttribute)) {
|
|
1006
|
-
tryAttributeElement(wrappingLabel);
|
|
936
|
+
else {
|
|
937
|
+
throw error;
|
|
1007
938
|
}
|
|
1008
|
-
}
|
|
1009
|
-
return offset;
|
|
939
|
+
}
|
|
1010
940
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
941
|
+
/**
|
|
942
|
+
* Retrieves the HTML snippet for a single element.
|
|
943
|
+
*
|
|
944
|
+
* This method:
|
|
945
|
+
* 1. Extracts a simplified HTML snippet representation of the element
|
|
946
|
+
* * For 'select' elements, the complete HTML (including options) is preserved
|
|
947
|
+
* * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
|
|
948
|
+
* * For all other elements, only the opening tag without children is captured
|
|
949
|
+
* 2. Strips any Donobu-specific attributes from the snippet
|
|
950
|
+
*
|
|
951
|
+
* @example
|
|
952
|
+
* const inspector = new PageInspector();
|
|
953
|
+
* const submitButton = page.querySelector('button[type="submit"]');
|
|
954
|
+
* const htmlSnippet = await inspector.getHtmlSnippet(submitButton);
|
|
955
|
+
* // htmlSnippet = "<button type=\"submit\">Submit</button>"
|
|
956
|
+
*/
|
|
957
|
+
async getHtmlSnippet(elementHandle) {
|
|
958
|
+
try {
|
|
959
|
+
// Evaluate in the element's context to get the HTML snippet
|
|
960
|
+
const htmlSnippet = await elementHandle.evaluate((element) => {
|
|
961
|
+
// Helper function to strip Donobu attributes
|
|
962
|
+
function stripDonobuAttrs(el) {
|
|
963
|
+
const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
|
|
964
|
+
let node = el;
|
|
965
|
+
while (node) {
|
|
966
|
+
Array.from(node.attributes).forEach((attr) => {
|
|
967
|
+
// Strip out Donobu attributes since those are not a part of the
|
|
968
|
+
// original HTML.
|
|
969
|
+
if (attr.name.startsWith('data-donobu')) {
|
|
970
|
+
node.removeAttribute(attr.name);
|
|
971
|
+
}
|
|
972
|
+
});
|
|
973
|
+
node = walker.nextNode();
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
// Helper function to serialize element
|
|
977
|
+
function serialise(el) {
|
|
978
|
+
const deepClone = el.tagName.toLowerCase() === 'select'
|
|
979
|
+
? el.cloneNode(true)
|
|
980
|
+
: el.cloneNode(false);
|
|
981
|
+
stripDonobuAttrs(deepClone);
|
|
982
|
+
if (el.tagName.toLowerCase() === 'select') {
|
|
983
|
+
return deepClone.outerHTML; // full markup incl. <option>s
|
|
984
|
+
}
|
|
985
|
+
// Get the text content of the original element
|
|
986
|
+
const textContent = el.textContent?.trim() || '';
|
|
987
|
+
if (textContent) {
|
|
988
|
+
// Truncate text if longer than 32 characters
|
|
989
|
+
const displayText = textContent.length > 32
|
|
990
|
+
? textContent.substring(0, 32) + '...'
|
|
991
|
+
: textContent;
|
|
992
|
+
// Return opening tag + text + closing tag
|
|
993
|
+
const fullTag = deepClone.outerHTML;
|
|
994
|
+
const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
|
|
995
|
+
const tagName = el.tagName.toLowerCase();
|
|
996
|
+
return `${openingTag}${displayText}</${tagName}>`;
|
|
997
|
+
}
|
|
998
|
+
else {
|
|
999
|
+
// opening tag only
|
|
1000
|
+
const html = deepClone.outerHTML;
|
|
1001
|
+
return html.slice(0, html.indexOf('>') + 1);
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
return serialise(element);
|
|
1005
|
+
});
|
|
1006
|
+
return htmlSnippet;
|
|
1007
|
+
}
|
|
1008
|
+
catch (error) {
|
|
1009
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
1010
|
+
throw new PageClosedException_1.PageClosedException();
|
|
1011
|
+
}
|
|
1012
|
+
else {
|
|
1013
|
+
throw error;
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
1016
|
}
|
|
1017
1017
|
}
|
|
1018
1018
|
exports.PageInspector = PageInspector;
|