donobu 2.15.1 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/control-panel.js +30 -3
- package/dist/assets/generated/parameter-schemas.json +33 -33
- package/dist/assets/generated/version +1 -1
- package/dist/assets/interactive-elements-tracker.js +351 -0
- package/dist/assets/page-interactions-tracker.js +39 -20
- package/dist/assets/smart-selector-generator.js +38 -15
- package/dist/bindings/FocusPage.d.ts +6 -1
- package/dist/bindings/FocusPage.d.ts.map +1 -1
- package/dist/bindings/FocusPage.js +2 -8
- package/dist/bindings/FocusPage.js.map +1 -1
- package/dist/bindings/PageInteractionTracker.d.ts +9 -2
- package/dist/bindings/PageInteractionTracker.d.ts.map +1 -1
- package/dist/bindings/PageInteractionTracker.js +27 -3
- package/dist/bindings/PageInteractionTracker.js.map +1 -1
- package/dist/bindings/ProposeToolCall.d.ts +20 -0
- package/dist/bindings/ProposeToolCall.d.ts.map +1 -0
- package/dist/bindings/ProposeToolCall.js +29 -0
- package/dist/bindings/ProposeToolCall.js.map +1 -0
- package/dist/bindings/ToggleDonobuAnnotations.d.ts +5 -2
- package/dist/bindings/ToggleDonobuAnnotations.d.ts.map +1 -1
- package/dist/bindings/ToggleDonobuAnnotations.js +12 -6
- package/dist/bindings/ToggleDonobuAnnotations.js.map +1 -1
- package/dist/esm/assets/control-panel.js +30 -3
- package/dist/esm/assets/generated/parameter-schemas.json +33 -33
- package/dist/esm/assets/generated/version +1 -1
- package/dist/esm/assets/interactive-elements-tracker.js +351 -0
- package/dist/esm/assets/page-interactions-tracker.js +39 -20
- package/dist/esm/assets/smart-selector-generator.js +38 -15
- package/dist/esm/bindings/FocusPage.d.ts +6 -1
- package/dist/esm/bindings/FocusPage.d.ts.map +1 -1
- package/dist/esm/bindings/FocusPage.js +2 -8
- package/dist/esm/bindings/FocusPage.js.map +1 -1
- package/dist/esm/bindings/PageInteractionTracker.d.ts +9 -2
- package/dist/esm/bindings/PageInteractionTracker.d.ts.map +1 -1
- package/dist/esm/bindings/PageInteractionTracker.js +27 -3
- package/dist/esm/bindings/PageInteractionTracker.js.map +1 -1
- package/dist/esm/bindings/ProposeToolCall.d.ts +20 -0
- package/dist/esm/bindings/ProposeToolCall.d.ts.map +1 -0
- package/dist/esm/bindings/ProposeToolCall.js +29 -0
- package/dist/esm/bindings/ProposeToolCall.js.map +1 -0
- package/dist/esm/bindings/ToggleDonobuAnnotations.d.ts +5 -2
- package/dist/esm/bindings/ToggleDonobuAnnotations.d.ts.map +1 -1
- package/dist/esm/bindings/ToggleDonobuAnnotations.js +12 -6
- package/dist/esm/bindings/ToggleDonobuAnnotations.js.map +1 -1
- package/dist/esm/managers/CodeGenerator.d.ts.map +1 -1
- package/dist/esm/managers/CodeGenerator.js +2 -1
- package/dist/esm/managers/CodeGenerator.js.map +1 -1
- package/dist/esm/managers/ControlPanel.d.ts +124 -0
- package/dist/esm/managers/ControlPanel.d.ts.map +1 -0
- package/dist/esm/managers/ControlPanel.js +282 -0
- package/dist/esm/managers/ControlPanel.js.map +1 -0
- package/dist/esm/managers/DonobuFlow.d.ts +4 -12
- package/dist/esm/managers/DonobuFlow.d.ts.map +1 -1
- package/dist/esm/managers/DonobuFlow.js +35 -140
- package/dist/esm/managers/DonobuFlow.js.map +1 -1
- package/dist/esm/managers/InteractionVisualizer.d.ts +9 -15
- package/dist/esm/managers/InteractionVisualizer.d.ts.map +1 -1
- package/dist/esm/managers/InteractionVisualizer.js +44 -44
- package/dist/esm/managers/InteractionVisualizer.js.map +1 -1
- package/dist/esm/managers/PageInspector.d.ts +242 -0
- package/dist/esm/managers/PageInspector.d.ts.map +1 -0
- package/dist/esm/managers/PageInspector.js +783 -0
- package/dist/esm/managers/PageInspector.js.map +1 -0
- package/dist/esm/managers/ToolManager.d.ts.map +1 -1
- package/dist/esm/managers/ToolManager.js +3 -5
- package/dist/esm/managers/ToolManager.js.map +1 -1
- package/dist/esm/models/InteractableElement.d.ts +2 -2
- package/dist/esm/models/ToolCallContext.d.ts +6 -2
- package/dist/esm/models/ToolCallContext.d.ts.map +1 -1
- package/dist/esm/playwrightTestExtensions.d.ts +5 -0
- package/dist/esm/playwrightTestExtensions.d.ts.map +1 -1
- package/dist/esm/playwrightTestExtensions.js +52 -13
- package/dist/esm/playwrightTestExtensions.js.map +1 -1
- package/dist/esm/tools/AnalyzePageTextTool.js +1 -1
- package/dist/esm/tools/AnalyzePageTextTool.js.map +1 -1
- package/dist/esm/tools/AssertPageTextTool.js +1 -1
- package/dist/esm/tools/AssertPageTextTool.js.map +1 -1
- package/dist/esm/tools/AssertTool.d.ts.map +1 -1
- package/dist/esm/tools/AssertTool.js +4 -4
- package/dist/esm/tools/AssertTool.js.map +1 -1
- package/dist/esm/tools/ChangeWebBrowserTabTool.d.ts.map +1 -1
- package/dist/esm/tools/ChangeWebBrowserTabTool.js +8 -7
- package/dist/esm/tools/ChangeWebBrowserTabTool.js.map +1 -1
- package/dist/esm/tools/ClickTool.js +1 -1
- package/dist/esm/tools/ClickTool.js.map +1 -1
- package/dist/esm/tools/CreateBrowserCookieReportTool.js +1 -1
- package/dist/esm/tools/CreateBrowserCookieReportTool.js.map +1 -1
- package/dist/esm/tools/CustomToolRunnerTool.d.ts.map +1 -1
- package/dist/esm/tools/CustomToolRunnerTool.js +1 -1
- package/dist/esm/tools/CustomToolRunnerTool.js.map +1 -1
- package/dist/esm/tools/DetectBrokenLinksTool.js +1 -1
- package/dist/esm/tools/DetectBrokenLinksTool.js.map +1 -1
- package/dist/esm/tools/ExtractGoogleStreetviewEntityDataTool.js +1 -1
- package/dist/esm/tools/ExtractGoogleStreetviewEntityDataTool.js.map +1 -1
- package/dist/esm/tools/ExtractPaymentProviderKeyTool.js +1 -1
- package/dist/esm/tools/ExtractPaymentProviderKeyTool.js.map +1 -1
- package/dist/esm/tools/ExtractPublicFacebookEntityDataTool.js +1 -1
- package/dist/esm/tools/ExtractPublicFacebookEntityDataTool.js.map +1 -1
- package/dist/esm/tools/GoForwardOrBackTool.js +1 -1
- package/dist/esm/tools/GoForwardOrBackTool.js.map +1 -1
- package/dist/esm/tools/GoToGoogleMapsStreetViewTool.js +1 -1
- package/dist/esm/tools/GoToGoogleMapsStreetViewTool.js.map +1 -1
- package/dist/esm/tools/GoToWebpageTool.js +1 -1
- package/dist/esm/tools/GoToWebpageTool.js.map +1 -1
- package/dist/esm/tools/HoverOverElementTool.d.ts.map +1 -1
- package/dist/esm/tools/HoverOverElementTool.js +1 -1
- package/dist/esm/tools/HoverOverElementTool.js.map +1 -1
- package/dist/esm/tools/NavigateWithinStreetView.js +1 -1
- package/dist/esm/tools/NavigateWithinStreetView.js.map +1 -1
- package/dist/esm/tools/PressKeyTool.js +1 -1
- package/dist/esm/tools/PressKeyTool.js.map +1 -1
- package/dist/esm/tools/ReloadPageTool.js +1 -1
- package/dist/esm/tools/ReloadPageTool.js.map +1 -1
- package/dist/esm/tools/ReplayableInteraction.d.ts +16 -1
- package/dist/esm/tools/ReplayableInteraction.d.ts.map +1 -1
- package/dist/esm/tools/ReplayableInteraction.js +169 -16
- package/dist/esm/tools/ReplayableInteraction.js.map +1 -1
- package/dist/esm/tools/RunAccessibilityTestTool.d.ts +8 -0
- package/dist/esm/tools/RunAccessibilityTestTool.d.ts.map +1 -1
- package/dist/esm/tools/RunAccessibilityTestTool.js +15 -2
- package/dist/esm/tools/RunAccessibilityTestTool.js.map +1 -1
- package/dist/esm/tools/RunInlineJavaScriptCodeTool.d.ts.map +1 -1
- package/dist/esm/tools/RunInlineJavaScriptCodeTool.js +1 -1
- package/dist/esm/tools/RunInlineJavaScriptCodeTool.js.map +1 -1
- package/dist/esm/tools/SaveWebpageAsPdfTool.js +1 -1
- package/dist/esm/tools/SaveWebpageAsPdfTool.js.map +1 -1
- package/dist/esm/tools/ScrollPageTool.d.ts +1 -1
- package/dist/esm/tools/ScrollPageTool.d.ts.map +1 -1
- package/dist/esm/tools/ScrollPageTool.js +47 -4
- package/dist/esm/tools/ScrollPageTool.js.map +1 -1
- package/dist/esm/tools/WaitTool.js +1 -1
- package/dist/esm/tools/WaitTool.js.map +1 -1
- package/dist/esm/utils/MiscUtils.d.ts +2 -2
- package/dist/esm/utils/MiscUtils.js +2 -2
- package/dist/esm/utils/PlaywrightUtils.d.ts +26 -127
- package/dist/esm/utils/PlaywrightUtils.d.ts.map +1 -1
- package/dist/esm/utils/PlaywrightUtils.js +133 -935
- package/dist/esm/utils/PlaywrightUtils.js.map +1 -1
- package/dist/managers/CodeGenerator.d.ts.map +1 -1
- package/dist/managers/CodeGenerator.js +2 -1
- package/dist/managers/CodeGenerator.js.map +1 -1
- package/dist/managers/ControlPanel.d.ts +124 -0
- package/dist/managers/ControlPanel.d.ts.map +1 -0
- package/dist/managers/ControlPanel.js +282 -0
- package/dist/managers/ControlPanel.js.map +1 -0
- package/dist/managers/DonobuFlow.d.ts +4 -12
- package/dist/managers/DonobuFlow.d.ts.map +1 -1
- package/dist/managers/DonobuFlow.js +35 -140
- package/dist/managers/DonobuFlow.js.map +1 -1
- package/dist/managers/InteractionVisualizer.d.ts +9 -15
- package/dist/managers/InteractionVisualizer.d.ts.map +1 -1
- package/dist/managers/InteractionVisualizer.js +44 -44
- package/dist/managers/InteractionVisualizer.js.map +1 -1
- package/dist/managers/PageInspector.d.ts +242 -0
- package/dist/managers/PageInspector.d.ts.map +1 -0
- package/dist/managers/PageInspector.js +783 -0
- package/dist/managers/PageInspector.js.map +1 -0
- package/dist/managers/ToolManager.d.ts.map +1 -1
- package/dist/managers/ToolManager.js +3 -5
- package/dist/managers/ToolManager.js.map +1 -1
- package/dist/models/InteractableElement.d.ts +2 -2
- package/dist/models/ToolCallContext.d.ts +6 -2
- package/dist/models/ToolCallContext.d.ts.map +1 -1
- package/dist/playwrightTestExtensions.d.ts +5 -0
- package/dist/playwrightTestExtensions.d.ts.map +1 -1
- package/dist/playwrightTestExtensions.js +52 -13
- package/dist/playwrightTestExtensions.js.map +1 -1
- package/dist/tools/AnalyzePageTextTool.js +1 -1
- package/dist/tools/AnalyzePageTextTool.js.map +1 -1
- package/dist/tools/AssertPageTextTool.js +1 -1
- package/dist/tools/AssertPageTextTool.js.map +1 -1
- package/dist/tools/AssertTool.d.ts.map +1 -1
- package/dist/tools/AssertTool.js +4 -4
- package/dist/tools/AssertTool.js.map +1 -1
- package/dist/tools/ChangeWebBrowserTabTool.d.ts.map +1 -1
- package/dist/tools/ChangeWebBrowserTabTool.js +8 -7
- package/dist/tools/ChangeWebBrowserTabTool.js.map +1 -1
- package/dist/tools/ClickTool.js +1 -1
- package/dist/tools/ClickTool.js.map +1 -1
- package/dist/tools/CreateBrowserCookieReportTool.js +1 -1
- package/dist/tools/CreateBrowserCookieReportTool.js.map +1 -1
- package/dist/tools/CustomToolRunnerTool.d.ts.map +1 -1
- package/dist/tools/CustomToolRunnerTool.js +1 -1
- package/dist/tools/CustomToolRunnerTool.js.map +1 -1
- package/dist/tools/DetectBrokenLinksTool.js +1 -1
- package/dist/tools/DetectBrokenLinksTool.js.map +1 -1
- package/dist/tools/ExtractGoogleStreetviewEntityDataTool.js +1 -1
- package/dist/tools/ExtractGoogleStreetviewEntityDataTool.js.map +1 -1
- package/dist/tools/ExtractPaymentProviderKeyTool.js +1 -1
- package/dist/tools/ExtractPaymentProviderKeyTool.js.map +1 -1
- package/dist/tools/ExtractPublicFacebookEntityDataTool.js +1 -1
- package/dist/tools/ExtractPublicFacebookEntityDataTool.js.map +1 -1
- package/dist/tools/GoForwardOrBackTool.js +1 -1
- package/dist/tools/GoForwardOrBackTool.js.map +1 -1
- package/dist/tools/GoToGoogleMapsStreetViewTool.js +1 -1
- package/dist/tools/GoToGoogleMapsStreetViewTool.js.map +1 -1
- package/dist/tools/GoToWebpageTool.js +1 -1
- package/dist/tools/GoToWebpageTool.js.map +1 -1
- package/dist/tools/HoverOverElementTool.d.ts.map +1 -1
- package/dist/tools/HoverOverElementTool.js +1 -1
- package/dist/tools/HoverOverElementTool.js.map +1 -1
- package/dist/tools/NavigateWithinStreetView.js +1 -1
- package/dist/tools/NavigateWithinStreetView.js.map +1 -1
- package/dist/tools/PressKeyTool.js +1 -1
- package/dist/tools/PressKeyTool.js.map +1 -1
- package/dist/tools/ReloadPageTool.js +1 -1
- package/dist/tools/ReloadPageTool.js.map +1 -1
- package/dist/tools/ReplayableInteraction.d.ts +16 -1
- package/dist/tools/ReplayableInteraction.d.ts.map +1 -1
- package/dist/tools/ReplayableInteraction.js +169 -16
- package/dist/tools/ReplayableInteraction.js.map +1 -1
- package/dist/tools/RunAccessibilityTestTool.d.ts +8 -0
- package/dist/tools/RunAccessibilityTestTool.d.ts.map +1 -1
- package/dist/tools/RunAccessibilityTestTool.js +15 -2
- package/dist/tools/RunAccessibilityTestTool.js.map +1 -1
- package/dist/tools/RunInlineJavaScriptCodeTool.d.ts.map +1 -1
- package/dist/tools/RunInlineJavaScriptCodeTool.js +1 -1
- package/dist/tools/RunInlineJavaScriptCodeTool.js.map +1 -1
- package/dist/tools/SaveWebpageAsPdfTool.js +1 -1
- package/dist/tools/SaveWebpageAsPdfTool.js.map +1 -1
- package/dist/tools/ScrollPageTool.d.ts +1 -1
- package/dist/tools/ScrollPageTool.d.ts.map +1 -1
- package/dist/tools/ScrollPageTool.js +47 -4
- package/dist/tools/ScrollPageTool.js.map +1 -1
- package/dist/tools/WaitTool.js +1 -1
- package/dist/tools/WaitTool.js.map +1 -1
- package/dist/utils/MiscUtils.d.ts +2 -2
- package/dist/utils/MiscUtils.js +2 -2
- package/dist/utils/PlaywrightUtils.d.ts +26 -127
- package/dist/utils/PlaywrightUtils.d.ts.map +1 -1
- package/dist/utils/PlaywrightUtils.js +133 -935
- package/dist/utils/PlaywrightUtils.js.map +1 -1
- package/package.json +1 -1
- package/dist/assets/clickable-elements-tracker.js +0 -63
- package/dist/assets/zaru-eyes.html +0 -141
- package/dist/esm/assets/clickable-elements-tracker.js +0 -63
- package/dist/esm/assets/zaru-eyes.html +0 -141
- package/dist/esm/managers/TemplateInterpolator.d.ts +0 -19
- package/dist/esm/managers/TemplateInterpolator.d.ts.map +0 -1
- package/dist/esm/managers/TemplateInterpolator.js +0 -86
- package/dist/esm/managers/TemplateInterpolator.js.map +0 -1
- package/dist/esm/managers/ToolTipper.d.ts +0 -32
- package/dist/esm/managers/ToolTipper.d.ts.map +0 -1
- package/dist/esm/managers/ToolTipper.js +0 -229
- package/dist/esm/managers/ToolTipper.js.map +0 -1
- package/dist/esm/models/BrowserState.d.ts +0 -26
- package/dist/esm/models/BrowserState.d.ts.map +0 -1
- package/dist/esm/models/BrowserState.js +0 -3
- package/dist/esm/models/BrowserState.js.map +0 -1
- package/dist/esm/models/ResolverContext.d.ts +0 -7
- package/dist/esm/models/ResolverContext.d.ts.map +0 -1
- package/dist/esm/models/ResolverContext.js +0 -3
- package/dist/esm/models/ResolverContext.js.map +0 -1
- package/dist/esm/models/ToolTemplateDataSource.d.ts +0 -36
- package/dist/esm/models/ToolTemplateDataSource.d.ts.map +0 -1
- package/dist/esm/models/ToolTemplateDataSource.js +0 -3
- package/dist/esm/models/ToolTemplateDataSource.js.map +0 -1
- package/dist/esm/test.d.ts +0 -78
- package/dist/esm/test.d.ts.map +0 -1
- package/dist/esm/test.js +0 -423
- package/dist/esm/test.js.map +0 -1
- package/dist/esm/utils/TemplateInterpolator.d.ts +0 -29
- package/dist/esm/utils/TemplateInterpolator.d.ts.map +0 -1
- package/dist/esm/utils/TemplateInterpolator.js +0 -206
- package/dist/esm/utils/TemplateInterpolator.js.map +0 -1
- package/dist/managers/TemplateInterpolator.d.ts +0 -19
- package/dist/managers/TemplateInterpolator.d.ts.map +0 -1
- package/dist/managers/TemplateInterpolator.js +0 -86
- package/dist/managers/TemplateInterpolator.js.map +0 -1
- package/dist/managers/ToolTipper.d.ts +0 -32
- package/dist/managers/ToolTipper.d.ts.map +0 -1
- package/dist/managers/ToolTipper.js +0 -229
- package/dist/managers/ToolTipper.js.map +0 -1
- package/dist/models/BrowserState.d.ts +0 -26
- package/dist/models/BrowserState.d.ts.map +0 -1
- package/dist/models/BrowserState.js +0 -3
- package/dist/models/BrowserState.js.map +0 -1
- package/dist/models/ResolverContext.d.ts +0 -7
- package/dist/models/ResolverContext.d.ts.map +0 -1
- package/dist/models/ResolverContext.js +0 -3
- package/dist/models/ResolverContext.js.map +0 -1
- package/dist/models/ToolTemplateDataSource.d.ts +0 -36
- package/dist/models/ToolTemplateDataSource.d.ts.map +0 -1
- package/dist/models/ToolTemplateDataSource.js +0 -3
- package/dist/models/ToolTemplateDataSource.js.map +0 -1
- package/dist/test.d.ts +0 -78
- package/dist/test.d.ts.map +0 -1
- package/dist/test.js +0 -423
- package/dist/test.js.map +0 -1
- package/dist/tsconfig.esm.tsbuildinfo +0 -1
- package/dist/utils/TemplateInterpolator.d.ts +0 -29
- package/dist/utils/TemplateInterpolator.d.ts.map +0 -1
- package/dist/utils/TemplateInterpolator.js +0 -211
- package/dist/utils/TemplateInterpolator.js.map +0 -1
|
@@ -3,12 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.InteractionVisualizer = void 0;
|
|
4
4
|
const Logger_1 = require("../utils/Logger");
|
|
5
5
|
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
|
|
6
|
+
const MiscUtils_1 = require("../utils/MiscUtils");
|
|
6
7
|
/**
|
|
7
8
|
* Class for visualizing interactions in a webpage with a virtual mouse and messages.
|
|
8
|
-
*
|
|
9
|
-
* This class assumes that the /donobu.css route exists and defines the
|
|
10
|
-
* `.donobu-message` and `.donobu-virtual-mouse` CSS classes (e.g. being served up by
|
|
11
|
-
* a virtual route registered with the browser context).
|
|
12
9
|
*/
|
|
13
10
|
class InteractionVisualizer {
|
|
14
11
|
/**
|
|
@@ -18,17 +15,16 @@ class InteractionVisualizer {
|
|
|
18
15
|
constructor(defaultMessageDurationMillis) {
|
|
19
16
|
this.defaultMessageDurationMillis = defaultMessageDurationMillis;
|
|
20
17
|
this.currentMousePosition = { x: 0, y: 0 };
|
|
21
|
-
this.isMouseVisible = false;
|
|
22
18
|
}
|
|
23
19
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* Blocks execution for the duration.
|
|
20
|
+
* Moves the virtual mouse to the element with a smooth animation and
|
|
21
|
+
* optionally displays a message at element.
|
|
27
22
|
*/
|
|
28
23
|
async pointAt(page, element, message, durationMillis = this.defaultMessageDurationMillis) {
|
|
29
|
-
if (!
|
|
24
|
+
if (!durationMillis || durationMillis <= 0) {
|
|
30
25
|
return;
|
|
31
26
|
}
|
|
27
|
+
await InteractionVisualizer.ensureRoutesRegistered(page.context());
|
|
32
28
|
try {
|
|
33
29
|
// Get element position for mouse movement
|
|
34
30
|
const boundingBox = await element.boundingBox();
|
|
@@ -39,14 +35,17 @@ class InteractionVisualizer {
|
|
|
39
35
|
// Calculate target position (center of the element)
|
|
40
36
|
const targetX = boundingBox.x + boundingBox.width / 2;
|
|
41
37
|
const targetY = boundingBox.y + boundingBox.height / 2;
|
|
38
|
+
const promises = [];
|
|
42
39
|
// Start mouse movement and message display in parallel
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
40
|
+
promises.push(this.moveVirtualMouse(page, targetX, targetY, durationMillis / 2));
|
|
41
|
+
if (message && !message.trim()) {
|
|
42
|
+
// Display the message for the element
|
|
43
|
+
await this.displayMessageNearElement(element, message);
|
|
44
|
+
// Create a promise for the message duration
|
|
45
|
+
promises.push(page.waitForTimeout(durationMillis));
|
|
46
|
+
}
|
|
48
47
|
// Wait for both mouse movement and message duration to complete
|
|
49
|
-
await Promise.all(
|
|
48
|
+
await Promise.all(promises);
|
|
50
49
|
// Remove only the message after the duration (keep the mouse)
|
|
51
50
|
await element.evaluate((_, [messageContainerId]) => {
|
|
52
51
|
const container = document.getElementById(messageContainerId);
|
|
@@ -73,6 +72,7 @@ class InteractionVisualizer {
|
|
|
73
72
|
* Moves the virtual mouse to the target position with a smooth animation.
|
|
74
73
|
*/
|
|
75
74
|
async moveVirtualMouse(page, targetX, targetY, durationMillis) {
|
|
75
|
+
await InteractionVisualizer.ensureRoutesRegistered(page.context());
|
|
76
76
|
await page.evaluate(([containerId, mouseId, startX, startY, endX, endY, duration]) => {
|
|
77
77
|
return new Promise((resolve) => {
|
|
78
78
|
// 1) Find or create the container
|
|
@@ -144,7 +144,6 @@ class InteractionVisualizer {
|
|
|
144
144
|
]);
|
|
145
145
|
// Update current mouse position
|
|
146
146
|
this.currentMousePosition = { x: targetX, y: targetY };
|
|
147
|
-
this.isMouseVisible = true;
|
|
148
147
|
}
|
|
149
148
|
/**
|
|
150
149
|
* Creates a message adjacent to the element, inside a shadow root to avoid CSS collisions.
|
|
@@ -184,40 +183,41 @@ class InteractionVisualizer {
|
|
|
184
183
|
}, [InteractionVisualizer.MESSAGE_CONTAINER_ID, message]);
|
|
185
184
|
}
|
|
186
185
|
/**
|
|
187
|
-
*
|
|
186
|
+
* Ensures routes are registered for the given browser context.
|
|
187
|
+
* @param browserContext - The browser context to register routes on.
|
|
188
188
|
*/
|
|
189
|
-
async
|
|
190
|
-
if (
|
|
189
|
+
static async ensureRoutesRegistered(browserContext) {
|
|
190
|
+
if (this.registeredContexts.has(browserContext)) {
|
|
191
191
|
return;
|
|
192
192
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
216
|
-
}, [InteractionVisualizer.MESSAGE_CONTAINER_ID]);
|
|
217
|
-
this.isMouseVisible = false;
|
|
193
|
+
try {
|
|
194
|
+
await browserContext.route('**/donobu-virtual-mouse.svg', async (route) => {
|
|
195
|
+
await route.fulfill({
|
|
196
|
+
status: 200,
|
|
197
|
+
contentType: 'image/svg+xml',
|
|
198
|
+
body: InteractionVisualizer.VIRTUAL_MOUSE_SVG,
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
await browserContext.route('**/donobu.css', async (route) => {
|
|
202
|
+
await route.fulfill({
|
|
203
|
+
status: 200,
|
|
204
|
+
contentType: 'text/css',
|
|
205
|
+
body: InteractionVisualizer.CSS,
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
this.registeredContexts.add(browserContext);
|
|
209
|
+
Logger_1.appLogger.debug('Routes registered for InteractionVisualizer');
|
|
210
|
+
}
|
|
211
|
+
catch (e) {
|
|
212
|
+
Logger_1.appLogger.error('Failed to register routes for InteractionVisualizer', e);
|
|
213
|
+
throw e;
|
|
214
|
+
}
|
|
218
215
|
}
|
|
219
216
|
}
|
|
220
217
|
exports.InteractionVisualizer = InteractionVisualizer;
|
|
218
|
+
InteractionVisualizer.VIRTUAL_MOUSE_SVG = MiscUtils_1.MiscUtils.getResourceFileAsString('donobu-virtual-mouse.svg');
|
|
219
|
+
InteractionVisualizer.CSS = MiscUtils_1.MiscUtils.getResourceFileAsString('donobu.css');
|
|
221
220
|
InteractionVisualizer.MESSAGE_CONTAINER_ID = 'donobu-message-shadow-container';
|
|
222
221
|
InteractionVisualizer.VIRTUAL_MOUSE_ID = 'virtual-mouse';
|
|
222
|
+
InteractionVisualizer.registeredContexts = new WeakSet();
|
|
223
223
|
//# sourceMappingURL=InteractionVisualizer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"InteractionVisualizer.js","sourceRoot":"","sources":["../../../src/managers/InteractionVisualizer.ts"],"names":[],"mappings":";;;AACA,4CAA4C;AAC5C,8DAA2D;
|
|
1
|
+
{"version":3,"file":"InteractionVisualizer.js","sourceRoot":"","sources":["../../../src/managers/InteractionVisualizer.ts"],"names":[],"mappings":";;;AACA,4CAA4C;AAC5C,8DAA2D;AAC3D,kDAA+C;AAE/C;;GAEG;AACH,MAAa,qBAAqB;IAWhC;;;OAGG;IACH,YAAmC,4BAAoC;QAApC,iCAA4B,GAA5B,4BAA4B,CAAQ;QAN/D,yBAAoB,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IAM4B,CAAC;IAE3E;;;OAGG;IACI,KAAK,CAAC,OAAO,CAClB,IAAU,EACV,OAAgB,EAChB,OAAgB,EAChB,iBAAyB,IAAI,CAAC,4BAA4B;QAE1D,IAAI,CAAC,cAAc,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;YAC3C,OAAO;QACT,CAAC;QAED,MAAM,qBAAqB,CAAC,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAEnE,IAAI,CAAC;YACH,0CAA0C;YAC1C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC;YAEhD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,kBAAS,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACtD,OAAO;YACT,CAAC;YAED,oDAAoD;YACpD,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,GAAG,CAAC,CAAC;YACtD,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YACvD,MAAM,QAAQ,GAAoB,EAAE,CAAC;YACrC,uDAAuD;YACvD,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,CAAC,CAAC,CAClE,CAAC;YAEF,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/B,sCAAsC;gBACtC,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBACvD,4CAA4C;gBAC5C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,CAAC;YACrD,CAAC;YAED,gEAAgE;YAChE,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC5B,8DAA8D;YAC9D,MAAM,OAAO,CAAC,QAAQ,CACpB,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,EAAE,EAAE;gBAC1B,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,kBAAkB,CAAC,CAAC;gBAC9D,IAAI,SAAS,EAAE,CAAC;oBACd,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC;oBACpC,IAAI,MAAM,EAAE,CAAC;wBACX,kDAAkD;wBAClD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CACzB,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAC3C,CAAC;wBACF,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC1C,CAAC;gBACH,CAAC;YACH,CAAC,EACD,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,CAC7C,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,iCAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzC,MAAM,CAAC,CAAC;YACV,CAAC;iBAAM,CAAC;gBACN,kBAAS,CAAC,KAAK,CAAC,sCAAsC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAC5B,IAAU,EACV,OAAe,EACf,OAAe,EACf,cAAsB;QAEtB,MAAM,qBAAqB,CAAC,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,CAAC,QAAQ,CACjB,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE;YAC/D,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACnC,kCAAkC;gBAClC,IAAI,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,WAAqB,CAAC,CAAC;gBAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBAC1C,SAAS,CAAC,EAAE,GAAG,WAAqB,CAAC;oBACrC,SAAS,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC;oBACtC,SAAS,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;oBAC1B,SAAS,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;oBAC3B,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;oBAC/B,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;oBAChC,SAAS,CAAC,KAAK,CAAC,aAAa,GAAG,MAAM,CAAC;oBACvC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBAErC,MAAM,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;oBAE5D,eAAe;oBACf,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBAC/C,OAAO,CAAC,GAAG,GAAG,YAAY,CAAC;oBAC3B,OAAO,CAAC,IAAI,GAAG,aAAa,CAAC;oBAC7B,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAClC,CAAC;gBAED,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC;gBACpC,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAC;oBACV,OAAO;gBACT,CAAC;gBAED,uBAAuB;gBACvB,IAAI,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,OAAiB,CAAC,CAAC;gBACrD,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACtC,KAAK,CAAC,EAAE,GAAG,OAAiB,CAAC;oBAC7B,KAAK,CAAC,SAAS,GAAG,sBAAsB,CAAC;oBACzC,iCAAiC;oBACjC,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,MAAM,IAAI,CAAC;oBACjC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,MAAM,IAAI,CAAC;oBAChC,KAAK,CAAC,KAAK,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC,yBAAyB;oBAChE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;oBAE1B,sEAAsE;oBACtE,KAAK,KAAK,CAAC,WAAW,CAAC;gBACzB,CAAC;gBAED,oBAAoB;gBACpB,KAAK,CAAC,KAAK,CAAC,kBAAkB,GAAG,GAAG,QAAQ,IAAI,CAAC;gBACjD,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,GAAI,IAAe,GAAG,EAAE,IAAI,CAAC,CAAC,eAAe;gBAChE,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,GAAI,IAAe,GAAG,CAAC,IAAI,CAAC,CAAC,iBAAiB;gBAEhE,2DAA2D;gBAC3D,MAAM,oBAAoB,GAAG,GAAG,EAAE;oBAChC,KAAK,EAAE,mBAAmB,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;oBAElE,IAAI,KAAK,EAAE,CAAC;wBACV,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe;wBACnD,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,eAAe;wBACvC,UAAU,CAAC,GAAG,EAAE;4BACd,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,qBAAqB;wBACzD,CAAC,EAAE,EAAE,CAAC,CAAC;oBACT,CAAC;oBACD,OAAO,EAAE,CAAC;gBACZ,CAAC,CAAC;gBAEF,KAAK,CAAC,gBAAgB,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;gBAE9D,8CAA8C;gBAC9C,UAAU,CAAC,OAAO,EAAG,QAAmB,GAAG,EAAE,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;QACL,CAAC,EACD;YACE,qBAAqB,CAAC,oBAAoB;YAC1C,qBAAqB,CAAC,gBAAgB;YACtC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3B,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3B,OAAO;YACP,OAAO;YACP,cAAc;SACf,CACF,CAAC;QAEF,gCAAgC;QAChC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,yBAAyB,CACrC,OAAgB,EAChB,OAAe;QAEf,MAAM,OAAO,CAAC,QAAQ,CACpB,CAAC,aAAa,EAAE,CAAC,kBAAkB,EAAE,OAAO,CAAC,EAAE,EAAE;YAC/C,iEAAiE;YACjE,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,kBAAkB,CAAC,CAAC;YAC9D,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC;gBACxC,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC;YAEpC,oBAAoB;YACpB,MAAM,cAAc,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACrD,cAAc,CAAC,SAAS,GAAG,gBAAgB,CAAC;YAC5C,cAAc,CAAC,WAAW,GAAG,OAAO,CAAC;YAErC,+CAA+C;YAC/C,cAAc,CAAC,KAAK,CAAC,IAAI,GAAG,SAAS,CAAC;YACtC,cAAc,CAAC,KAAK,CAAC,GAAG,GAAG,SAAS,CAAC;YAErC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAEnC,4BAA4B;YAC5B,MAAM,IAAI,GAAG,aAAa,CAAC,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;YACnC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC;YAE1C,cAAc,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;YACrC,cAAc,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC;YAEpC,4CAA4C;YAC5C,MAAM,OAAO,GAAG,cAAc,CAAC,qBAAqB,EAAE,CAAC;YACvD,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC;YACxD,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACtB,cAAc,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,aAAa,GAAG,EAAE,IAAI,CAAC;YAC5D,CAAC;YACD,2CAA2C;YAC3C,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACrB,cAAc,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,MAAM,CAAC,OAAO,GAAG,EAAE,IAAI,CAAC;YACzD,CAAC;QACH,CAAC,EACD,CAAC,qBAAqB,CAAC,oBAAoB,EAAE,OAAO,CAAC,CACtD,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACzC,cAA8B;QAE9B,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAChD,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,KAAK,CACxB,6BAA6B,EAC7B,KAAK,EAAE,KAAK,EAAE,EAAE;gBACd,MAAM,KAAK,CAAC,OAAO,CAAC;oBAClB,MAAM,EAAE,GAAG;oBACX,WAAW,EAAE,eAAe;oBAC5B,IAAI,EAAE,qBAAqB,CAAC,iBAAiB;iBAC9C,CAAC,CAAC;YACL,CAAC,CACF,CAAC;YAEF,MAAM,cAAc,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBAC1D,MAAM,KAAK,CAAC,OAAO,CAAC;oBAClB,MAAM,EAAE,GAAG;oBACX,WAAW,EAAE,UAAU;oBACvB,IAAI,EAAE,qBAAqB,CAAC,GAAG;iBAChC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5C,kBAAS,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAS,CAAC,KAAK,CAAC,qDAAqD,EAAE,CAAC,CAAC,CAAC;YAC1E,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;;AA/QH,sDAgRC;AA/QyB,uCAAiB,GAAG,qBAAS,CAAC,uBAAuB,CAC3E,0BAA0B,CAC3B,AAFwC,CAEvC;AACsB,yBAAG,GAAG,qBAAS,CAAC,uBAAuB,CAAC,YAAY,CAAC,AAAlD,CAAmD;AACtD,0CAAoB,GAC1C,iCAAiC,AADS,CACR;AACZ,sCAAgB,GAAG,eAAe,AAAlB,CAAmB;AACnC,wCAAkB,GAAG,IAAI,OAAO,EAAkB,AAAhC,CAAiC"}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { Page } from 'playwright';
|
|
2
|
+
import { InteractableElement } from '../models/InteractableElement';
|
|
3
|
+
/**
|
|
4
|
+
* A class for identifying, attributing, and annotating interactable elements on web pages.
|
|
5
|
+
*
|
|
6
|
+
* The PageInspector provides functionality to:
|
|
7
|
+
* - Find and attribute interactable elements with unique identifiers
|
|
8
|
+
* - Retrieve information about attributed elements
|
|
9
|
+
* - Add visual annotations (numbered indicators) to interactable elements
|
|
10
|
+
* - Clean up both attributes and annotations
|
|
11
|
+
*
|
|
12
|
+
* Interactable elements are determined using comprehensive heuristics including:
|
|
13
|
+
* - Standard HTML interactive elements (buttons, inputs, links, etc.)
|
|
14
|
+
* - Elements with ARIA roles indicating interactivity
|
|
15
|
+
* - Elements with event handlers or CSS classes suggesting interactivity
|
|
16
|
+
* - Elements that are visible, enabled, and accessible at their coordinates
|
|
17
|
+
*
|
|
18
|
+
* This class is designed to work with Playwright's Page and Frame objects
|
|
19
|
+
* and handles cross-frame navigation, shadow DOM, and various edge cases.
|
|
20
|
+
*
|
|
21
|
+
* WARNING: It is REQUIRED that the {@link PageInspector.INIT_SCRIPT} has been
|
|
22
|
+
* run in the browser context before calling this class's methods.
|
|
23
|
+
*
|
|
24
|
+
* @example
|
|
25
|
+
* ```typescript
|
|
26
|
+
* // Basic usage
|
|
27
|
+
* await browserContext.addInitScript(PageInspector.INIT_SCRIPT);
|
|
28
|
+
* const inspector = new PageInspector();
|
|
29
|
+
*
|
|
30
|
+
* const page = await browserContext.newPage();
|
|
31
|
+
* await page.goto("https://google.com");
|
|
32
|
+
*
|
|
33
|
+
* // Find and attribute interactable elements
|
|
34
|
+
* await inspector.attributeInteractableElements(page);
|
|
35
|
+
*
|
|
36
|
+
* // Get information about interactable elements
|
|
37
|
+
* const elements = await inspector.getAttributedInteractableElements(page);
|
|
38
|
+
*
|
|
39
|
+
* // Add visual annotations to the page
|
|
40
|
+
* await inspector.annotateInteractableElements(page);
|
|
41
|
+
*
|
|
42
|
+
* // Clean up when done
|
|
43
|
+
* await inspector.removeDonobuAnnotations(page);
|
|
44
|
+
* await inspector.deattributeVisibleInteractableElements(page);
|
|
45
|
+
* ```
|
|
46
|
+
*
|
|
47
|
+
* @remarks
|
|
48
|
+
* This class uses custom HTML attributes (`data-donobu-interactable` by default)
|
|
49
|
+
* to mark elements, and creates a shadow DOM container for annotations to avoid
|
|
50
|
+
* style conflicts with the target page.
|
|
51
|
+
*
|
|
52
|
+
* All methods will throw a {@link PageClosedException} if the page is closed
|
|
53
|
+
* during operation.
|
|
54
|
+
*/
|
|
55
|
+
export declare class PageInspector {
|
|
56
|
+
readonly interactableElementAttribute: string;
|
|
57
|
+
readonly interactableAnnotationAttribute: string;
|
|
58
|
+
/**
|
|
59
|
+
* A JavaScript code snippet intended to run as an initialization
|
|
60
|
+
* script for `DonobuFlow` flows. This script helps other scripts to know
|
|
61
|
+
* non-traditional interactive elements via calling the
|
|
62
|
+
* {@link window.__donobu.getInteractiveElements} function.
|
|
63
|
+
*/
|
|
64
|
+
static readonly INIT_SCRIPT: string;
|
|
65
|
+
constructor(interactableElementAttribute?: string, interactableAnnotationAttribute?: string);
|
|
66
|
+
/**
|
|
67
|
+
* Assigns a globally unique attribute to all visible and interactable elements in the page.
|
|
68
|
+
*
|
|
69
|
+
* This method performs the following steps:
|
|
70
|
+
* 1. Removes any pre-existing interactable element attributes from the page
|
|
71
|
+
* 2. Assigns sequential numeric values as attributes to interactable elements in the main frame
|
|
72
|
+
* 3. Processes child frames that are visible in the viewport and assigns attributes to their interactable elements
|
|
73
|
+
*
|
|
74
|
+
* The method identifies "interactable" elements based on tag names, ARIA roles, CSS classes, and other heuristics.
|
|
75
|
+
* Only elements that are:
|
|
76
|
+
* - Visible (non-zero dimensions and not hidden via CSS)
|
|
77
|
+
* - More than 50% in the viewport
|
|
78
|
+
* - Not disabled or inert
|
|
79
|
+
* - Actually reachable at their coordinates (topmost in z-index)
|
|
80
|
+
* will receive the attribute.
|
|
81
|
+
*
|
|
82
|
+
* @param page - The Playwright Page object to process
|
|
83
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
84
|
+
* @returns {Promise<void>} A promise that resolves when all elements have been attributed
|
|
85
|
+
*/
|
|
86
|
+
attributeInteractableElements(page: Page): Promise<void>;
|
|
87
|
+
/**
|
|
88
|
+
* Retrieves all elements that have been previously attributed with the interactable element attribute.
|
|
89
|
+
*
|
|
90
|
+
* This method:
|
|
91
|
+
* 1. Searches all frames in the page (including the main frame and child frames)
|
|
92
|
+
* 2. Collects elements with the {@link interactableElementAttribute} attribute
|
|
93
|
+
* 3. Creates an {@link InteractableElement} object for each attributed element
|
|
94
|
+
*
|
|
95
|
+
* For each interactable element, it extracts:
|
|
96
|
+
* - The attribute value (serving as a unique identifier)
|
|
97
|
+
* - A simplified HTML snippet representation of the element
|
|
98
|
+
* * For 'select' elements, the complete HTML (including options) is preserved
|
|
99
|
+
* * For all other elements, only the opening tag without children is captured
|
|
100
|
+
*
|
|
101
|
+
* Note: This method only finds elements that have been previously attributed using
|
|
102
|
+
* the {@link attributeInteractableElements} method.
|
|
103
|
+
*
|
|
104
|
+
* @param page - The Playwright Page object to process
|
|
105
|
+
* @returns {Promise<InteractableElement[]>} A promise that resolves to an array of
|
|
106
|
+
* interactable elements with their attribute values and HTML snippets
|
|
107
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
108
|
+
*
|
|
109
|
+
* @example
|
|
110
|
+
* const inspector = new PageInspector();
|
|
111
|
+
* await inspector.attributeInteractableElements(page);
|
|
112
|
+
* const elements = await inspector.getAttributedInteractableElements(page);
|
|
113
|
+
* // elements = [{ donobuAttributeValue: "0", htmlSnippet: "<button id=\"submit\">"}]
|
|
114
|
+
*/
|
|
115
|
+
getAttributedInteractableElements(page: Page): Promise<InteractableElement[]>;
|
|
116
|
+
/**
|
|
117
|
+
* Visually annotates all interactable elements with numbered indicators on the page.
|
|
118
|
+
*
|
|
119
|
+
* This method:
|
|
120
|
+
* 1. Processes all accessible frames in the page
|
|
121
|
+
* 2. Creates (or reuses) a shadow DOM container to isolate annotation styling
|
|
122
|
+
* 3. Places circular numbered indicators over each element that has the
|
|
123
|
+
* {@link interactableElementAttribute} attribute
|
|
124
|
+
*
|
|
125
|
+
* The annotations:
|
|
126
|
+
* - Are positioned at the center of each interactable element
|
|
127
|
+
* - Have the same numeric value as the element's attribute
|
|
128
|
+
* - Are styled as black circles with red borders and white text
|
|
129
|
+
* - Are placed in a shadow DOM to avoid style conflicts with the page
|
|
130
|
+
* - Have the {@link interactableAnnotationAttribute} for identification
|
|
131
|
+
* - Are non-interactive (pointer-events: none)
|
|
132
|
+
*
|
|
133
|
+
* Note: This method requires elements to be previously attributed using the
|
|
134
|
+
* {@link attributeInteractableElements} method to find the elements to annotate.
|
|
135
|
+
*
|
|
136
|
+
* @param page - The Playwright Page object to process
|
|
137
|
+
* @returns {Promise<void>} A promise that resolves when all elements have been annotated
|
|
138
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
139
|
+
*
|
|
140
|
+
* @example
|
|
141
|
+
* const inspector = new PageInspector();
|
|
142
|
+
* await inspector.attributeInteractableElements(page);
|
|
143
|
+
* await inspector.annotateInteractableElements(page);
|
|
144
|
+
* // Visible numbered annotations now appear over all interactable elements
|
|
145
|
+
*/
|
|
146
|
+
annotateInteractableElements(page: Page): Promise<void>;
|
|
147
|
+
/**
|
|
148
|
+
* Removes all visual annotations from the page that were created by
|
|
149
|
+
* the {@link annotateInteractableElements} method.
|
|
150
|
+
*
|
|
151
|
+
* This method:
|
|
152
|
+
* 1. Processes all accessible frames in the page
|
|
153
|
+
* 2. Finds and removes the shadow DOM container with ID 'annotation-shadow-container'
|
|
154
|
+
* that contains all the annotations
|
|
155
|
+
*
|
|
156
|
+
* This effectively removes all numbered indicators that were previously placed
|
|
157
|
+
* over interactable elements, leaving the page in its original visual state.
|
|
158
|
+
* Note that this only removes the visual annotations, not the
|
|
159
|
+
* {@link interactableElementAttribute} attributes on the elements themselves.
|
|
160
|
+
*
|
|
161
|
+
* @param page - The Playwright Page object to process
|
|
162
|
+
* @returns {Promise<void>} A promise that resolves when all annotations have been removed
|
|
163
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
164
|
+
*
|
|
165
|
+
* @example
|
|
166
|
+
* const inspector = new PageInspector();
|
|
167
|
+
* await inspector.attributeInteractableElements(page);
|
|
168
|
+
* await inspector.annotateInteractableElements(page);
|
|
169
|
+
* // ... do some operations with the annotations visible ...
|
|
170
|
+
* await inspector.removeDonobuAnnotations(page);
|
|
171
|
+
* // All visual annotations are now removed from the page
|
|
172
|
+
*/
|
|
173
|
+
removeDonobuAnnotations(page: Page): Promise<void>;
|
|
174
|
+
/**
|
|
175
|
+
* Removes all interactable element attributes that were previously added to elements in the page.
|
|
176
|
+
*
|
|
177
|
+
* This method:
|
|
178
|
+
* 1. Processes all accessible frames in the page
|
|
179
|
+
* 2. Finds all elements with the {@link interactableElementAttribute} attribute
|
|
180
|
+
* 3. Removes this attribute from each element
|
|
181
|
+
*
|
|
182
|
+
* This effectively undoes the changes made by the {@link attributeInteractableElements} method,
|
|
183
|
+
* returning the page's DOM to its original state without the custom attributes.
|
|
184
|
+
* Note that this does not affect any visual annotations - to remove those, use
|
|
185
|
+
* the {@link removeDonobuAnnotations} method separately.
|
|
186
|
+
*
|
|
187
|
+
* This method is automatically called at the beginning of {@link attributeInteractableElements}
|
|
188
|
+
* to ensure a clean state before adding new attributes, but can also be called
|
|
189
|
+
* independently to clean up the DOM.
|
|
190
|
+
*
|
|
191
|
+
* @param page - The Playwright Page object to process
|
|
192
|
+
* @returns {Promise<void>} A promise that resolves when all attributes have been removed
|
|
193
|
+
* @throws {PageClosedException} If the page is closed during processing
|
|
194
|
+
*
|
|
195
|
+
* @example
|
|
196
|
+
* const inspector = new PageInspector();
|
|
197
|
+
* await inspector.attributeInteractableElements(page);
|
|
198
|
+
* // ... perform operations with attributed elements ...
|
|
199
|
+
* await inspector.deattributeInteractableElements(page);
|
|
200
|
+
* // All interactable element attributes are now removed from the page
|
|
201
|
+
*/
|
|
202
|
+
deattributeInteractableElements(page: Page): Promise<void>;
|
|
203
|
+
/**
|
|
204
|
+
* Converts an HTML attribute to a JavaScript attribute. For example,
|
|
205
|
+
* "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
|
|
206
|
+
* prefix, and the conversion from kebab-case to camelCase.
|
|
207
|
+
*/
|
|
208
|
+
static convertToJsAttribute(htmlAttribute: string): string;
|
|
209
|
+
/**
|
|
210
|
+
* An internal method that is injected into page/frame contexts to find and attribute interactable elements.
|
|
211
|
+
*
|
|
212
|
+
* This method:
|
|
213
|
+
* 1. Identifies potentially interactable elements using a comprehensive selector
|
|
214
|
+
* 2. Filters elements based on visibility, position in viewport, and interactability
|
|
215
|
+
* 3. Assigns unique sequential numeric values to the interactable attribute
|
|
216
|
+
*
|
|
217
|
+
* The method uses several criteria to determine if an element is truly interactable:
|
|
218
|
+
* - Element must be visible (non-zero dimensions, not hidden via CSS)
|
|
219
|
+
* - Element must have at least 50% of its area within the viewport
|
|
220
|
+
* - Element must not be disabled, inert, or have pointer-events:none
|
|
221
|
+
* - Element must be the topmost element at its coordinates (using point sampling)
|
|
222
|
+
*
|
|
223
|
+
* Special handling is provided for label elements, which will attribute their
|
|
224
|
+
* associated form controls as well.
|
|
225
|
+
*
|
|
226
|
+
* This method can process both standard DOM elements and elements within shadow roots,
|
|
227
|
+
* ensuring thorough coverage of modern web applications.
|
|
228
|
+
*
|
|
229
|
+
* @param arg - A tuple containing [offset: number, interactableAttribute: string]
|
|
230
|
+
* where offset is the starting value for sequential numbering and
|
|
231
|
+
* interactableAttribute is the attribute name to assign
|
|
232
|
+
* @returns The updated offset after assigning attributes (for sequential numbering across frames)
|
|
233
|
+
* @private
|
|
234
|
+
*
|
|
235
|
+
* @remarks
|
|
236
|
+
* This method is designed to be injected into the page context using page.evaluate()
|
|
237
|
+
* and should not be called directly from Node.js code.
|
|
238
|
+
*/
|
|
239
|
+
private static attributeElementsInContext;
|
|
240
|
+
private static frameFilter;
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=PageInspector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PageInspector.d.ts","sourceRoot":"","sources":["../../../src/managers/PageInspector.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,IAAI,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAMpE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,aAAa;aAYN,4BAA4B,EAAE,MAAM;aACpC,+BAA+B,EAAE,MAAM;IAZzD;;;;;OAKG;IACH,gBAAuB,WAAW,SAEhC;gBAGgB,4BAA4B,GAAE,MAAmC,EACjE,+BAA+B,GAAE,MAAiC;IAGpF;;;;;;;;;;;;;;;;;;;OAmBG;IACU,6BAA6B,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAoErE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACU,iCAAiC,CAC5C,IAAI,EAAE,IAAI,GACT,OAAO,CAAC,mBAAmB,EAAE,CAAC;IA4DjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACU,4BAA4B,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAmJpE;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACU,uBAAuB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB/D;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACU,+BAA+B,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;OAIG;WACW,oBAAoB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM;IAkBjE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAgVzC,OAAO,CAAC,MAAM,CAAC,WAAW;CAQ3B"}
|