donobu 2.15.1 → 2.16.0

package/dist/assets/control-panel.js

@@ -1,4 +1,31 @@
 (() => {
+  if (!window.__donobu) {
+    Object.defineProperty(window, '__donobu', {
+      value: {},
+      enumerable: false,
+      writable: true,
+      configurable: false,
+    });
+  }
+
+  if (!window.__donobu.controlPanelRegistered) {
+    Object.defineProperty(window.__donobu, 'controlPanelRegistered', {
+      value: true,
+      enumerable: false,
+      writable: false,
+      configurable: false,
+    });
+  }
+
+  if (!window.__donobu.nextState) {
+    Object.defineProperty(window.__donobu, 'nextState', {
+      value: null,
+      enumerable: false,
+      writable: true,
+      configurable: false,
+    });
+  }
+
   /**
    * Wait until document.body is available, then call `init()`.
    */
@@ -358,7 +385,7 @@
         isTransitioningToPause = false;
 
         // Expose next state so external script can pick it up
-        window.donobuNextState = 'RESUMING';
+        window.__donobu.nextState = 'RESUMING';
 
         updateButtonStates();
       }
@@ -371,7 +398,7 @@
         isTransitioningToPause = true;
 
         // Expose next state
-        window.donobuNextState = 'PAUSED';
+        window.__donobu.nextState = 'PAUSED';
 
         updateButtonStates();
       }
@@ -384,7 +411,7 @@
         isTransitioningToEnd = true;
 
         // Expose next state
-        window.donobuNextState = 'SUCCESS';
+        window.__donobu.nextState = 'SUCCESS';
 
         updateButtonStates();
       }

package/dist/assets/generated/parameter-schemas.json

@@ -14,6 +14,35 @@
     "additionalProperties": false,
     "required": ["targetId", "bindingName"]
   },
+  "BaseToolGptParameters": {
+    "description": "This interface is intended to be the base for all tool parameters that are generated by a GPT.",
+    "type": "object",
+    "properties": {
+      "rationale": {
+        "description": "The reason why this particular action is being taken. It MUST relate to the overall objective.",
+        "type": "string"
+      }
+    },
+    "additionalProperties": false,
+    "required": ["rationale"]
+  },
+  "RunAccessibilityTestToolCoreParameters": {
+    "type": "object",
+    "additionalProperties": false,
+    "required": [],
+    "properties": {}
+  },
+  "RunAccessibilityTestToolGptParameters": {
+    "type": "object",
+    "properties": {
+      "rationale": {
+        "description": "The reason why this particular action is being taken. It MUST relate to the overall objective.",
+        "type": "string"
+      }
+    },
+    "additionalProperties": false,
+    "required": ["rationale"]
+  },
   "ToolParameters": {
     "anyOf": [
       {
@@ -88,7 +117,7 @@
             "type": "object",
             "additionalProperties": false
           },
-          "__@validatorSymbol@9457": {
+          "__@validatorSymbol@9578": {
             "description": "Used to mark validator functions so we can support both Zod and custom schemas.",
             "type": "boolean",
             "const": true
@@ -100,33 +129,21 @@
             "$ref": "#/definitions/JSONSchema7",
             "description": "The JSON Schema for the schema. It is passed to the providers."
           },
-          "__@schemaSymbol@9462": {
+          "__@schemaSymbol@9583": {
             "description": "Used to mark schemas so we can support both Zod and custom schemas.",
             "type": "boolean",
             "const": true
           }
         },
         "required": [
-          "__@schemaSymbol@9462",
-          "__@validatorSymbol@9457",
+          "__@schemaSymbol@9583",
+          "__@validatorSymbol@9578",
           "_type",
           "jsonSchema"
         ]
       }
     ]
   },
-  "BaseToolGptParameters": {
-    "description": "This interface is intended to be the base for all tool parameters that are generated by a GPT.",
-    "type": "object",
-    "properties": {
-      "rationale": {
-        "description": "The reason why this particular action is being taken. It MUST relate to the overall objective.",
-        "type": "string"
-      }
-    },
-    "additionalProperties": false,
-    "required": ["rationale"]
-  },
   "ChooseSelectOptionToolCoreParameters": {
     "type": "object",
     "properties": {
@@ -555,23 +572,6 @@
     "additionalProperties": false,
     "required": ["rationale", "url"]
   },
-  "RunAccessibilityTestToolCoreParameters": {
-    "type": "object",
-    "additionalProperties": false,
-    "required": [],
-    "properties": {}
-  },
-  "RunAccessibilityTestToolGptParameters": {
-    "type": "object",
-    "properties": {
-      "rationale": {
-        "description": "The reason why this particular action is being taken. It MUST relate to the overall objective.",
-        "type": "string"
-      }
-    },
-    "additionalProperties": false,
-    "required": ["rationale"]
-  },
   "ScrollPageToolCoreParameters": {
     "type": "object",
     "properties": {

package/dist/assets/generated/version

@@ -1 +1 @@
-1114
+1131

package/dist/assets/interactive-elements-tracker.js

@@ -0,0 +1,351 @@
+// interactive-elements-tracker.js
+(() => {
+  // Store all elements deemed to be interactive
+  const interactiveElements = new Set();
+
+  // Original method references for monkey patching
+  const originalAddEventListener = EventTarget.prototype.addEventListener;
+  const originalRemoveEventListener = EventTarget.prototype.removeEventListener;
+
+  // Events that indicate interactivity
+  const pointerEvents = [
+    'click',
+    'mousedown',
+    'pointerdown',
+    'mouseup',
+    'pointerup',
+    'touchstart',
+    'focus',
+    'blur',
+    'change',
+    'input',
+    'submit',
+  ];
+
+  // Helper function to check if an element is in shadow DOM
+  const isInShadowDOM = (element) => {
+    return (
+      element instanceof Element && element.getRootNode() instanceof ShadowRoot
+    );
+  };
+
+  // Function to check if an element has interactive CSS properties
+  const hasInteractiveStyling = (element) => {
+    if (!(element instanceof Element)) return false;
+
+    try {
+      const computedStyle = window.getComputedStyle(element);
+
+      // Check for style properties that suggest interactivity
+      return (
+        computedStyle.cursor === 'pointer' ||
+        computedStyle.cursor === 'hand' ||
+        element.getAttribute('role') === 'button' ||
+        element.getAttribute('role') === 'link' ||
+        element.getAttribute('aria-haspopup') === 'true' ||
+        element.getAttribute('tabindex') === '0' ||
+        element.getAttribute('contenteditable') === 'true'
+      );
+    } catch (error) {
+      return false; // Handle potential errors in computed style
+    }
+  };
+
+  // Function to check for React-specific attributes that suggest interactivity
+  const hasReactInteractiveProps = (element) => {
+    if (!(element instanceof Element)) return false;
+
+    try {
+      // Check for React event handler attributes and data attributes
+      const reactEventAttrs = [
+        'data-donobu-interactable',
+        'onClick',
+        'onMouseDown',
+        'onMouseUp',
+        'onPointerDown',
+        'onPointerUp',
+        'onTouchStart',
+        'onChange',
+        'onInput',
+        'onFocus',
+        'onBlur',
+        'onSubmit',
+      ];
+
+      // Check for React component CSS classes that suggest interactivity
+      const interactiveClasses = [
+        'cursor-pointer',
+        'hover:bg-', // Common Tailwind hover classes
+        'active:bg-',
+        'focus:outline-',
+        'hover:text-',
+      ];
+
+      // Check for attributes
+      for (const attr of reactEventAttrs) {
+        if (
+          element.hasAttribute(attr) ||
+          element.getAttribute(attr.toLowerCase()) !== null ||
+          (element.dataset && attr in element.dataset)
+        ) {
+          return true;
+        }
+      }
+
+      // Check for class names suggesting interactivity
+      const className = element.className || '';
+      if (typeof className === 'string') {
+        for (const cls of interactiveClasses) {
+          if (className.includes(cls)) {
+            return true;
+          }
+        }
+      }
+
+      return false;
+    } catch (error) {
+      return false; // Handle any potential errors
+    }
+  };
+
+  // Monkey patch addEventListener to track elements with event listeners
+  EventTarget.prototype.addEventListener = function (type, listener, options) {
+    if (pointerEvents.includes(type) && this !== window && this !== document) {
+      interactiveElements.add(this);
+
+      // If this element is in shadow DOM, ensure it's accessible
+      if (isInShadowDOM(this)) {
+        try {
+          const shadowRoot = this.getRootNode();
+          if (shadowRoot.host) {
+            interactiveElements.add(shadowRoot.host);
+          }
+        } catch (error) {
+          // Ignore errors if getRootNode fails
+        }
+      }
+    }
+    return originalAddEventListener.call(this, type, listener, options);
+  };
+
+  // Restore original removeEventListener behavior
+  EventTarget.prototype.removeEventListener = function (
+    type,
+    listener,
+    options,
+  ) {
+    if (pointerEvents.includes(type) && this !== window && this !== document) {
+      interactiveElements.delete(this);
+    }
+    return originalRemoveEventListener.call(this, type, listener, options);
+  };
+
+  // Scan DOM for potentially interactive elements
+  const scanForInteractiveElements = () => {
+    try {
+      // Common interactive element selectors
+      const interactiveSelectors = [
+        'a',
+        'button',
+        'input',
+        'select',
+        'textarea',
+        'summary',
+        '[role="button"]',
+        '[role="link"]',
+        '[role="menuitem"]',
+        '[role="tab"]',
+        '[tabindex]',
+        '[contenteditable="true"]',
+        '[data-donobu-interactable]',
+        '.cursor-pointer',
+      ];
+
+      // Only proceed if document is available
+      if (document && document.body) {
+        // Query for potential interactive elements
+        const potentialElements = document.querySelectorAll(
+          interactiveSelectors.join(','),
+        );
+
+        potentialElements.forEach((element) => {
+          // Add element if it has interactive styling or React props
+          if (
+            hasInteractiveStyling(element) ||
+            hasReactInteractiveProps(element)
+          ) {
+            interactiveElements.add(element);
+          }
+        });
+
+        // Handle shadow DOM
+        document.querySelectorAll('*').forEach((element) => {
+          if (element.shadowRoot) {
+            try {
+              const shadowElements = element.shadowRoot.querySelectorAll(
+                interactiveSelectors.join(','),
+              );
+              shadowElements.forEach((shadowEl) => {
+                if (
+                  hasInteractiveStyling(shadowEl) ||
+                  hasReactInteractiveProps(shadowEl)
+                ) {
+                  interactiveElements.add(shadowEl);
+                  interactiveElements.add(element); // Add the host element as well
+                }
+              });
+            } catch (error) {
+              // Continue if there's an error with a particular shadow root
+            }
+          }
+        });
+      }
+    } catch (error) {
+      console.error('Error scanning for interactive elements:', error);
+    }
+  };
+
+  // Initialize MutationObserver safely
+  let observer = null;
+
+  const initializeMutationObserver = () => {
+    if (!document || !document.body) return;
+
+    try {
+      observer = new MutationObserver((mutations) => {
+        let shouldRescan = false;
+
+        for (const mutation of mutations) {
+          if (mutation.type === 'childList' && mutation.addedNodes.length > 0) {
+            shouldRescan = true;
+            break;
+          }
+        }
+
+        if (shouldRescan) {
+          scanForInteractiveElements();
+        }
+      });
+
+      // Start observing the document body
+      observer.observe(document.body, {
+        childList: true,
+        subtree: true,
+      });
+    } catch (error) {
+      console.error('Error initializing MutationObserver:', error);
+    }
+  };
+
+  if (!window.__donobu) {
+    Object.defineProperty(window, '__donobu', {
+      value: {},
+      enumerable: false,
+      writable: true,
+      configurable: false,
+    });
+  }
+
+  // Add components without overwriting the namespace
+  if (!window.__donobu.getInteractiveElements) {
+    Object.defineProperty(window.__donobu, 'getInteractiveElements', {
+      value: () => {
+        // Trigger a scan to ensure we have the latest elements
+        scanForInteractiveElements();
+
+        const allInteractiveElements = [];
+
+        interactiveElements.forEach((element) => {
+          if (element instanceof Element) {
+            allInteractiveElements.push(element);
+
+            // Include shadow DOM elements
+            if (element.shadowRoot) {
+              try {
+                const shadowElements = Array.from(
+                  element.shadowRoot.querySelectorAll('*'),
+                ).filter((el) => interactiveElements.has(el));
+
+                allInteractiveElements.push(...shadowElements);
+              } catch (error) {
+                // Continue if there's an error with a particular shadow element
+              }
+            }
+          }
+        });
+
+        return allInteractiveElements;
+      },
+      enumerable: false,
+      writable: false,
+      configurable: false,
+    });
+  }
+
+  if (!window.__donobu.debugInteractiveElement) {
+    Object.defineProperty(window.__donobu, 'debugInteractiveElement', {
+      value: (element) => {
+        if (!(element instanceof Element)) {
+          return { isInteractive: false, reason: 'Not a DOM element' };
+        }
+
+        const reasons = [];
+
+        if (hasInteractiveStyling(element)) {
+          reasons.push('Has interactive CSS styling');
+        }
+
+        if (hasReactInteractiveProps(element)) {
+          reasons.push('Has React interactive props or classes');
+        }
+
+        // Check common interactive elements
+        if (
+          element.tagName === 'A' ||
+          element.tagName === 'BUTTON' ||
+          element.tagName === 'INPUT' ||
+          element.tagName === 'SELECT'
+        ) {
+          reasons.push(`Is a ${element.tagName} element`);
+        }
+
+        // Check if element has an explicit event listener
+        if (interactiveElements.has(element)) {
+          reasons.push('Has registered event listener');
+        }
+
+        return {
+          isInteractive: reasons.length > 0,
+          reasons: reasons,
+        };
+      },
+      enumerable: false,
+      writable: false,
+      configurable: false,
+    });
+  }
+
+  // Initialize things safely after DOM content is loaded
+  const initialize = () => {
+    scanForInteractiveElements();
+    initializeMutationObserver();
+  };
+
+  // Set up a safer initialization approach
+  if (document.readyState === 'loading') {
+    document.addEventListener('DOMContentLoaded', initialize);
+  } else {
+    // DOM already ready, initialize now
+    initialize();
+  }
+
+  // Periodically scan for interactive elements (React/SPA behavior)
+  const rescanInterval = 2000; // milliseconds
+  setInterval(() => {
+    try {
+      scanForInteractiveElements();
+    } catch (error) {
+      console.error('Error during periodic scan:', error);
+    }
+  }, rescanInterval);
+})();

package/dist/assets/page-interactions-tracker.js

@@ -1,16 +1,37 @@
 /**
  * Donobu script for tracking interactions with the current page. This scripts assumes the
  * following:
- *   1. There is an exposed binding to call back to Java named "donobuTrackInteraction".
- *   2. The smart-selector-generator.js script has defined the "donobuGenerateSmartSelectors"
+ *   1. There is an exposed binding to call back to NodeJS named "__donobuTrackInteraction".
+ *   2. The smart-selector-generator.js script has defined the "__donobu.generateSmartSelectors"
  *      method on the window object.
  */
 (() => {
-  if (window.donobuListenersRegistered) {
-    return;
+  if (!window.__donobu) {
+    Object.defineProperty(window, '__donobu', {
+      value: {},
+      enumerable: false,
+      writable: true,
+      configurable: false,
+    });
   }
 
-  window.donobuStartDragPosition = { x: 0, y: 0 };
+  if (!window.__donobu.interactionListenersRegistered) {
+    Object.defineProperty(window.__donobu, 'interactionListenersRegistered', {
+      value: true,
+      enumerable: false,
+      writable: false,
+      configurable: false,
+    });
+  }
+
+  if (!window.__donobu.startDragPosition) {
+    Object.defineProperty(window.__donobu, 'startDragPosition', {
+      value: { x: 0, y: 0 },
+      enumerable: false,
+      writable: true,
+      configurable: false,
+    });
+  }
 
   // Helper function to check if an event target is inside the Donobu control panel.
   const isDonobuControlPanelInteraction = (element) => {
@@ -40,20 +61,20 @@
             type: eventType,
             x: e.clientX,
             y: e.clientY,
-            selectors: window.donobuGenerateSmartSelectors(e.target),
+            selectors: window.__donobu.generateSmartSelectors(e.target),
             timestamp: new Date().getTime(),
           };
 
           if (eventType === 'mousedown') {
-            donobuStartDragPosition = { x: e.clientX, y: e.clientY };
+            window.__donobu.startDragPosition = { x: e.clientX, y: e.clientY };
           } else if (eventType === 'mouseup') {
             eventData.dragDistance = Math.sqrt(
-              Math.pow(e.clientX - donobuStartDragPosition.x, 2) +
-                Math.pow(e.clientY - donobuStartDragPosition.y, 2),
+              Math.pow(e.clientX - window.__donobu.startDragPosition.x, 2) +
+                Math.pow(e.clientY - window.__donobu.startDragPosition.y, 2),
             );
           }
 
-          donobuTrackInteraction(eventData);
+          __donobuTrackInteraction(eventData);
         },
         true,
       );
@@ -70,7 +91,7 @@
           return;
         }
 
-        donobuTrackInteraction({
+        __donobuTrackInteraction({
           type: eventType,
           key: e.key,
           keyCode: e.keyCode,
@@ -78,7 +99,7 @@
           ctrlKey: e.ctrlKey,
           metaKey: e.metaKey,
           shiftKey: e.shiftKey,
-          selectors: window.donobuGenerateSmartSelectors(e.target),
+          selectors: window.__donobu.generateSmartSelectors(e.target),
           timestamp: new Date().getTime(),
         });
       },
@@ -98,7 +119,7 @@
 
         const eventData = {
           type: eventType,
-          selectors: window.donobuGenerateSmartSelectors(e.target),
+          selectors: window.__donobu.generateSmartSelectors(e.target),
           timestamp: new Date().getTime(),
         };
 
@@ -115,7 +136,7 @@
           }
         }
 
-        donobuTrackInteraction(eventData);
+        __donobuTrackInteraction(eventData);
       },
       true,
     );
@@ -127,7 +148,7 @@
   //  window.addEventListener(
   //    "scroll",
   //    function (e) {
-  //      donobuTrackInteraction({
+  //      __donobuTrackInteraction({
   //        type: "scroll",
   //        scrollX: window.scrollX,
   //        scrollY: window.scrollY,
@@ -141,7 +162,7 @@
   //  window.addEventListener(
   //    "resize",
   //    function (e) {
-  //      donobuTrackInteraction({
+  //      __donobuTrackInteraction({
   //        type: "resize",
   //        width: window.innerWidth,
   //        height: window.innerHeight,
@@ -168,10 +189,10 @@
             identifier: touch.identifier,
           }));
 
-          donobuTrackInteraction({
+          __donobuTrackInteraction({
             type: eventType,
             touches: touches,
-            selectors: window.donobuGenerateSmartSelectors(e.target),
+            selectors: window.__donobu.generateSmartSelectors(e.target),
             timestamp: new Date().getTime(),
           });
         },
@@ -179,6 +200,4 @@
       );
     },
   );
-
-  window.donobuListenersRegistered = true;
 })();