@empiricalrun/test-gen 0.35.1 → 0.35.4

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @empiricalrun/test-gen
 
+## 0.35.4
+
+### Patch Changes
+
+- 4bddc82: fix: annotate all non-blocked elements, outside the viewport too
+
+## 0.35.3
+
+### Patch Changes
+
+- 19f5344: fix: added test url in trace metadata
+
+## 0.35.2
+
+### Patch Changes
+
+- 2821db3: fix: handle agent getting stuck in loop
+- 0fe1f6e: fix: load annotate-elements script from disk
+
 ## 0.35.1
 
 ### Patch Changes

package/browser-injected-scripts/annotate-elements.js

@@ -0,0 +1,522 @@
+/* eslint-disable no-undef */
+/**
+ * Annotates all clickable elements on the page with unique hint markers.
+ * Returns an object containing annotations and methods to enable/disable them.
+ *
+ * @param {Object} options - Configuration options for hint markers.
+ * @param {string} options.hintCharacterSet - Characters to use for generating hint identifiers.
+ * @param {number} options.maxHints - Maximum number of hints to generate.
+ * @param {string} options.markerClass - CSS class to apply to hint markers.
+ * @returns {Object} An object containing annotations map and enable/disable methods.
+ */
+window.annotateClickableElements = function annotateClickableElements(
+  options = {},
+) {
+  const {
+    hintCharacterSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ", // Default set of characters for hints
+    maxHints = 1000, // Maximum number of hints to generate
+    markerClass = "hint-marker", // CSS class for markers
+  } = options;
+
+  const annotationsMap = {};
+  let annotationsContainer = null;
+
+  /**
+   * Checks if the provided DOM element is clickable by ensuring that
+   * no other elements are covering it, regardless of its current
+   * viewport visibility.
+   *
+   * @param {Element} element - The DOM element to check.
+   * @returns {boolean} - Returns true if the element is clickable; otherwise, false.
+   */
+
+  function isElementClickNotBlocked(element) {
+    const rect = element.getBoundingClientRect();
+
+    // Calculate the center point of the element
+    const centerX = rect.left + rect.width / 2;
+    const centerY = rect.top + rect.height / 2;
+
+    // check if element is within the viewport
+    if (
+      centerX < 0 ||
+      centerY < 0 ||
+      centerX > (window.innerWidth || document.documentElement.clientWidth) ||
+      centerY > (window.innerHeight || document.documentElement.clientHeight)
+    ) {
+      // Determine the viewport dimensions
+      const viewportWidth =
+        window.innerWidth || document.documentElement.clientWidth;
+      const viewportHeight =
+        window.innerHeight || document.documentElement.clientHeight;
+
+      // Calculate the new scroll positions to bring the element into the center of the viewport
+      const newScrollX = centerX - viewportWidth / 2;
+      const newScrollY = centerY - viewportHeight / 2;
+
+      // Scroll the window to the new positions
+      window.scrollTo({
+        top: newScrollY,
+        left: newScrollX,
+      });
+
+      const newRect = element.getBoundingClientRect();
+      const newCenterX = newRect.left + newRect.width / 2;
+      const newCenterY = newRect.top + newRect.height / 2;
+      const topElement = document.elementFromPoint(newCenterX, newCenterY);
+      return element.contains(topElement);
+    }
+
+    // Retrieve the topmost element at the center point
+    const topElement = document.elementFromPoint(centerX, centerY);
+
+    // Check if the topmost element is the target element or one of its descendants
+    return element.contains(topElement);
+  }
+
+  /**
+   * Generates a sequence of hint strings based on the provided character set.
+   * For example, with 'abc' as the set and maxHints 5, it generates: 'a', 'b', 'c', 'aa', 'ab'
+   * @param {string} charset - Characters to use.
+   * @param {number} max - Maximum number of hints to generate.
+   * @returns {string[]} Array of hint strings.
+   */
+  function generateHintStrings(charset, max) {
+    const hints = [];
+    let length = 1;
+
+    while (hints.length < max) {
+      const combos = cartesianProduct(Array(length).fill(charset.split("")));
+      for (const combo of combos) {
+        hints.push(combo.join(""));
+        if (hints.length >= max) break;
+      }
+      length++;
+    }
+
+    return hints;
+  }
+
+  /**
+   * Creates a Cartesian product of arrays.
+   * @param {Array<Array>} arrays - Array of arrays for Cartesian product.
+   * @returns {Array<Array>} Cartesian product.
+   */
+  function cartesianProduct(arrays) {
+    return arrays.reduce(
+      (acc, curr) =>
+        acc
+          .map((a) => curr.map((b) => a.concat([b])))
+          .reduce((a, b) => a.concat(b), []),
+      [[]],
+    );
+  }
+
+  /**
+   * Checks if an element is visible and interactable.
+   * Enhanced version inspired by getLocalHintsForElement.
+   * @param {Element} element - DOM element.
+   * @returns {boolean} True if the element is clickable and visible.
+   */
+  function isElementClickable(element) {
+    if (!(element instanceof Element)) return false;
+
+    const tagName = element.tagName.toLowerCase();
+    let isClickable = false;
+
+    // Check for aria-disabled
+    const ariaDisabled = element.getAttribute("aria-disabled");
+    if (ariaDisabled && ["", "true"].includes(ariaDisabled.toLowerCase())) {
+      return false; // Element should not be clickable if aria-disabled is true
+    }
+
+    // Check for visibility
+    const style = window.getComputedStyle(element);
+    if (
+      style.display === "none" ||
+      style.visibility === "hidden" ||
+      parseFloat(style.opacity) === 0 ||
+      style.pointerEvents === "none"
+    ) {
+      return false;
+    }
+
+    // Check if element is disabled (for applicable elements)
+    if (element.disabled) return false;
+
+    // Check for AngularJS click handlers
+    if (!isElementClickable._checkForAngularJs) {
+      isElementClickable._checkForAngularJs = (function () {
+        const angularElements = document.getElementsByClassName("ng-scope");
+        if (angularElements.length === 0) {
+          return () => false;
+        } else {
+          const ngAttributes = [];
+          for (const prefix of ["", "data-", "x-"]) {
+            for (const separator of ["-", ":", "_"]) {
+              ngAttributes.push(`${prefix}ng${separator}click`);
+            }
+          }
+          return function (el) {
+            for (const attribute of ngAttributes) {
+              if (el.hasAttribute(attribute)) return true;
+            }
+            return false;
+          };
+        }
+      })();
+    }
+
+    if (!isClickable && isElementClickable._checkForAngularJs(element)) {
+      isClickable = true;
+    }
+
+    // Check for onclick attribute or listener
+    if (
+      element.hasAttribute("onclick") ||
+      typeof element.onclick === "function"
+    ) {
+      isClickable = true;
+    }
+
+    // Check for jsaction attribute (commonly used in frameworks like Google's)
+    if (!isClickable && element.hasAttribute("jsaction")) {
+      const jsactionRules = element.getAttribute("jsaction").split(";");
+      for (const jsactionRule of jsactionRules) {
+        const ruleSplit = jsactionRule.trim().split(":");
+        if (ruleSplit.length >= 1 && ruleSplit.length <= 2) {
+          const [eventType] = ruleSplit[0].trim().split(".");
+          if (eventType === "click") {
+            isClickable = true;
+            break;
+          }
+        }
+      }
+    }
+
+    // Check for role attributes that imply clickability
+    if (!isClickable) {
+      const role = element.getAttribute("role");
+      const clickableRoles = [
+        "button",
+        "tab",
+        "link",
+        "checkbox",
+        "menuitem",
+        "menuitemcheckbox",
+        "menuitemradio",
+        "radio",
+        "switch",
+      ];
+      if (role && clickableRoles.includes(role.toLowerCase())) {
+        isClickable = true;
+      }
+    }
+
+    // Check for contentEditable
+    if (!isClickable) {
+      const contentEditable = element.getAttribute("contentEditable");
+      if (
+        contentEditable != null &&
+        ["", "contenteditable", "true"].includes(contentEditable.toLowerCase())
+      ) {
+        isClickable = true;
+      }
+    }
+
+    // Tag-specific clickability
+    const focusableTags = [
+      "a",
+      "button",
+      "input",
+      "select",
+      "textarea",
+      "object",
+      "embed",
+      "label",
+      "details",
+    ];
+    if (focusableTags.includes(tagName)) {
+      switch (tagName) {
+        case "a":
+          // Ensure it's not just a named anchor without href
+          if (element.hasAttribute("href")) {
+            isClickable = true;
+          }
+          break;
+        case "input": {
+          const type = (element.getAttribute("type") || "").toLowerCase();
+          if (
+            type !== "hidden" &&
+            !element.disabled &&
+            !(element.readOnly && isInputSelectable(element))
+          ) {
+            isClickable = true;
+          }
+          break;
+        }
+        case "textarea":
+          if (!element.disabled && !element.readOnly) {
+            isClickable = true;
+          }
+          break;
+        case "button":
+        case "select":
+          if (!element.disabled) {
+            isClickable = true;
+          }
+          break;
+        case "object":
+        case "embed":
+          isClickable = true;
+          break;
+        case "label":
+          if (element.control && !element.control.disabled) {
+            isClickable = true;
+          }
+          break;
+        case "details":
+          isClickable = true;
+          break;
+        default:
+          break;
+      }
+    }
+
+    // Check for class names containing 'button' as a possible click target
+    if (!isClickable) {
+      const className = element.getAttribute("class");
+      if (className && className.toLowerCase().includes("button")) {
+        isClickable = true;
+      } else if (element.classList.contains("cursor-pointer")) {
+        isClickable = true;
+      } else if (element.style.cursor === "pointer") {
+        isClickable = true;
+      }
+    }
+
+    // Check for tabindex
+    if (!isClickable) {
+      const tabIndexValue = element.getAttribute("tabindex");
+      const tabIndex = tabIndexValue ? parseInt(tabIndexValue) : -1;
+      if (tabIndex >= 0 && !isNaN(tabIndex)) {
+        isClickable = true;
+      }
+    }
+
+    return isClickable;
+  }
+
+  /**
+   * Helper function to determine if an input is selectable (e.g., text, search).
+   * @param {Element} input - The input element.
+   * @returns {boolean} True if the input type is selectable.
+   */
+  function isInputSelectable(input) {
+    const selectableTypes = [
+      "text",
+      "search",
+      "password",
+      "url",
+      "email",
+      "number",
+      "tel",
+    ];
+    const type = (input.getAttribute("type") || "").toLowerCase();
+    return selectableTypes.includes(type);
+  }
+
+  /**
+   * Generates a unique CSS selector path for a given element.
+   * @param {Element} element - The DOM element.
+   * @returns {string} A unique CSS selector string.
+   */
+  function getUniqueSelector(element) {
+    if (element.id) {
+      return `#${CSS.escape(element.id)}`;
+    }
+
+    const parts = [];
+    while (element && element.nodeType === Node.ELEMENT_NODE) {
+      let part = element.nodeName.toLowerCase();
+
+      // Use classList instead of className to avoid errors
+      if (element.classList && element.classList.length > 0) {
+        const classes = Array.from(element.classList)
+          .filter((cls) => cls.length > 0)
+          .map((cls) => `.${CSS.escape(cls)}`)
+          .join("");
+        part += classes;
+      }
+
+      // Add nth-child if necessary
+      const parent = element.parentNode;
+      if (parent) {
+        const siblings = Array.from(parent.children).filter(
+          (sibling) => sibling.nodeName === element.nodeName,
+        );
+        if (siblings.length > 1) {
+          const index =
+            Array.prototype.indexOf.call(parent.children, element) + 1;
+          part += `:nth-child(${index})`;
+        }
+      }
+
+      parts.unshift(part);
+      element = element.parentElement;
+    }
+
+    return parts.join(" > ");
+  }
+
+  /**
+   * Creates a hint marker element positioned at the top-left of the target element.
+   * @param {Element} el - The clickable DOM element.
+   * @param {string} hint - The hint identifier.
+   * @param {string} nodePath - Unique CSS selector path for the element.
+   * @returns {HTMLElement} The created hint marker element.
+   */
+  function createHintMarker(el, hint) {
+    const rect = el.getBoundingClientRect();
+
+    const marker = document.createElement("div");
+    marker.textContent = hint;
+    marker.className = markerClass;
+
+    // Style the marker
+    Object.assign(marker.style, {
+      position: "absolute",
+      top: `${rect.top + window.scrollY}px`,
+      left: `${rect.left + window.scrollX}px`,
+      background:
+        "-webkit-gradient(linear, 0% 0%, 0% 100%, from(rgb(255, 247, 133)), to(rgb(255, 197, 66)))",
+      padding: "1px 3px 0px",
+      borderRadius: "3px",
+      border: "1px solid rgb(227, 190, 35)",
+      fontSize: "11px",
+      pointerEvents: "none",
+      zIndex: "10000",
+      whiteSpace: "nowrap",
+      overflow: "hidden",
+      boxShadow: "rgba(0, 0, 0, 0.3) 0px 3px 7px 0px",
+      letterSpacing: 0,
+      minHeight: 0,
+      lineHeight: "100%",
+      color: "rgb(48, 37, 5)",
+      fontFamily: "Helvetica, Arial, sans-serif",
+      fontWeight: "bold",
+      textShadow: "rgba(255, 255, 255, 0.6) 0px 1px 0px",
+    });
+
+    annotationsContainer.appendChild(marker);
+
+    return marker;
+  }
+
+  /**
+   * Clears all existing hint markers from the annotations container.
+   */
+  function clearAnnotations() {
+    if (annotationsContainer) {
+      annotationsContainer.remove();
+      annotationsContainer = null;
+    }
+  }
+
+  /**
+   * Initializes the annotations by creating the container and adding markers.
+   */
+  function initializeAnnotations() {
+    clearAnnotations(); // Clear any existing annotations
+
+    annotationsContainer = document.createElement("div");
+    annotationsContainer.className = "annotations";
+    // Ensure the container covers the entire page
+    Object.assign(annotationsContainer.style, {
+      position: "absolute",
+      top: "0",
+      left: "0",
+      width: "100%",
+      height: "100%",
+      pointerEvents: "none", // Allow clicks to pass through
+      zIndex: "9999", // Ensure it's above other elements
+    });
+    document.body.appendChild(annotationsContainer);
+
+    const clickableElements = Array.from(document.querySelectorAll("*")).filter(
+      (el) => {
+        const isClickable = isElementClickable(el);
+
+        const originalScrollX = window.scrollX;
+        const originalScrollY = window.scrollY;
+
+        const isClickNotBlocked = isElementClickNotBlocked(el);
+        // Restore the original scroll positions
+        window.scrollTo(originalScrollX, originalScrollY);
+
+        return isClickable && isClickNotBlocked;
+      },
+    );
+    const hints = generateHintStrings(
+      hintCharacterSet,
+      Math.min(maxHints, clickableElements.length),
+    );
+
+    clickableElements.slice(0, maxHints).forEach((el, index) => {
+      const hint = hints[index];
+      const nodePath = getUniqueSelector(el);
+      const rect = el.getBoundingClientRect();
+
+      // Create the hint marker
+      createHintMarker(el, hint, nodePath);
+      el.style.boxShadow = `inset 0 0 0px 2px red`;
+
+      // Populate the annotations map
+      annotationsMap[hint] = {
+        node: el,
+        nodePath: nodePath,
+        rect: {
+          top: rect.top + window.scrollY,
+          left: rect.left + window.scrollX,
+          width: rect.width,
+          height: rect.height,
+          right: rect.right + window.scrollX,
+          bottom: rect.bottom + window.scrollY,
+        },
+      };
+    });
+  }
+
+  /**
+   * Enables the annotations by making the annotations container visible.
+   */
+  function enable() {
+    if (!annotationsContainer) {
+      initializeAnnotations();
+    } else {
+      annotationsContainer.style.display = "block";
+    }
+  }
+
+  /**
+   * removes all generated hints from DOM
+   */
+  function destroy() {
+    if (annotationsContainer) {
+      Object.values(annotationsMap).forEach((annotation) => {
+        annotation.node.style.boxShadow = "none";
+      });
+
+      annotationsContainer.parentNode.removeChild(annotationsContainer);
+    }
+  }
+
+  // Initialize annotations upon first run
+  enable();
+
+  // Return the desired object
+  return {
+    annotations: annotationsMap,
+    destroy,
+  };
+};

package/dist/actions/index.d.ts

@@ -15,6 +15,14 @@ export declare class PlaywrightActions {
         importPaths: string[];
     };
     getLastCodeLines(count: number): string[];
+    /**
+     * Function to check if the last three actions are repeated.
+     * If the steps / code are repeated then it means the generation is stuck in a loop.
+     *
+     * @return {boolean}
+     * @memberof PlaywrightActions
+     */
+    isStuckInLoop(): boolean;
     isComplete(): boolean;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAShB,OAAO,CAAC,IAAI;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;IACJ,OAAO,CAAC,cAAc,CAA2B;gBAE7B,IAAI,EAAE,IAAI;IAexB,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW;IAgDrB,wBAAwB,IAAI,YAAY,EAAE;IAmB1C,sBAAsB,IAAI,YAAY,EAAE;IAUxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAO9B,UAAU;CAUX"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAShB,OAAO,CAAC,IAAI;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;IACJ,OAAO,CAAC,cAAc,CAA2B;gBAE7B,IAAI,EAAE,IAAI;IAexB,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW;IAgDrB,wBAAwB,IAAI,YAAY,EAAE;IAkB1C,sBAAsB,IAAI,YAAY,EAAE;IAUxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAO9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;CAUX"}

package/dist/actions/index.js

@@ -88,7 +88,6 @@ class PlaywrightActions {
             assert_1.assertTextVisibilityActionGenerator,
             reload_page_1.reloadActionGenerator,
             text_content_1.textContentActionGenerator,
-            skill_1.skillActionGenerator,
         ]
             .map((a) => a(this.page, {
             stateVariables: this.stateVariables,
@@ -115,6 +114,19 @@ class PlaywrightActions {
         const actions = this.recordedActions.filter((a) => a.name !== done_1.PLAYWRIGHT_DONE_ACTION_NAME);
         return actions.slice(-count).map((a) => a.code);
     }
+    /**
+     * Function to check if the last three actions are repeated.
+     * If the steps / code are repeated then it means the generation is stuck in a loop.
+     *
+     * @return {boolean}
+     * @memberof PlaywrightActions
+     */
+    isStuckInLoop() {
+        const lastThreeLinesOfCode = this.getLastCodeLines(3);
+        const areLastActionsRepeatitive = lastThreeLinesOfCode.length === 3 &&
+            lastThreeLinesOfCode.every((a) => a === lastThreeLinesOfCode[0]);
+        return areLastActionsRepeatitive;
+    }
     isComplete() {
         const [doneAction] = this.recordedActions.filter((a) => a.name === done_1.PLAYWRIGHT_DONE_ACTION_NAME);
         // filter out done action from recorded actions aftet execution is marked complete

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,iBAwIA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,iBAqIA"}

package/dist/agent/browsing/index.js

@@ -122,10 +122,8 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
             const lastThreeActions = executedActions.slice(-3);
             const lastThreeActionsFailed = lastThreeActions.every((a) => a.isError);
             // get last 3 lines of code
-            const lastThreeLinesOfCode = actions.getLastCodeLines(3);
-            const areLastActionsRepeatitive = lastThreeLinesOfCode.length === 3 &&
-                lastThreeLinesOfCode.every((a) => a === lastThreeLinesOfCode[0]);
-            if (lastThreeActionsFailed || areLastActionsRepeatitive) {
+            const isStuckInLoop = actions.isStuckInLoop();
+            if (lastThreeActionsFailed || isStuckInLoop) {
                 // TODO: this should be sent to dashboard
                 const error = "Agent is not able to figure out next browser action, ending retries";
                 logger.error(error);

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAgBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAKzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAkBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAK3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAgBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAKzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAgCxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -8,6 +8,7 @@ const llm_1 = require("@empiricalrun/llm");
 const child_process_1 = require("child_process");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const minimatch_1 = require("minimatch");
+const path_1 = __importDefault(require("path"));
 const ts_morph_1 = require("ts-morph");
 const api_1 = __importDefault(require("tsx/cjs/api"));
 const logger_1 = require("../../bin/logger");
@@ -146,12 +147,19 @@ function newContentsWithTestOnly(existingContents, originalTestBlock, updatedTes
     }
 }
 async function injectPwLocatorGenerator(page) {
-    const scriptResps = await Promise.all([
+    const pathToInstalledTestGen = require.resolve(".").split("dist")[0];
+    const annotateElementPath = path_1.default.join(pathToInstalledTestGen, "dist", "browser-injected-scripts", "annotate-elements.js");
+    if (!fs_extra_1.default.existsSync(annotateElementPath)) {
+        throw new Error(`annotate-elements.js not found at path: ${annotateElementPath}`);
+    }
+    const remoteScriptResponses = await Promise.all([
         "https://assets-test.empirical.run/pw-selector.js",
         "https://code.jquery.com/jquery-3.7.1.min.js",
-        "https://assets-test.empirical.run/annotate-elements.js",
     ].map((url) => fetch(url)));
-    const scripts = await Promise.all(scriptResps.map((s) => s.text()));
+    const scripts = await Promise.all([
+        ...remoteScriptResponses.map((r) => r.text()),
+        fs_extra_1.default.readFile(annotateElementPath, "utf-8"),
+    ]);
     page.on("load", async () => {
         // add script for subsequent page load events
         try {

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAwFA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GAiRA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAwFA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GA4RA"}

package/dist/agent/master/run.js

@@ -102,15 +102,17 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
     const useHints = options?.useHints || false;
     const logger = new logger_1.CustomLogger({ useReporter: false });
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
+    const session = (0, session_1.getSessionDetails)();
     // add timeout for the page to settle in
     await page.waitForTimeout(3000);
     const trace = llm_1.langfuseInstance?.trace({
         name: "test-generator",
         id: crypto.randomUUID(),
-        version: (0, session_1.getSessionDetails)().version,
+        version: session.version,
         metadata: {
-            generationId: (0, session_1.getSessionDetails)().generationId,
-            sessionId: (0, session_1.getSessionDetails)().sessionId,
+            generationId: session.generationId,
+            sessionId: session.sessionId,
+            testUrl: session.testUrl,
         },
         tags: [
             options.metadata?.projectName,
@@ -265,7 +267,10 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
                         await actions.executeAction(currentToolCall.function.name, {
                             ...JSON.parse(currentToolCall.function.arguments),
                             ...args,
-                        });
+                        }, masterAgentActionSpan);
+                    }
+                    if (actions.isStuckInLoop()) {
+                        throw new Error("Agent is not able to figure out next action when using hints");
                     }
                 }
                 else {

package/dist/agent/master/with-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAExC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAMF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,IAAI;SACL,GAAG;MACN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAgFA,CAAC"}
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAExC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAMF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,IAAI;SACL,GAAG;MACN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CA6EA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -73,10 +73,7 @@ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations,
                     ],
                 },
             ],
-            tools: actions
-                .getBrowsingActionSchemas()
-                .filter((action) => action.function.name !== "skill_usage"),
-            model: "gpt-4o-2024-08-06",
+            tools: actions.getBrowsingActionSchemas(),
             modelParameters: {
                 temperature: 0.5,
                 max_completion_tokens: 4000,

package/dist/bin/index.js

@@ -55,6 +55,13 @@ async function runAgent(testGenConfig) {
             testGenConfig.options?.metadata.environment || "",
         ].filter((s) => !!s),
     });
+    trace?.update({
+        metadata: {
+            generationId: session.generationId,
+            sessionId: session.sessionId,
+            testUrl: session.testUrl,
+        },
+    });
     if (trace) {
         try {
             await new reporter_1.TestGenUpdatesReporter().sendAgentTraceUrl(trace.getTraceUrl());
@@ -104,6 +111,8 @@ async function runAgent(testGenConfig) {
     (0, session_1.setSessionDetails)({
         sessionId: testGenConfig.options?.metadata.testSessionId,
         generationId: testGenConfig.options?.metadata.generationId,
+        testCaseId: testGenConfig.testCase.id,
+        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
     });
     let testGenFailed = false;
     try {

package/dist/browser-injected-scripts/annotate-elements.js

@@ -0,0 +1,522 @@
+/* eslint-disable no-undef */
+/**
+ * Annotates all clickable elements on the page with unique hint markers.
+ * Returns an object containing annotations and methods to enable/disable them.
+ *
+ * @param {Object} options - Configuration options for hint markers.
+ * @param {string} options.hintCharacterSet - Characters to use for generating hint identifiers.
+ * @param {number} options.maxHints - Maximum number of hints to generate.
+ * @param {string} options.markerClass - CSS class to apply to hint markers.
+ * @returns {Object} An object containing annotations map and enable/disable methods.
+ */
+window.annotateClickableElements = function annotateClickableElements(
+  options = {},
+) {
+  const {
+    hintCharacterSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ", // Default set of characters for hints
+    maxHints = 1000, // Maximum number of hints to generate
+    markerClass = "hint-marker", // CSS class for markers
+  } = options;
+
+  const annotationsMap = {};
+  let annotationsContainer = null;
+
+  /**
+   * Checks if the provided DOM element is clickable by ensuring that
+   * no other elements are covering it, regardless of its current
+   * viewport visibility.
+   *
+   * @param {Element} element - The DOM element to check.
+   * @returns {boolean} - Returns true if the element is clickable; otherwise, false.
+   */
+
+  function isElementClickNotBlocked(element) {
+    const rect = element.getBoundingClientRect();
+
+    // Calculate the center point of the element
+    const centerX = rect.left + rect.width / 2;
+    const centerY = rect.top + rect.height / 2;
+
+    // check if element is within the viewport
+    if (
+      centerX < 0 ||
+      centerY < 0 ||
+      centerX > (window.innerWidth || document.documentElement.clientWidth) ||
+      centerY > (window.innerHeight || document.documentElement.clientHeight)
+    ) {
+      // Determine the viewport dimensions
+      const viewportWidth =
+        window.innerWidth || document.documentElement.clientWidth;
+      const viewportHeight =
+        window.innerHeight || document.documentElement.clientHeight;
+
+      // Calculate the new scroll positions to bring the element into the center of the viewport
+      const newScrollX = centerX - viewportWidth / 2;
+      const newScrollY = centerY - viewportHeight / 2;
+
+      // Scroll the window to the new positions
+      window.scrollTo({
+        top: newScrollY,
+        left: newScrollX,
+      });
+
+      const newRect = element.getBoundingClientRect();
+      const newCenterX = newRect.left + newRect.width / 2;
+      const newCenterY = newRect.top + newRect.height / 2;
+      const topElement = document.elementFromPoint(newCenterX, newCenterY);
+      return element.contains(topElement);
+    }
+
+    // Retrieve the topmost element at the center point
+    const topElement = document.elementFromPoint(centerX, centerY);
+
+    // Check if the topmost element is the target element or one of its descendants
+    return element.contains(topElement);
+  }
+
+  /**
+   * Generates a sequence of hint strings based on the provided character set.
+   * For example, with 'abc' as the set and maxHints 5, it generates: 'a', 'b', 'c', 'aa', 'ab'
+   * @param {string} charset - Characters to use.
+   * @param {number} max - Maximum number of hints to generate.
+   * @returns {string[]} Array of hint strings.
+   */
+  function generateHintStrings(charset, max) {
+    const hints = [];
+    let length = 1;
+
+    while (hints.length < max) {
+      const combos = cartesianProduct(Array(length).fill(charset.split("")));
+      for (const combo of combos) {
+        hints.push(combo.join(""));
+        if (hints.length >= max) break;
+      }
+      length++;
+    }
+
+    return hints;
+  }
+
+  /**
+   * Creates a Cartesian product of arrays.
+   * @param {Array<Array>} arrays - Array of arrays for Cartesian product.
+   * @returns {Array<Array>} Cartesian product.
+   */
+  function cartesianProduct(arrays) {
+    return arrays.reduce(
+      (acc, curr) =>
+        acc
+          .map((a) => curr.map((b) => a.concat([b])))
+          .reduce((a, b) => a.concat(b), []),
+      [[]],
+    );
+  }
+
+  /**
+   * Checks if an element is visible and interactable.
+   * Enhanced version inspired by getLocalHintsForElement.
+   * @param {Element} element - DOM element.
+   * @returns {boolean} True if the element is clickable and visible.
+   */
+  function isElementClickable(element) {
+    if (!(element instanceof Element)) return false;
+
+    const tagName = element.tagName.toLowerCase();
+    let isClickable = false;
+
+    // Check for aria-disabled
+    const ariaDisabled = element.getAttribute("aria-disabled");
+    if (ariaDisabled && ["", "true"].includes(ariaDisabled.toLowerCase())) {
+      return false; // Element should not be clickable if aria-disabled is true
+    }
+
+    // Check for visibility
+    const style = window.getComputedStyle(element);
+    if (
+      style.display === "none" ||
+      style.visibility === "hidden" ||
+      parseFloat(style.opacity) === 0 ||
+      style.pointerEvents === "none"
+    ) {
+      return false;
+    }
+
+    // Check if element is disabled (for applicable elements)
+    if (element.disabled) return false;
+
+    // Check for AngularJS click handlers
+    if (!isElementClickable._checkForAngularJs) {
+      isElementClickable._checkForAngularJs = (function () {
+        const angularElements = document.getElementsByClassName("ng-scope");
+        if (angularElements.length === 0) {
+          return () => false;
+        } else {
+          const ngAttributes = [];
+          for (const prefix of ["", "data-", "x-"]) {
+            for (const separator of ["-", ":", "_"]) {
+              ngAttributes.push(`${prefix}ng${separator}click`);
+            }
+          }
+          return function (el) {
+            for (const attribute of ngAttributes) {
+              if (el.hasAttribute(attribute)) return true;
+            }
+            return false;
+          };
+        }
+      })();
+    }
+
+    if (!isClickable && isElementClickable._checkForAngularJs(element)) {
+      isClickable = true;
+    }
+
+    // Check for onclick attribute or listener
+    if (
+      element.hasAttribute("onclick") ||
+      typeof element.onclick === "function"
+    ) {
+      isClickable = true;
+    }
+
+    // Check for jsaction attribute (commonly used in frameworks like Google's)
+    if (!isClickable && element.hasAttribute("jsaction")) {
+      const jsactionRules = element.getAttribute("jsaction").split(";");
+      for (const jsactionRule of jsactionRules) {
+        const ruleSplit = jsactionRule.trim().split(":");
+        if (ruleSplit.length >= 1 && ruleSplit.length <= 2) {
+          const [eventType] = ruleSplit[0].trim().split(".");
+          if (eventType === "click") {
+            isClickable = true;
+            break;
+          }
+        }
+      }
+    }
+
+    // Check for role attributes that imply clickability
+    if (!isClickable) {
+      const role = element.getAttribute("role");
+      const clickableRoles = [
+        "button",
+        "tab",
+        "link",
+        "checkbox",
+        "menuitem",
+        "menuitemcheckbox",
+        "menuitemradio",
+        "radio",
+        "switch",
+      ];
+      if (role && clickableRoles.includes(role.toLowerCase())) {
+        isClickable = true;
+      }
+    }
+
+    // Check for contentEditable
+    if (!isClickable) {
+      const contentEditable = element.getAttribute("contentEditable");
+      if (
+        contentEditable != null &&
+        ["", "contenteditable", "true"].includes(contentEditable.toLowerCase())
+      ) {
+        isClickable = true;
+      }
+    }
+
+    // Tag-specific clickability
+    const focusableTags = [
+      "a",
+      "button",
+      "input",
+      "select",
+      "textarea",
+      "object",
+      "embed",
+      "label",
+      "details",
+    ];
+    if (focusableTags.includes(tagName)) {
+      switch (tagName) {
+        case "a":
+          // Ensure it's not just a named anchor without href
+          if (element.hasAttribute("href")) {
+            isClickable = true;
+          }
+          break;
+        case "input": {
+          const type = (element.getAttribute("type") || "").toLowerCase();
+          if (
+            type !== "hidden" &&
+            !element.disabled &&
+            !(element.readOnly && isInputSelectable(element))
+          ) {
+            isClickable = true;
+          }
+          break;
+        }
+        case "textarea":
+          if (!element.disabled && !element.readOnly) {
+            isClickable = true;
+          }
+          break;
+        case "button":
+        case "select":
+          if (!element.disabled) {
+            isClickable = true;
+          }
+          break;
+        case "object":
+        case "embed":
+          isClickable = true;
+          break;
+        case "label":
+          if (element.control && !element.control.disabled) {
+            isClickable = true;
+          }
+          break;
+        case "details":
+          isClickable = true;
+          break;
+        default:
+          break;
+      }
+    }
+
+    // Check for class names containing 'button' as a possible click target
+    if (!isClickable) {
+      const className = element.getAttribute("class");
+      if (className && className.toLowerCase().includes("button")) {
+        isClickable = true;
+      } else if (element.classList.contains("cursor-pointer")) {
+        isClickable = true;
+      } else if (element.style.cursor === "pointer") {
+        isClickable = true;
+      }
+    }
+
+    // Check for tabindex
+    if (!isClickable) {
+      const tabIndexValue = element.getAttribute("tabindex");
+      const tabIndex = tabIndexValue ? parseInt(tabIndexValue) : -1;
+      if (tabIndex >= 0 && !isNaN(tabIndex)) {
+        isClickable = true;
+      }
+    }
+
+    return isClickable;
+  }
+
+  /**
+   * Helper function to determine if an input is selectable (e.g., text, search).
+   * @param {Element} input - The input element.
+   * @returns {boolean} True if the input type is selectable.
+   */
+  function isInputSelectable(input) {
+    const selectableTypes = [
+      "text",
+      "search",
+      "password",
+      "url",
+      "email",
+      "number",
+      "tel",
+    ];
+    const type = (input.getAttribute("type") || "").toLowerCase();
+    return selectableTypes.includes(type);
+  }
+
+  /**
+   * Generates a unique CSS selector path for a given element.
+   * @param {Element} element - The DOM element.
+   * @returns {string} A unique CSS selector string.
+   */
+  function getUniqueSelector(element) {
+    if (element.id) {
+      return `#${CSS.escape(element.id)}`;
+    }
+
+    const parts = [];
+    while (element && element.nodeType === Node.ELEMENT_NODE) {
+      let part = element.nodeName.toLowerCase();
+
+      // Use classList instead of className to avoid errors
+      if (element.classList && element.classList.length > 0) {
+        const classes = Array.from(element.classList)
+          .filter((cls) => cls.length > 0)
+          .map((cls) => `.${CSS.escape(cls)}`)
+          .join("");
+        part += classes;
+      }
+
+      // Add nth-child if necessary
+      const parent = element.parentNode;
+      if (parent) {
+        const siblings = Array.from(parent.children).filter(
+          (sibling) => sibling.nodeName === element.nodeName,
+        );
+        if (siblings.length > 1) {
+          const index =
+            Array.prototype.indexOf.call(parent.children, element) + 1;
+          part += `:nth-child(${index})`;
+        }
+      }
+
+      parts.unshift(part);
+      element = element.parentElement;
+    }
+
+    return parts.join(" > ");
+  }
+
+  /**
+   * Creates a hint marker element positioned at the top-left of the target element.
+   * @param {Element} el - The clickable DOM element.
+   * @param {string} hint - The hint identifier.
+   * @param {string} nodePath - Unique CSS selector path for the element.
+   * @returns {HTMLElement} The created hint marker element.
+   */
+  function createHintMarker(el, hint) {
+    const rect = el.getBoundingClientRect();
+
+    const marker = document.createElement("div");
+    marker.textContent = hint;
+    marker.className = markerClass;
+
+    // Style the marker
+    Object.assign(marker.style, {
+      position: "absolute",
+      top: `${rect.top + window.scrollY}px`,
+      left: `${rect.left + window.scrollX}px`,
+      background:
+        "-webkit-gradient(linear, 0% 0%, 0% 100%, from(rgb(255, 247, 133)), to(rgb(255, 197, 66)))",
+      padding: "1px 3px 0px",
+      borderRadius: "3px",
+      border: "1px solid rgb(227, 190, 35)",
+      fontSize: "11px",
+      pointerEvents: "none",
+      zIndex: "10000",
+      whiteSpace: "nowrap",
+      overflow: "hidden",
+      boxShadow: "rgba(0, 0, 0, 0.3) 0px 3px 7px 0px",
+      letterSpacing: 0,
+      minHeight: 0,
+      lineHeight: "100%",
+      color: "rgb(48, 37, 5)",
+      fontFamily: "Helvetica, Arial, sans-serif",
+      fontWeight: "bold",
+      textShadow: "rgba(255, 255, 255, 0.6) 0px 1px 0px",
+    });
+
+    annotationsContainer.appendChild(marker);
+
+    return marker;
+  }
+
+  /**
+   * Clears all existing hint markers from the annotations container.
+   */
+  function clearAnnotations() {
+    if (annotationsContainer) {
+      annotationsContainer.remove();
+      annotationsContainer = null;
+    }
+  }
+
+  /**
+   * Initializes the annotations by creating the container and adding markers.
+   */
+  function initializeAnnotations() {
+    clearAnnotations(); // Clear any existing annotations
+
+    annotationsContainer = document.createElement("div");
+    annotationsContainer.className = "annotations";
+    // Ensure the container covers the entire page
+    Object.assign(annotationsContainer.style, {
+      position: "absolute",
+      top: "0",
+      left: "0",
+      width: "100%",
+      height: "100%",
+      pointerEvents: "none", // Allow clicks to pass through
+      zIndex: "9999", // Ensure it's above other elements
+    });
+    document.body.appendChild(annotationsContainer);
+
+    const clickableElements = Array.from(document.querySelectorAll("*")).filter(
+      (el) => {
+        const isClickable = isElementClickable(el);
+
+        const originalScrollX = window.scrollX;
+        const originalScrollY = window.scrollY;
+
+        const isClickNotBlocked = isElementClickNotBlocked(el);
+        // Restore the original scroll positions
+        window.scrollTo(originalScrollX, originalScrollY);
+
+        return isClickable && isClickNotBlocked;
+      },
+    );
+    const hints = generateHintStrings(
+      hintCharacterSet,
+      Math.min(maxHints, clickableElements.length),
+    );
+
+    clickableElements.slice(0, maxHints).forEach((el, index) => {
+      const hint = hints[index];
+      const nodePath = getUniqueSelector(el);
+      const rect = el.getBoundingClientRect();
+
+      // Create the hint marker
+      createHintMarker(el, hint, nodePath);
+      el.style.boxShadow = `inset 0 0 0px 2px red`;
+
+      // Populate the annotations map
+      annotationsMap[hint] = {
+        node: el,
+        nodePath: nodePath,
+        rect: {
+          top: rect.top + window.scrollY,
+          left: rect.left + window.scrollX,
+          width: rect.width,
+          height: rect.height,
+          right: rect.right + window.scrollX,
+          bottom: rect.bottom + window.scrollY,
+        },
+      };
+    });
+  }
+
+  /**
+   * Enables the annotations by making the annotations container visible.
+   */
+  function enable() {
+    if (!annotationsContainer) {
+      initializeAnnotations();
+    } else {
+      annotationsContainer.style.display = "block";
+    }
+  }
+
+  /**
+   * removes all generated hints from DOM
+   */
+  function destroy() {
+    if (annotationsContainer) {
+      Object.values(annotationsMap).forEach((annotation) => {
+        annotation.node.style.boxShadow = "none";
+      });
+
+      annotationsContainer.parentNode.removeChild(annotationsContainer);
+    }
+  }
+
+  // Initialize annotations upon first run
+  enable();
+
+  // Return the desired object
+  return {
+    annotations: annotationsMap,
+    destroy,
+  };
+};

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,iBAoCxD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,iBAsCxD"}

package/dist/index.js

@@ -30,6 +30,8 @@ async function createTest(task, page) {
     (0, session_1.setSessionDetails)({
         sessionId: testGenConfig.options?.metadata.testSessionId,
         generationId: testGenConfig.options?.metadata.generationId,
+        testCaseId: testGenConfig.testCase.id,
+        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
     });
     const fileService = new client_1.default(Number(port));
     const { code, importPaths } = await (0, run_1.createTestUsingMasterAgent)({

package/dist/session/index.d.ts

@@ -1,11 +1,14 @@
 declare function getSessionDetails(): {
     generationId: number | undefined;
     sessionId: number | undefined;
+    testUrl: string | undefined;
     version: string;
 };
-export declare function setSessionDetails({ sessionId, generationId, }: {
+export declare function setSessionDetails({ sessionId, generationId, testCaseId, projectRepoName, }: {
     sessionId: number;
     generationId: number;
+    testCaseId: number;
+    projectRepoName: string;
 }): void;
 export declare function shouldStopSession(): Promise<boolean>;
 export declare function getSessionState(): Promise<"started" | "completed" | "request_complete">;

package/dist/session/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAgBA,iBAAS,iBAAiB;;;;EAMzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB,QAGA;AAED,wBAAsB,iBAAiB,qBAGtC;AAED,wBAAsB,eAAe,0DAqBpC;AAED,wBAAsB,UAAU,kBAqB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAkBA,iBAAS,iBAAiB;;;;;EAOzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,EACZ,UAAU,EACV,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB,QAIA;AAED,wBAAsB,iBAAiB,qBAGtC;AAED,wBAAsB,eAAe,0DAqBpC;AAED,wBAAsB,UAAU,kBAqB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/session/index.js

@@ -9,6 +9,7 @@ const sessionDetails = {
     sessionId: undefined,
     version: package_json_1.default.version,
     generationId: undefined,
+    testUrl: undefined,
 };
 const DASHBOARD_DOMAIN = process.env.DASHBOARD_DOMAIN ||
     (process.env.CI === "true" ? "https://dash.empirical.run" : "");
@@ -16,13 +17,15 @@ function getSessionDetails() {
     return {
         generationId: sessionDetails.generationId,
         sessionId: sessionDetails.sessionId,
+        testUrl: sessionDetails.testUrl,
         version: package_json_1.default.version,
     };
 }
 exports.getSessionDetails = getSessionDetails;
-function setSessionDetails({ sessionId, generationId, }) {
+function setSessionDetails({ sessionId, generationId, testCaseId, projectRepoName, }) {
     sessionDetails.sessionId = sessionId;
     sessionDetails.generationId = generationId;
+    sessionDetails.testUrl = `${process.env.DASHBOARD_DOMAIN ?? ""}/${projectRepoName}/test-cases/${testCaseId}`;
 }
 exports.setSessionDetails = setSessionDetails;
 async function shouldStopSession() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.35.1",
+  "version": "0.35.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -57,7 +57,7 @@
   },
   "scripts": {
     "dev": "tsc --build --watch",
-    "build": "tsc --build",
+    "build": "tsc --build && cp -r browser-injected-scripts dist",
     "clean": "tsc --build --clean",
     "lint": "eslint .",
     "test": "vitest run",