@empiricalrun/test-gen 0.42.4 → 0.42.9

package/dist/browser-injected-scripts/annotate-elements.js

@@ -1,4 +1,4 @@
-/* eslint-disable no-undef */
+/* eslint-disable autofix/no-unused-vars */
 /**
  * Annotates all clickable elements on the page with unique hint markers.
  * Returns an object containing annotations and methods to enable/disable them.
@@ -9,7 +9,10 @@
  * @param {string} options.markerClass - CSS class to apply to hint markers.
  * @returns {Object} An object containing annotations map and enable/disable methods.
  */
-function annotateClickableElements({ options = {}, preference } = {}) {
+function annotateElementsWithPreference({
+  options = {},
+  preference = {},
+} = {}) {
   const {
     hintCharacterSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ", // Default set of characters for hints
     maxHints = 1000, // Maximum number of hints to generate
@@ -35,38 +38,18 @@ function annotateClickableElements({ options = {}, preference } = {}) {
     if (
       centerX < 0 ||
       centerY < 0 ||
-      centerX > (windowToAnnotate.innerWidth || document.documentElement.clientWidth) ||
-      centerY > (windowToAnnotate.innerHeight || document.documentElement.clientHeight)
+      centerX >
+        (windowToAnnotate.innerWidth || document.documentElement.clientWidth) ||
+      centerY >
+        (windowToAnnotate.innerHeight || document.documentElement.clientHeight)
     ) {
-      // Determine the viewport dimensions
-      const viewportWidth =
-      windowToAnnotate.innerWidth || document.documentElement.clientWidth;
-      const viewportHeight =
-      windowToAnnotate.innerHeight || document.documentElement.clientHeight;
-
-      // Calculate the new scroll positions to bring the element into the center of the viewport
-      const newScrollX = centerX - viewportWidth / 2;
-      const newScrollY = centerY - viewportHeight / 2;
-
-      // Scroll the window to the new positions
-      windowToAnnotate.scrollTo({
-        top: newScrollY,
-        left: newScrollX,
-      });
-
-      const newRect = element.getBoundingClientRect();
-      const newCenterX = newRect.left + newRect.width / 2;
-      const newCenterY = newRect.top + newRect.height / 2;
-      const topElement = document.elementFromPoint(newCenterX, newCenterY);
-      const doesElementContainTopElement = element.contains(topElement);
-
-
-      // Restore the original scroll positions
-      windowToAnnotate.scrollTo(originalScrollX, originalScrollY);
-      return doesElementContainTopElement;
+      return false;
     }
 
-    const topElement = windowToAnnotate.document.elementFromPoint(centerX, centerY);
+    const topElement = windowToAnnotate.document.elementFromPoint(
+      centerX,
+      centerY,
+    );
 
     // Check if the topmost element is the target element or one of its descendants
     return element.contains(topElement);
@@ -98,7 +81,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
         acc
           .map((a) => curr.map((b) => a.concat([b])))
           .reduce((a, b) => a.concat(b), []),
-      [[]]
+      [[]],
     );
   }
 
@@ -121,7 +104,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
       style.display === "none" ||
       style.visibility === "hidden" ||
       // This is done for cases where opacity is undefined
-      // parseFloat(style.opacity) === 0 
+      // parseFloat(style.opacity) === 0
       style.pointerEvents === "none"
     ) {
       return false;
@@ -173,11 +156,11 @@ function annotateClickableElements({ options = {}, preference } = {}) {
     // Check for onclick attribute or listener
     if (
       (element.hasAttribute("onclick") ||
-      typeof element.onclick === "function") 
+        typeof element.onclick === "function") &&
       // check for parent element having same on click
-      && element.parentNode.onclick !== element.onclick
+      element.parentNode.onclick !== element.onclick &&
       // parent element should not be a focusable tag like button
-      && focusableTags.indexOf(element.parentNode.tagName.toLowerCase()) === -1
+      focusableTags.indexOf(element.parentNode.tagName.toLowerCase()) === -1
     ) {
       isClickable = true;
     }
@@ -227,7 +210,6 @@ function annotateClickableElements({ options = {}, preference } = {}) {
       }
     }
 
-    
     if (focusableTags.includes(tagName)) {
       switch (tagName) {
         case "a":
@@ -321,7 +303,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
   // Create a hint marker
   function createHintMarker(el, hint, parentElement, windowToAnnotate) {
     const rect = el.getBoundingClientRect();
-    
+
     // Create the marker element
     const marker = document.createElement("div");
     marker.textContent = hint;
@@ -350,13 +332,12 @@ function annotateClickableElements({ options = {}, preference } = {}) {
       fontWeight: "bold",
       textShadow: "rgba(255, 255, 255, 0.6) 0px 1px 0px",
     });
-  
+
     // Attach the marker to the specified parent element
     parentElement.appendChild(marker);
     parentElements.push(parentElement);
     return marker;
   }
-  
 
   // Clear existing annotations
   //TODO: Handle clearing annotations
@@ -371,51 +352,113 @@ function annotateClickableElements({ options = {}, preference } = {}) {
   function isInputElement(el) {
     // Check if it's an input with a text-like type
     if (el instanceof HTMLInputElement) {
-      const textTypes = ['text', 'email', 'number', 'password', 'search', 'tel', 'url', 'checkbox'];
+      const textTypes = [
+        "text",
+        "email",
+        "number",
+        "password",
+        "search",
+        "tel",
+        "url",
+        "checkbox",
+      ];
       return textTypes.includes(el.type.toLowerCase());
     }
-    
+
     // Check if it's a textarea
     if (el instanceof HTMLTextAreaElement) {
       return true;
     }
-    
+
     // Check if it's contentEditable
     return el.isContentEditable;
   }
 
+  function isRequiredTextPresent(el, assertionText) {
+    assertionText = assertionText?.trim().toLowerCase();
+    if (assertionText) {
+      let elementText = el.innerText?.trim().toLowerCase();
+      let placeHolderText = el.placeholder?.trim().toLowerCase();
+      if (elementText === assertionText || placeHolderText === assertionText) {
+        //Check if the text for parent and child is same or not, if its same don't annotate again
+        if (el.parentNode.innerText?.toLowerCase() !== elementText) {
+          return true;
+        }
+      }
+      return false;
+    }
+    return true;
+  }
+
+  // This checks if the element is scrollable or not
+  function isElementScrollable(elem) {
+    function getComputedStyle(elem) {
+      return window.getComputedStyle(elem, null);
+    }
+
+    function getActualCss(elem, style) {
+      return getComputedStyle(elem)[style];
+    }
+
+    function isYScrollable(elem) {
+      return (
+        elem.offsetHeight < elem.scrollHeight &&
+        autoOrScroll(getActualCss(elem, "overflow-y"))
+      );
+    }
+
+    function autoOrScroll(text) {
+      return text == "scroll" || text == "auto";
+    }
+
+    // This doesn't annotate the elements with horizontal scroll
+    return isYScrollable(elem);
+  }
+
   // Initialize annotations for a given window (including iframes)
   function initializeAnnotations(windowToAnnotate, parentHints, depth) {
     const container =
-    parentHints?.nodeName === "IFRAME"
-      ? parentHints.contentWindow.document.body
-      : annotationsContainer;
+      parentHints?.nodeName === "IFRAME"
+        ? parentHints.contentWindow.document.body
+        : annotationsContainer;
 
     // Ensure the container exists
     if (!container) return;
-  
+
     // Filter for clickable elements
-    const clickableElements = Array.from(windowToAnnotate.document.querySelectorAll("*")).filter((el) => {
+    const elementsToAnnotate = Array.from(
+      windowToAnnotate.document.querySelectorAll("*"),
+    ).filter((el) => {
+      // Here based on the action type we filter the elements
+      // and annotate only those elements
+      switch (preference.actionType) {
+        case "fill":
+          return isInputElement(el);
 
-      //If preference is fill then it should only annotate input elements
-      if(preference === "fill"){
-        if (!isInputElement(el)) {
-          return false;
-        }
-      }
+        case "assert_text":
+          return isRequiredTextPresent(el, preference.assertionText);
+
+        case "scroll":
+          return isElementScrollable(el);
 
-      const isClickable = isElementClickable(el, windowToAnnotate);
-      const isClickNotBlocked = isElementClickNotBlocked(el, windowToAnnotate);
-      return isClickable && isClickNotBlocked;
-    });  
+        default:
+          return (
+            isElementClickable(el, windowToAnnotate) &&
+            isElementClickNotBlocked(el, windowToAnnotate)
+          );
+      }
+    });
     // Generate hint strings for the clickable elements
-    const hints = generateHintStrings(hintCharacterSet, Math.min(maxHints, clickableElements.length));
-  
+    const hints = generateHintStrings(
+      hintCharacterSet,
+      Math.min(maxHints, elementsToAnnotate.length),
+    );
+
     // Create markers for the elements
-    clickableElements.slice(0, maxHints).forEach((el, index) => {
+    elementsToAnnotate.slice(0, maxHints).forEach((el, index) => {
       const hint = hints[index];
       const rect = el.getBoundingClientRect();
-  
+
       // Use createHintMarker with the specified container
       createHintMarker(el, hint, container, windowToAnnotate);
       el.style.boxShadow = `inset 0 0 0px 2px red`;
@@ -429,23 +472,28 @@ function annotateClickableElements({ options = {}, preference } = {}) {
           width: rect.width,
           height: rect.height,
         },
-        depth: [...depth ]
+        depth: [...depth],
       };
     });
-  
+
     // Process iframes recursively
-    Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach((iframe) => {
-      try {
-        const frameWindow = iframe.contentWindow;
-        if (frameWindow && iframe.offsetWidth > 0 && iframe.offsetHeight > 0) {
-          initializeAnnotations(frameWindow, iframe,  [...depth, iframe]);
+    Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach(
+      (iframe) => {
+        try {
+          const frameWindow = iframe.contentWindow;
+          if (
+            frameWindow &&
+            iframe.offsetWidth > 0 &&
+            iframe.offsetHeight > 0
+          ) {
+            initializeAnnotations(frameWindow, iframe, [...depth, iframe]);
+          }
+        } catch (e) {
+          console.warn("Cannot access iframe:", e);
         }
-      } catch (e) {
-        console.warn("Cannot access iframe:", e);
-      }
-    });
+      },
+    );
   }
-  
 
   // Initialize and enable annotations
   function enable() {

package/dist/browser-injected-scripts/annotate-elements.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=annotate-elements.spec.d.ts.map

package/dist/browser-injected-scripts/annotate-elements.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"annotate-elements.spec.d.ts","sourceRoot":"","sources":["../../src/browser-injected-scripts/annotate-elements.spec.ts"],"names":[],"mappings":""}

package/dist/browser-injected-scripts/annotate-elements.spec.js

@@ -0,0 +1,186 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// @ts-nocheck
+const test_1 = require("@playwright/test");
+const path_1 = __importDefault(require("path"));
+const action_tool_calls_1 = require("../agent/master/action-tool-calls");
+(0, test_1.test)("should annotate all links on empirical landing page", async ({ page, }) => {
+    await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-1.html");
+    await page.addScriptTag({
+        path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
+    });
+    const annotations = await page.evaluate(() => {
+        // eslint-disable-next-line no-undef
+        const { annotations } = annotateElementsWithPreference();
+        return Object.entries(annotations).map(([, config]) => ({
+            innerText: config.node.innerText,
+            tagName: config.node.tagName,
+            href: config.node.href,
+        }));
+    });
+    test_1.test.expect(annotations).toEqual([
+        {
+            innerText: "Empirical",
+            tagName: "A",
+            href: "https://assets-test.empirical.run/",
+        },
+        {
+            innerText: "Blog",
+            tagName: "A",
+            href: "https://assets-test.empirical.run/blog",
+        },
+        {
+            innerText: "Contact us",
+            tagName: "A",
+            href: "https://assets-test.empirical.run/contact",
+        },
+        {
+            href: "https://dash.empirical.run/",
+            innerText: "Login ↗\n(opens in a new tab)",
+            tagName: "A",
+        },
+        {
+            innerText: "Get early access",
+            tagName: "A",
+            href: "https://assets-test.empirical.run/contact",
+        },
+    ]);
+});
+(0, test_1.test)("should annotate all important items on quizizz page", async ({ page, }) => {
+    await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
+    await page.addScriptTag({
+        path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
+    });
+    const annotations = await page.evaluate(() => {
+        // eslint-disable-next-line no-undef
+        const { annotations } = annotateElementsWithPreference();
+        return Object.entries(annotations).map(([hint, config]) => ({
+            hint,
+            innerText: config.node.innerText.toLowerCase().trim(),
+            tagName: config.node.tagName,
+            testId: config.node.getAttribute("data-testid"),
+            href: config.node.href,
+        }));
+    });
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("enter code")))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("get help")))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("create") &&
+        item.testId === "create-content-button"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("explore") &&
+        item.href === "https://quizizz.com/admin"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("library") &&
+        item.href === "https://quizizz.com/admin/my-library/createdByMe"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("reports") &&
+        item.href === "https://quizizz.com/admin/reports"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("classes") &&
+        item.href === "https://quizizz.com/admin/classes"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("accommodations") &&
+        item.href ===
+            "https://quizizz.com/admin/differentiation/accommodations"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("quizizz ai") &&
+        item.href === "https://quizizz.com/admin/quizizz-ai"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("start your free trial") &&
+        item.href === "https://quizizz.com/super-pricing"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.innerText.includes("upgrade") &&
+        item.href === "https://quizizz.com/super-pricing?backto=/admin"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.tagName === "INPUT" &&
+        item.testId === "emphasized-search-bar-input"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.tagName === "BUTTON" &&
+        item.innerText.includes("verify details") &&
+        item.testId === "verify-profile-cta"))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("for you")))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("assessments")))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("lessons")))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.tagName === "BUTTON" &&
+        item.innerText.includes("interactive videos")))
+        .toBeTruthy();
+    test_1.test
+        .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("passages")))
+        .toBeTruthy();
+});
+(0, test_1.test)("should only annotate input fields on quizizz page", async ({ page }) => {
+    await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
+    await page.addScriptTag({
+        path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
+    });
+    const preference = {
+        actionType: action_tool_calls_1.ActionType.FILL,
+    };
+    const annotations = await page.evaluate((preference) => {
+        // eslint-disable-next-line no-undef
+        const { annotations } = annotateElementsWithPreference({
+            preference: preference,
+        });
+        return Object.entries(annotations).map(([hint, config]) => ({
+            hint,
+            innerText: config.node.innerText.toLowerCase().trim(),
+            tagName: config.node.tagName,
+            testId: config.node.getAttribute("data-testid"),
+            href: config.node.href,
+        }));
+    }, preference);
+    test_1.test.expect(annotations.length).toBe(1);
+    test_1.test.expect(annotations[0].testId).toBe("emphasized-search-bar-input");
+});
+(0, test_1.test)("should only annotate given text on quizziz page", async ({ page }) => {
+    await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
+    await page.addScriptTag({
+        path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
+    });
+    const preference = {
+        actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
+        assertionText: "What are you teaching today?",
+    };
+    const annotations = await page.evaluate((preference) => {
+        // eslint-disable-next-line no-undef
+        const { annotations } = annotateElementsWithPreference({
+            preference: preference,
+        });
+        return Object.entries(annotations).map(([hint, config]) => ({
+            hint,
+            innerText: config.node.innerText?.toLowerCase().trim(),
+            tagName: config.node.tagName,
+            testId: config.node.getAttribute("data-testid"),
+            href: config.node.href,
+        }));
+    }, preference);
+    console.log("Annotations:", annotations);
+    test_1.test.expect(annotations.length).toBe(1);
+    test_1.test.expect(annotations[0]?.tagName).toBe("DIV");
+});

package/dist/browser-injected-scripts/annotate-elements.spec.ts

@@ -3,6 +3,9 @@
 import { test } from "@playwright/test";
 import path from "path";
 
+import { ActionType } from "../agent/master/action-tool-calls";
+import { AnnotationPreference } from "../agent/master/element-annotation";
+
 test("should annotate all links on empirical landing page", async ({
   page,
 }) => {
@@ -15,9 +18,10 @@ test("should annotate all links on empirical landing page", async ({
   });
 
   const annotations = await page.evaluate(() => {
-    const { annotations } = annotateClickableElements();
+    // eslint-disable-next-line no-undef
+    const { annotations } = annotateElementsWithPreference();
 
-    return Object.entries(annotations).map(([hint, config]) => ({
+    return Object.entries(annotations).map(([, config]) => ({
       innerText: config.node.innerText,
       tagName: config.node.tagName,
       href: config.node.href,
@@ -50,21 +54,6 @@ test("should annotate all links on empirical landing page", async ({
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
-    {
-      innerText: "Playwright\n(opens in a new tab)",
-      tagName: "A",
-      href: "https://github.com/microsoft/playwright",
-    },
-    {
-      innerText: "Meet with us",
-      tagName: "A",
-      href: "https://assets-test.empirical.run/contact",
-    },
-    {
-      innerText: "Privacy Policy",
-      tagName: "A",
-      href: "https://assets-test.empirical.run/privacy.html",
-    },
   ]);
 });
 
@@ -80,7 +69,8 @@ test("should annotate all important items on quizizz page", async ({
   });
 
   const annotations = await page.evaluate(() => {
-    const { annotations } = annotateClickableElements();
+    // eslint-disable-next-line no-undef
+    const { annotations } = annotateElementsWithPreference();
 
     return Object.entries(annotations).map(([hint, config]) => ({
       hint,
@@ -148,7 +138,7 @@ test("should annotate all important items on quizizz page", async ({
         (item) =>
           item.innerText.includes("accommodations") &&
           item.href ===
-          "https://quizizz.com/admin/differentiation/accommodations",
+            "https://quizizz.com/admin/differentiation/accommodations",
       ),
     )
     .toBeTruthy();
@@ -248,9 +238,7 @@ test("should annotate all important items on quizizz page", async ({
     .toBeTruthy();
 });
 
-test("should only annotate input fields on quizizz page", async ({
-  page,
-}) => {
+test("should only annotate input fields on quizizz page", async ({ page }) => {
   await page.goto(
     "https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html",
   );
@@ -259,9 +247,14 @@ test("should only annotate input fields on quizizz page", async ({
     path: path.resolve(__dirname, "./annotate-elements.js"),
   });
 
-  const preference = "fill";
+  const preference = {
+    actionType: ActionType.FILL,
+  };
   const annotations = await page.evaluate((preference) => {
-    const { annotations } = annotateClickableElements({preference: preference});
+    // eslint-disable-next-line no-undef
+    const { annotations } = annotateElementsWithPreference({
+      preference: preference,
+    });
 
     return Object.entries(annotations).map(([hint, config]) => ({
       hint,
@@ -270,8 +263,41 @@ test("should only annotate input fields on quizizz page", async ({
       testId: config.node.getAttribute("data-testid"),
       href: config.node.href,
     }));
-  },preference);
+  }, preference);
+
+  test.expect(annotations.length).toBe(1);
+  test.expect(annotations[0].testId).toBe("emphasized-search-bar-input");
+});
+
+test("should only annotate given text on quizziz page", async ({ page }) => {
+  await page.goto(
+    "https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html",
+  );
+
+  await page.addScriptTag({
+    path: path.resolve(__dirname, "./annotate-elements.js"),
+  });
+
+  const preference: AnnotationPreference = {
+    actionType: ActionType.ASSERT_TEXT,
+    assertionText: "What are you teaching today?",
+  };
+  const annotations = await page.evaluate((preference) => {
+    // eslint-disable-next-line no-undef
+    const { annotations } = annotateElementsWithPreference({
+      preference: preference,
+    });
+
+    return Object.entries(annotations).map(([hint, config]) => ({
+      hint,
+      innerText: config.node.innerText?.toLowerCase().trim(),
+      tagName: config.node.tagName,
+      testId: config.node.getAttribute("data-testid"),
+      href: config.node.href,
+    }));
+  }, preference);
 
+  console.log("Annotations:", annotations);
   test.expect(annotations.length).toBe(1);
-  test.expect(annotations[0].testId).toBe('emphasized-search-bar-input');
+  test.expect(annotations[0]?.tagName).toBe("DIV");
 });

package/dist/evals/master-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA0C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
+{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UAwC1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/master-agent.evals.js

@@ -3,10 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.masterGetNextActionEvaluator = void 0;
 const actions_1 = require("../actions");
 const skill_1 = require("../actions/skill");
+const action_tool_calls_1 = require("../agent/master/action-tool-calls");
 const next_action_1 = require("../agent/master/next-action");
 const page_1 = require("../page");
 const masterGetNextActionEvaluator = async ({ item, trace, }) => {
-    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, annotatedPageScreenshot, disableSkills, skills = [], annotations, } = item.input;
+    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, disableSkills, skills = [], } = item.input;
     const page = {};
     skill_1.testCaseSkills.updateSkills(skills);
     const actions = new actions_1.PlaywrightActions(new page_1.TestGenPage(page, "page"));
@@ -18,16 +19,16 @@ const masterGetNextActionEvaluator = async ({ item, trace, }) => {
         trace,
         options,
         pageScreenshot,
-        annotatedPageScreenshot,
         actions,
+        actionTypes: action_tool_calls_1.ActionType,
         disableSkills,
-        annotations,
+        page,
     });
     return {
         scores: [
             {
                 name: "action_correctness",
-                value: item.expectedOutput.function.name === output?.function.name ? 1 : 0,
+                value: item.expectedOutput.function.name === output?.actionType ? 1 : 0,
             },
         ],
         output,

package/dist/prompts/lib/ts-transformer.d.ts

@@ -0,0 +1,4 @@
+import * as ts from "typescript";
+declare function createHandlebarsTransformer(): ts.TransformerFactory<ts.SourceFile>;
+export default createHandlebarsTransformer;
+//# sourceMappingURL=ts-transformer.d.ts.map

package/dist/prompts/lib/ts-transformer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ts-transformer.d.ts","sourceRoot":"","sources":["../../../src/prompts/lib/ts-transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,YAAY,CAAC;AAEjC,iBAAS,2BAA2B,IAAI,EAAE,CAAC,kBAAkB,CAAC,EAAE,CAAC,UAAU,CAAC,CAkG3E;AAED,eAAe,2BAA2B,CAAC"}