@empiricalrun/test-gen 0.46.5 → 0.46.7

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.46.7
+
+### Patch Changes
+
+- f2e9d28: fix: button in class name does not mean clickable
+
+## 0.46.6
+
+### Patch Changes
+
+- 8065639: fix: check for svg in descendants of provided html element and not direct children
+- f5f12f5: feat: Move icons knowledge to .empiricalrun directory
+
 ## 0.46.5
 
 ### Patch Changes

package/dist/agent/master/browser-tests/index.spec.js

@@ -71,7 +71,7 @@ test("agent can click icons accurately", async ({ page, server }) => {
     (0, test_1.expect)(response.code).toContain("page.locator");
     (0, test_1.expect)(response.code).toContain("click()");
     // Validate icons registry
-    const iconsRegistryFile = path_1.default.join(process.cwd(), "icons.json");
+    const iconsRegistryFile = path_1.default.join(process.cwd(), ".empiricalrun", "icons.json");
     const icons = JSON.parse(fs_1.default.readFileSync(iconsRegistryFile, "utf-8"));
     (0, test_1.expect)(icons.length).toBeGreaterThan(0);
     // Commenting out this check since with parallel test executions,
@@ -103,7 +103,7 @@ test("fill action with multiple pages", async ({ context }) => {
     const page1 = await context.newPage();
     const page2 = await context.newPage();
     const response = await (0, run_1.createTestUsingMasterAgent)({
-        task: `goto empirical.run on page1 and goto google.com in page2. Enter text empirical on page2 and click on search.`,
+        task: `goto empirical.run on page1 and goto github.com/search in page2. Enter text empirical on page2 and click on search.`,
         page: page2,
         options: {},
         scopeVars: {
@@ -116,7 +116,6 @@ test("fill action with multiple pages", async ({ context }) => {
     console.log(response.code);
     const lines = response.code.split("\n");
     (0, test_1.expect)(lines.find((l) => l.match(/^await page1\.goto(.+)empirical\.run/))).toBeTruthy();
-    (0, test_1.expect)(lines.find((l) => l.match(/^await page2\.goto(.+)google\.com/))).toBeTruthy();
+    (0, test_1.expect)(lines.find((l) => l.match(/^await page2\.goto(.+)github\.com\/search/))).toBeTruthy();
     (0, test_1.expect)(lines.find((l) => l.match(/^await page2(.+)fill(.+)empirical/))).toBeTruthy();
-    (0, test_1.expect)(lines.find((l) => l.match(/^await page2(.+)click/))).toBeTruthy();
 });

package/dist/agent/master/icon-descriptor/index.d.ts

@@ -11,7 +11,7 @@ export declare function generateKey(htmlString: string): string;
 export declare function reverseKey(hash: string): string;
 export declare function createNodeFromHTML(htmlString: string): {
     node: Element | null;
-    children: Element[];
+    descendant: Element[];
 };
 export declare function getIconDescription({ htmlString, pageHtml, trace, }: {
     htmlString: string;

package/dist/agent/master/icon-descriptor/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/icon-descriptor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAWrD,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,IAAI,CAAC;CACjB,CAAC;AAEF,wBAAgB,kBAAkB,IAAI,KAAK,CAAC,aAAa,CAAC,CAczD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,GAC9B,OAAO,CAAC,IAAI,CAAC,CAGf;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG/C;AAED,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM;;;EAUpD;AAqKD,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,QAAQ,EACR,KAAK,GACN,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAkC9B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/icon-descriptor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAarD,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,IAAI,CAAC;CACjB,CAAC;AAEF,wBAAgB,kBAAkB,IAAI,KAAK,CAAC,aAAa,CAAC,CAoDzD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,GAC9B,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG/C;AAED,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM;;;EAUpD;AAqKD,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,QAAQ,EACR,KAAK,GACN,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAkC9B"}

package/dist/agent/master/icon-descriptor/index.js

@@ -10,18 +10,49 @@ const jsdom_1 = require("jsdom");
 const path_1 = __importDefault(require("path"));
 const constants_1 = require("../../../constants");
 const normalize_svg_1 = require("./normalize-svg");
-const ICONS_KNOWLEDGE_PATH = path_1.default.join(process.cwd(), "icons.json");
+const OLD_ICONS_KNOWLEDGE_PATH = path_1.default.join(process.cwd(), "icons.json");
+const EMPIRICAL_RUN_DIR = path_1.default.join(process.cwd(), ".empiricalrun");
+const ICONS_KNOWLEDGE_PATH = path_1.default.join(EMPIRICAL_RUN_DIR, "icons.json");
 function loadIconsKnowledge() {
-    if (!fs_1.default.existsSync(ICONS_KNOWLEDGE_PATH)) {
+    // Check if file exists in the new location
+    if (fs_1.default.existsSync(ICONS_KNOWLEDGE_PATH)) {
+        const raw = fs_1.default.readFileSync(ICONS_KNOWLEDGE_PATH, "utf-8");
+        if (raw) {
+            try {
+                return JSON.parse(raw);
+            }
+            catch (err) {
+                console.log("error parsing iconsKnowledge json");
+                return [];
+            }
+        }
         return [];
     }
-    const raw = fs_1.default.readFileSync(ICONS_KNOWLEDGE_PATH, "utf-8");
-    if (raw) {
+    // Check if file exists in the old location and move it if it does
+    if (fs_1.default.existsSync(OLD_ICONS_KNOWLEDGE_PATH)) {
         try {
-            return JSON.parse(raw);
+            const raw = fs_1.default.readFileSync(OLD_ICONS_KNOWLEDGE_PATH, "utf-8");
+            let iconData = [];
+            if (raw) {
+                try {
+                    iconData = JSON.parse(raw);
+                }
+                catch (err) {
+                    console.log("error parsing iconsKnowledge json from old location");
+                }
+            }
+            // Create the directory and move the file
+            if (!fs_1.default.existsSync(EMPIRICAL_RUN_DIR)) {
+                fs_1.default.mkdirSync(EMPIRICAL_RUN_DIR, { recursive: true });
+            }
+            // Save to new location
+            fs_1.default.writeFileSync(ICONS_KNOWLEDGE_PATH, JSON.stringify(iconData, null, 2), "utf-8");
+            // Remove the old file
+            fs_1.default.unlinkSync(OLD_ICONS_KNOWLEDGE_PATH);
+            return iconData;
         }
         catch (err) {
-            console.log("error parsing iconsKnowledge json");
+            console.log("Error moving icons knowledge file:", err);
             return [];
         }
     }
@@ -29,8 +60,16 @@ function loadIconsKnowledge() {
 }
 exports.loadIconsKnowledge = loadIconsKnowledge;
 async function saveIconsKnowledge(iconsData) {
+    // Ensure the directory exists
+    if (!fs_1.default.existsSync(EMPIRICAL_RUN_DIR)) {
+        fs_1.default.mkdirSync(EMPIRICAL_RUN_DIR, { recursive: true });
+    }
     const content = JSON.stringify(iconsData, null, 2);
     fs_1.default.writeFileSync(ICONS_KNOWLEDGE_PATH, content, "utf-8");
+    // If old file exists, remove it
+    if (fs_1.default.existsSync(OLD_ICONS_KNOWLEDGE_PATH)) {
+        fs_1.default.unlinkSync(OLD_ICONS_KNOWLEDGE_PATH);
+    }
 }
 exports.saveIconsKnowledge = saveIconsKnowledge;
 function generateKey(htmlString) {
@@ -51,7 +90,7 @@ function createNodeFromHTML(htmlString) {
     const node = document.body.firstElementChild;
     return {
         node, // Return the first node
-        children: node?.children ? Array.from(node.children) : [], // Convert HTMLCollection to array
+        descendant: node ? Array.from(node.querySelectorAll("*")) : [], // Convert HTMLCollection to array
     };
 }
 exports.createNodeFromHTML = createNodeFromHTML;
@@ -95,7 +134,7 @@ function processSvgWithUseElements(svgElement, document) {
 function getHtmlForDescription(elementHtml, pageHtml) {
     const dom = new jsdom_1.JSDOM(pageHtml);
     const page = dom.window.document;
-    const { node, children } = createNodeFromHTML(elementHtml);
+    const { node, descendant } = createNodeFromHTML(elementHtml);
     if (!node) {
         return undefined;
     }
@@ -103,7 +142,7 @@ function getHtmlForDescription(elementHtml, pageHtml) {
         processSvgWithUseElements(node, page);
         return node.outerHTML;
     }
-    const svgChildren = children?.filter((child) => child?.tagName?.toLowerCase() === "svg");
+    const svgChildren = descendant?.filter((child) => child?.tagName?.toLowerCase() === "svg");
     if (svgChildren && svgChildren.length > 0) {
         for (const svgChild of svgChildren) {
             processSvgWithUseElements(svgChild, page);

package/dist/browser-injected-scripts/annotate-elements.js

@@ -262,12 +262,9 @@ function annotateElementsWithPreference({
       }
     }
 
-    // Check for class names containing 'button' as a possible click target
+    // Some special handling for Tailwind CSS and Vue
     if (!isClickable) {
-      const className = element.getAttribute("class");
-      if (className && className.toLowerCase().includes("button")) {
-        isClickable = true;
-      } else if (element.classList.contains("cursor-pointer")) {
+      if (element.classList.contains("cursor-pointer")) {
         isClickable = true;
       } else if (element.classList.contains("v-list-item--link")) {
         // vue specific click handling

package/dist/browser-injected-scripts/annotate-elements.spec.js

@@ -183,7 +183,7 @@ const action_tool_calls_1 = require("../agent/master/action-tool-calls");
     test_1.test.expect(annotations.length).toBe(1);
     test_1.test.expect(annotations[0]?.tagName).toBe("DIV");
 });
-test_1.test.fail("should not annotate children that don't have onClick handler", async ({ page }) => {
+(0, test_1.test)("should not annotate children that don't have onClick handler", async ({ page, }) => {
     await page.setContent(`<div class="button-outer" onclick="alert('clicked')">
       <div class="button-inner">Click me</div>
     </div>`);

package/dist/browser-injected-scripts/annotate-elements.spec.ts

@@ -300,34 +300,33 @@ test("should only annotate given text on quizziz page", async ({ page }) => {
   test.expect(annotations[0]?.tagName).toBe("DIV");
 });
 
-test.fail(
-  "should not annotate children that don't have onClick handler",
-  async ({ page }) => {
-    await page.setContent(
-      `<div class="button-outer" onclick="alert('clicked')">
+test("should not annotate children that don't have onClick handler", async ({
+  page,
+}) => {
+  await page.setContent(
+    `<div class="button-outer" onclick="alert('clicked')">
       <div class="button-inner">Click me</div>
     </div>`,
-    );
+  );
 
-    await page.addScriptTag({
-      path: path.resolve(__dirname, "./annotate-elements.js"),
-    });
+  await page.addScriptTag({
+    path: path.resolve(__dirname, "./annotate-elements.js"),
+  });
 
-    const annotations = await page.evaluate(() => {
-      // eslint-disable-next-line no-undef
-      const { annotations } = annotateElementsWithPreference();
-
-      return Object.entries(annotations).map(([hint, config]) => ({
-        hint,
-        innerText: config.node.innerText?.toLowerCase().trim(),
-        tagName: config.node.tagName,
-        testId: config.node.getAttribute("data-testid"),
-        className: config.node.className,
-      }));
-    });
+  const annotations = await page.evaluate(() => {
+    // eslint-disable-next-line no-undef
+    const { annotations } = annotateElementsWithPreference();
 
-    console.log(annotations);
-    test.expect(annotations.length).toBe(1);
-    test.expect(annotations[0].className).toBe("button-outer");
-  },
-);
+    return Object.entries(annotations).map(([hint, config]) => ({
+      hint,
+      innerText: config.node.innerText?.toLowerCase().trim(),
+      tagName: config.node.tagName,
+      testId: config.node.getAttribute("data-testid"),
+      className: config.node.className,
+    }));
+  });
+
+  console.log(annotations);
+  test.expect(annotations.length).toBe(1);
+  test.expect(annotations[0].className).toBe("button-outer");
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.46.5",
+  "version": "0.46.7",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -73,9 +73,9 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/reporter": "^0.23.1",
+    "@empiricalrun/llm": "^0.9.35",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/llm": "^0.9.35"
+    "@empiricalrun/reporter": "^0.23.1"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",

package/playwright.config.ts

@@ -11,4 +11,14 @@ export default defineConfig({
   timeout: 240_000,
   fullyParallel: true,
   workers: "50%",
+  use: {
+    launchOptions: {
+      args: [
+        "--disable-web-security",
+        "--disable-site-isolation-trials",
+        "--disable-features=IsolateOrigins,site-per-process",
+      ],
+    },
+    bypassCSP: true,
+  },
 });