@empiricalrun/test-gen 0.38.5 → 0.38.6

package/browser-injected-scripts/annotate-elements.spec.ts

@@ -15,7 +15,7 @@ test("should annotate all links on empirical landing page", async ({
   });
 
   const annotations = await page.evaluate(() => {
-    const { annotations } = window.annotateClickableElements();
+    const { annotations } = annotateClickableElements();
 
     return Object.entries(annotations).map(([hint, config]) => ({
       innerText: config.node.innerText,
@@ -80,7 +80,7 @@ test("should annotate all important items on quizizz page", async ({
   });
 
   const annotations = await page.evaluate(() => {
-    const { annotations } = window.annotateClickableElements();
+    const { annotations } = annotateClickableElements();
 
     return Object.entries(annotations).map(([hint, config]) => ({
       hint,

package/dist/actions/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,IAAI,EACV,iBAAiB,CAAC,EAAE,MAAM,gBAoF3B;AAED,wBAAgB,cAAc,WAG7B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,IAAI,EACV,iBAAiB,CAAC,EAAE,MAAM,gBAkH3B;AAED,wBAAgB,cAAc,WAG7B"}

package/dist/actions/utils/index.js

@@ -16,6 +16,31 @@ async function getPlaywrightLocatorUsingCssSelector(cssSelector, xpath, page, el
             selectedElem =
                 // @ts-ignore
                 window?.annotationInstance?.annotations?.[elementAnnotation]?.node;
+            const elementDepth = 
+            //@ts-ignore
+            window?.annotationInstance?.annotations?.[elementAnnotation].depth;
+            const frameLocatorStr = elementDepth
+                // @ts-ignore
+                .map((e, index) => {
+                const locator = index === 0
+                    ? //To handle the parent iframe
+                        window.playwright.generateLocator(e)
+                    : elementDepth[index - 1].contentWindow.playwright.generateLocator(e);
+                return locator.replace(/^locator\(/, "frameLocator(");
+            })
+                .join(".");
+            let elementLocator;
+            //If the element is inside an iframe, we need to append the locator for the iframe
+            if (elementDepth.length > 0) {
+                elementLocator =
+                    elementDepth[elementDepth.length - 1].contentWindow.playwright.generateLocator(selectedElem);
+            }
+            else {
+                elementLocator = window.playwright.generateLocator(selectedElem);
+            }
+            return frameLocatorStr
+                ? `${frameLocatorStr}.${elementLocator}`
+                : elementLocator;
         }
         else if (locator.xpath) {
             try {

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAK3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA6FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAgCxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAK3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA6FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuGxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -179,15 +179,83 @@ async function injectPwLocatorGenerator(page) {
         fs_extra_1.default.readFile(annotateElementPath, "utf-8"),
     ]);
     page.on("load", async () => {
-        // add script for subsequent page load events
         try {
             await Promise.all(scripts.map((s) => page.addScriptTag({ content: s })));
+            await page.evaluate(async () => {
+                //@ts-ignore
+                const injectScriptInIframe = (iframeDoc) => {
+                    try {
+                        [
+                            "https://assets-test.empirical.run/pw-selector.js",
+                            "https://code.jquery.com/jquery-3.7.1.min.js",
+                        ].forEach((url) => {
+                            const scr = iframeDoc.createElement("script");
+                            scr.src = url;
+                            console.log("Injecting script in iframe", scr);
+                            iframeDoc.head.appendChild(scr);
+                        });
+                    }
+                    catch (e) {
+                        console.warn("Error injecting script in iframe:", e);
+                    }
+                };
+                const iframes = document.getElementsByTagName("iframe");
+                for (const iframe of iframes) {
+                    iframe.getBoundingClientRect();
+                    const rect = iframe.getBoundingClientRect();
+                    const isVisible = rect.width > 0 && rect.height > 0;
+                    if (isVisible) {
+                        //@ts-ignore
+                        const iframeContent = iframe.contentDocument || iframe.contentWindow?.document;
+                        if (iframeContent) {
+                            injectScriptInIframe(iframeContent);
+                        }
+                    }
+                }
+            });
         }
         catch (e) {
-            // skip errors which can occur while the page is getting stable
+            console.warn("Error during script injection on page load:", e);
         }
     });
-    await Promise.all(scripts.map((s) => page.addScriptTag({ content: s })));
+    try {
+        await Promise.all(scripts.map((s) => page.addScriptTag({ content: s })));
+        await page.evaluate(async () => {
+            //@ts-ignore
+            const injectScriptInIframe = (iframeDoc) => {
+                try {
+                    [
+                        "https://assets-test.empirical.run/pw-selector.js",
+                        "https://code.jquery.com/jquery-3.7.1.min.js",
+                    ].forEach((url) => {
+                        const scr = iframeDoc.createElement("script");
+                        scr.src = url;
+                        console.log("Injecting script in iframe", scr);
+                        iframeDoc.head.appendChild(scr);
+                    });
+                }
+                catch (e) {
+                    console.warn("Error injecting script in iframe:", e);
+                }
+            };
+            const iframes = document.getElementsByTagName("iframe");
+            for (const iframe of iframes) {
+                iframe.getBoundingClientRect();
+                const rect = iframe.getBoundingClientRect();
+                const isVisible = rect.width > 0 && rect.height > 0;
+                if (isVisible) {
+                    //@ts-ignore
+                    const iframeContent = iframe.contentDocument || iframe.contentWindow?.document;
+                    if (iframeContent) {
+                        injectScriptInIframe(iframeContent);
+                    }
+                }
+            }
+        });
+    }
+    catch (e) {
+        console.warn("Error injecting script in iframe:", e);
+    }
 }
 exports.injectPwLocatorGenerator = injectPwLocatorGenerator;
 /**

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAclD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAoBrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FA2FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAiUA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAclD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAoBrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FA2FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAsTA"}

package/dist/agent/master/run.js

@@ -202,17 +202,17 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         const pageScreenshot = buffer.toString("base64");
         let output;
         let generatedCodeSteps = [];
-        let annotations;
         let annotatedPageScreenshot;
+        let annotationKeys = [];
         if (useHints) {
             await page.waitForTimeout(2000);
-            const annotationResult = await page.evaluate(() => {
+            annotationKeys = await page.evaluate(() => {
                 // @ts-ignore
-                window.annotationInstance = window.annotateClickableElements();
+                // eslint-disable-next-line no-undef
+                window.annotationInstance = annotateClickableElements();
                 // @ts-ignore
-                return window.annotationInstance;
+                return Object.keys(window.annotationInstance.annotations);
             });
-            annotations = annotationResult?.annotations || {};
             await page.waitForTimeout(2000);
             const annonationBuffer = await page.screenshot({
             //This is done to improve element annotation accuracy, anyways it doesn't annotate elements which are out of viewport
@@ -236,7 +236,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         if (await (0, session_1.shouldStopSession)()) {
             break;
         }
-        const annotationKeys = annotations ? Object.keys(annotations) : [];
         const toolCall = await getNextAction({
             task,
             executedActions: masterAgentActions,
@@ -275,12 +274,12 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                         name: "trigger-hints-flow",
                         input: {
                             outputFromGetNextAction: output,
-                            generatedAnnotations: annotations,
+                            generatedAnnotations: annotationKeys,
                         },
                     });
                     const result = await (0, with_hints_1.triggerHintsFlow)({
                         outputFromGetNextAction: output,
-                        generatedAnnotations: annotations,
+                        generatedAnnotations: annotationKeys,
                         page: testGenPage,
                         llm,
                         trace: triggerHintsFlowSpan,

package/dist/agent/master/with-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,WAAW;SACZ,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAqGA,CAAC"}
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,WAAW;SACZ,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAuGA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -37,12 +37,12 @@ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations,
     try {
         const hasElementAnnotation = outputFromGetNextAction?.elementAnnotation?.length &&
             outputFromGetNextAction?.elementAnnotation?.trim()?.length &&
-            outputFromGetNextAction?.elementAnnotation in
-                (generatedAnnotations || {});
+            generatedAnnotations?.includes(outputFromGetNextAction?.elementAnnotation);
         trace?.event({
             name: "has-element-annotation",
             output: {
                 hasElementAnnotation,
+                generatedAnnotations,
             },
         });
         if (!hasElementAnnotation) {