@empiricalrun/test-gen 0.53.12 → 0.53.13

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @empiricalrun/test-gen
 
+## 0.53.13
+
+### Patch Changes
+
+- b847558: feat: cua codegen works for clicks in iframes
+- bb402a4: feat: accept framelocators in test-gen tool calls
+- aa38bee: feat: add valid playwright project names to system prompt
+
 ## 0.53.12
 
 ### Patch Changes

package/dist/agent/chat/agent-loop.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAEL,UAAU,EAMX,MAAM,wBAAwB,CAAC;AAYhC,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAyChE,wBAAsB,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,QAAQ,EACR,KAAK,GACN,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IAC3B,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,iBA2CA"}
+{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAEL,UAAU,EAMX,MAAM,wBAAwB,CAAC;AAYhC,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAyChE,wBAAsB,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,QAAQ,EACR,KAAK,GACN,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IAC3B,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,iBA4CA"}

package/dist/agent/chat/agent-loop.js

@@ -49,6 +49,7 @@ const log = (...args) => {
 };
 async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, }) {
     const systemPrompt = await (0, prompt_1.buildSystemPrompt)();
+    trace?.update({ input: { systemPrompt } });
     const { tools, toolExecutors } = getTools(selectedModel);
     while (!chatModel.askUserForInput) {
         const toolCalls = chatModel.getPendingToolCalls();

package/dist/agent/chat/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/repo.ts"],"names":[],"mappings":"AAmCA,wBAAsB,cAAc,oBAwBnC"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/repo.ts"],"names":[],"mappings":"AAqCA,wBAAsB,cAAc,oBAqCnC"}

package/dist/agent/chat/repo.js

@@ -7,6 +7,7 @@ exports.getRepoContext = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 const repo_tree_1 = require("../../utils/repo-tree");
+const utils_1 = require("../browsing/utils");
 async function getAllMarkdownFiles() {
     const dir = path_1.default.join(process.cwd(), ".empiricalrun");
     if (!fs_extra_1.default.existsSync(dir)) {
@@ -48,6 +49,17 @@ While specifying paths to files, use relative paths from the current working dir
 - Correct path: "tests/lesson.spec.ts"
 - Incorrect path: "/repo/tests/lesson.spec.ts" or "${path_1.default.basename(process.cwd())}/tests/lesson.spec.ts"
 `;
+    const playwrightConfig = await (0, utils_1.readPlaywrightConfig)(process.cwd());
+    const validProjectNames = await (0, utils_1.getValidProjectNames)(playwrightConfig);
+    if (validProjectNames.length > 0) {
+        REPO_CONTEXT_PROMPT += `
+## Playwright configuration
+
+This repo is configured with these Playwright projects (in the playwright.config.ts file):
+${validProjectNames.map((name) => `- ${name}`).join("\n")}
+
+`;
+    }
     const knowledge = await knowledgeContext();
     if (knowledge.length > 0) {
         REPO_CONTEXT_PROMPT += `

package/dist/agent/cua/computer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,KAAK,cAAc,GACf,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAG7C;AAgCD,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAMD;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAqID"}
+{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIvC,KAAK,cAAc,GACf,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAG7C;AAgCD,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAMD;AA4DD,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAiID"}

package/dist/agent/cua/computer.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.handleModelAction = exports.handlePageGoto = exports.getScreenshot = void 0;
+const utils_1 = require("../browsing/utils");
 async function getScreenshot(page) {
     const screenshotBytes = await page.screenshot();
     return Buffer.from(screenshotBytes).toString("base64");
@@ -43,6 +44,53 @@ async function handlePageGoto(page, url) {
     };
 }
 exports.handlePageGoto = handlePageGoto;
+async function getLocatorForClick(page, { x, y }) {
+    const locator = await page.evaluate(([x, y]) => {
+        const element = document.elementFromPoint(x, y);
+        const bbox = element?.getBoundingClientRect();
+        return {
+            locator: window.playwright.generateLocator(element),
+            isIframe: element?.tagName === "IFRAME",
+            x0: bbox?.x,
+            y0: bbox?.y,
+            src: element?.getAttribute("src"),
+            id: element?.getAttribute("id"),
+        };
+    }, [x, y]);
+    if (!locator.isIframe) {
+        return locator.locator;
+    }
+    await (0, utils_1.injectPwLocatorGenerator)(page);
+    const relativeX = x - locator.x0;
+    const relativeY = y - locator.y0;
+    // TODO: Reuse locator.locator for the frameLocator.
+    let frameEl = locator.src
+        ? page.locator(`[src="${locator.src}"]`)
+        : page.locator(`#${locator.id}`);
+    const frameLocator = locator.src
+        ? `locator('[src="${locator.src}"]')`
+        : `locator('#${locator.id}')`;
+    const elementLocatorInsideFrame = await frameEl
+        .contentFrame()
+        .locator(":root")
+        .evaluate((rootElement, coords) => {
+        const xPos = coords[0];
+        const yPos = coords[1];
+        if (xPos === undefined || yPos === undefined) {
+            throw new Error("Coordinates are undefined in evaluate call");
+        }
+        const element = document.elementFromPoint(xPos, yPos);
+        return window.playwright.generateLocator(element);
+    }, [relativeX, relativeY]);
+    return `${frameLocator}.contentFrame().${elementLocatorInsideFrame}`;
+}
+async function getLocatorForFill(page) {
+    const locator = await page.evaluate(() => {
+        const element = document.activeElement;
+        return window.playwright.generateLocator(element);
+    });
+    return locator;
+}
 async function handleModelAction(page, action) {
     const actionType = action.type;
     let actionCode = "";
@@ -61,19 +109,18 @@ async function handleModelAction(page, action) {
                     pwButton = "middle";
                 }
                 if (pwButton) {
-                    const locator = await page.evaluate(([x, y]) => {
-                        const element = document.elementFromPoint(x, y);
-                        return window.playwright.generateLocator(element);
-                    }, [x, y]);
+                    const locator = await getLocatorForClick(page, { x, y });
                     actionCode = `await page.${locator}.click();\n`;
                     await page.mouse.click(x, y, { button: pwButton });
                 }
                 if (button === "back" || button === "forward") {
                     // Do page navigations, since there is no way to click on the back/forward buttons
                     if (button === "back") {
+                        actionCode = `await page.goBack();\n`;
                         await page.goBack();
                     }
                     else if (button === "forward") {
+                        actionCode = `await page.goForward();\n`;
                         await page.goForward();
                     }
                 }
@@ -83,6 +130,8 @@ async function handleModelAction(page, action) {
                 const { x, y } = action;
                 console.log(`Action: doubleclick at (${x}, ${y})`);
                 actionSummary = `Double click at (${x}, ${y})`;
+                const locator = await getLocatorForClick(page, { x, y });
+                actionCode = `await page.${locator}.dblclick();\n`;
                 await page.mouse.dblclick(x, y, { button: "left" });
                 break;
             }
@@ -96,6 +145,7 @@ async function handleModelAction(page, action) {
             case "drag": {
                 const { path } = action;
                 console.log(`Action: drag along path ${path}`);
+                // TODO: actionCode is not implemented
                 actionSummary = `Drag along path ${path}`;
                 if (!path || path.length === 0) {
                     break;
@@ -138,10 +188,7 @@ async function handleModelAction(page, action) {
                 console.log(`Action: type text '${text}'`);
                 actionSummary = `Type text '${text}'`;
                 await page.keyboard.type(text);
-                const locator = await page.evaluate(() => {
-                    const element = document.activeElement;
-                    return window.playwright.generateLocator(element);
-                });
+                const locator = await getLocatorForFill(page);
                 actionCode = `await page.${locator}.fill("${text}");\n`;
                 break;
             }

package/dist/agent/master/browser-tests/cua.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cua.spec.d.ts.map

package/dist/agent/master/browser-tests/cua.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cua.spec.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/browser-tests/cua.spec.ts"],"names":[],"mappings":""}

package/dist/agent/master/browser-tests/cua.spec.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const run_1 = require("../run");
+const fixtures_1 = require("./fixtures");
+(0, fixtures_1.test)("computer use agent generates code for iframes", async ({ page, server, }) => {
+    await page.goto(`${server.baseURL}/iframe-elements.html`);
+    const result = await (0, run_1.createTestUsingComputerUseAgent)({
+        page,
+        task: `Click on both of the "Enter your name" input fields`,
+    });
+    (0, fixtures_1.expect)(result.code).toMatch(/page\.getByPlaceholder\(['"]Enter your name['"]?\)\.click/);
+    (0, fixtures_1.expect)(result.code).toMatch(/locator\(['"]#nested-frame['"]?\)\.contentFrame\(\)\.getByPlaceholder\(['"]Enter your name['"]?\)/);
+});

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { Page } from "playwright";
+import { FrameLocator, Page } from "playwright";
 import { ScopeVars } from "./types";
-export declare function createTest(task: string, page: Page, scope?: ScopeVars): Promise<void>;
+export declare function createTest(task: string, pageRef: Page | FrameLocator, scope?: ScopeVars): Promise<void>;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAoBpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAsD3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQhD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAoBpC,wBAAsB,UAAU,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,IAAI,GAAG,YAAY,EAC5B,KAAK,CAAC,EAAE,SAAS,iBAwDlB"}

package/dist/index.js

@@ -21,11 +21,12 @@ function setupProcessListeners(cleanup) {
         events.forEach((event) => process.removeListener(event, cleanup));
     };
 }
-async function createTest(task, page, scope) {
+async function createTest(task, pageRef, scope) {
     const removeListeners = setupProcessListeners(flushEvents);
     try {
         const testConfigArg = process.env.TEST_GEN_TOKEN;
         const testGenConfig = (0, scenarios_1.loadTestConfigs)(testConfigArg);
+        const useComputerUseAgent = testGenConfig.options?.useComputerUseAgent;
         if (testGenConfig.options && testGenConfig.options.metadata) {
             (0, reporter_1.setReporterConfig)({
                 projectRepoName: testGenConfig.options?.metadata.projectRepoName,
@@ -39,8 +40,9 @@ async function createTest(task, page, scope) {
                 projectRepoName: testGenConfig.options?.metadata.projectRepoName,
             });
         }
+        // pageRef can be a FrameLocator, in which case we need to get the Page where the iframe is located
+        let page = "owner" in pageRef ? pageRef.owner().page() : pageRef;
         const fileServiceClient = new client_1.default();
-        const useComputerUseAgent = testGenConfig.options?.useComputerUseAgent;
         let agentResult;
         if (useComputerUseAgent) {
             agentResult = await (0, cua_1.createTestUsingComputerUseAgent)({

package/dist/tools/test-gen-browser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AA8EnD,eAAO,MAAM,4BAA4B,EAAE,IA0E1C,CAAC"}
+{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAyFnD,eAAO,MAAM,4BAA4B,EAAE,IA0E1C,CAAC"}

package/dist/tools/test-gen-browser.js

@@ -66,6 +66,17 @@ await extPage
 // Instead, add the TODO before or after the multi-line statement
 \`\`\`
 
+The TODO comment must be inside a test block, not outside of it. For example, this is invalid:
+
+\`\`\`
+// --- BEGIN INVALID EXAMPLE ---
+test("Example test code", async ({ page }) => {
+  await page.goto("https://example.com");
+});
+// TODO(agent on page): Click on the login button
+// --- END INVALID EXAMPLE ---
+\`\`\`
+
 After execution, the browser agent will return a summary of actions that it took, and the generated Playwright code for them.
 You can then use the text editor tool to replace the TODO comment with the generated Playwright code.
 `;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.53.12",
+  "version": "0.53.13",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"