@empiricalrun/test-gen 0.60.0 → 0.62.0

package/dist/agent/cua/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAS/D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAMlC,wBAAsB,sBAAsB,CAAC,IAAI,EAAE,IAAI,iBAoBtD;AAED,wBAAsB,+BAA+B,CAAC,EACpD,IAAI,EACJ,IAAI,EACJ,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CAqLD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAS3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAwBlC,wBAAsB,+BAA+B,CAAC,EACpD,IAAI,EACJ,IAAI,EACJ,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B,CAAC,CAyLD"}

package/dist/agent/cua/index.js

@@ -3,37 +3,32 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.startPlaywrightCodegen = startPlaywrightCodegen;
 exports.createTestUsingComputerUseAgent = createTestUsingComputerUseAgent;
 const llm_1 = require("@empiricalrun/llm");
 const openai_1 = __importDefault(require("openai"));
-const utils_1 = require("../browsing/utils");
 const computer_1 = require("./computer");
 const model_1 = require("./model");
-async function startPlaywrightCodegen(page) {
-    // TODO: Use this method to offload code generation to Playwright
-    // Unclear how to retrieve source code that is generated
-    await page.evaluate(() => {
-        setTimeout(() => {
-            // First, we start recording
-            // @ts-ignore
-            console.log(window["__pw_recorderSetMode"]("recording"));
-            // Then, we will resume the effect of pause()
-            // @ts-ignore
-            console.log(window["__pw_resume"]());
-            // Then, we remove highlights that Playwright shows on the screen
-            // @ts-ignore
-            const glassPane = document.querySelector("x-pw-glass");
-            if (glassPane) {
-                glassPane.remove();
-            }
-        }, 3000);
-    });
-    await page.pause();
+const element_from_point_1 = require("./pw-codegen/element-from-point");
+function getCodegen() {
+    return new element_from_point_1.ElementFromPointCodegen();
+    // TODO: Add support for page.pause approach
+    // We can use PlaywrightPauseCodegen if playwright patch was successful,
+    // IPC port is available and PW_CODEGEN_NO_INSPECTOR env var is set
+}
+function getStructuredArtifactInput(screenshotBytes, actionName) {
+    return {
+        name: `${actionName}`,
+        contentType: "image/png",
+        data: Buffer.from(screenshotBytes, "base64"),
+    };
 }
 async function createTestUsingComputerUseAgent({ page, task, trace, }) {
-    await (0, utils_1.injectPwLocatorGenerator)(page);
+    const codegen = getCodegen();
+    await codegen.initialize(page);
     const screenshotBytes = await (0, computer_1.getScreenshot)(page);
+    const artifacts = [
+        getStructuredArtifactInput(screenshotBytes, "Initial Screen"),
+    ];
     const viewport = page.viewportSize();
     let screenWidth = viewport?.width || 1280;
     let screenHeight = viewport?.height || 720;
@@ -101,7 +96,7 @@ async function createTestUsingComputerUseAgent({ page, task, trace, }) {
         const functionCall = functionCalls[0];
         if (functionCall) {
             const args = JSON.parse(functionCall.arguments);
-            const { actionSummary, actionCode } = await (0, computer_1.handlePageGoto)(page, args.url);
+            const { actionSummary, actionCode } = await (0, computer_1.executeModelAction)(page, { type: "goto", url: args.url }, codegen);
             executedActionSummary = actionSummary;
             actionsSummary.push(`Action executed: ${actionSummary}`);
             if (actionCode) {
@@ -119,19 +114,15 @@ async function createTestUsingComputerUseAgent({ page, task, trace, }) {
             const computerCall = computerCalls[0];
             const action = computerCall.action;
             // Execute the action and take a screenshot
-            const { actionSummary, actionCode } = await (0, computer_1.handleModelAction)(page, action);
+            const { actionSummary, actionCode } = await (0, computer_1.executeModelAction)(page, action, codegen);
             executedActionSummary = actionSummary;
             actionsSummary.push(`Action executed: ${actionSummary}`);
-            if (actionCode) {
-                actionsSummary.push(`Generated code: ${actionCode}`);
-                generatedCode += actionCode;
-            }
-            else {
-                actionsSummary.push(`No code generated: Will rely on Playwright's ability to auto-wait or auto-scroll`);
-            }
+            actionsSummary.push(`Generated code: ${actionCode}`);
+            generatedCode += actionCode;
             // Allow time for changes to take effect.
             await new Promise((resolve) => setTimeout(resolve, 1000));
             const screenshotBytes = await (0, computer_1.getScreenshot)(page);
+            artifacts.push(getStructuredArtifactInput(screenshotBytes, actionSummary));
             // Populate toolCallOutput
             toolCallOutput = {
                 type: "computer_call_output",
@@ -174,7 +165,9 @@ async function createTestUsingComputerUseAgent({ page, task, trace, }) {
     return {
         actionsSummary: actionsSummary.join("\n"),
         code: generatedCode,
-        // TODO: Does not support skills, so import paths are empty
+        // TODO: Does not support skills (from helper methods in pages/ dir),
+        // and therefore, import paths are empty
         importPaths: [],
+        artifacts,
     };
 }

package/dist/agent/cua/pw-codegen/element-from-point.d.ts

@@ -0,0 +1,8 @@
+import type { Page } from "playwright";
+import { Action, BasePlaywrightCodegen } from "./types";
+export declare class ElementFromPointCodegen implements BasePlaywrightCodegen {
+    private page;
+    initialize(page: Page): Promise<void>;
+    codeForAction(action: Action): Promise<string>;
+}
+//# sourceMappingURL=element-from-point.d.ts.map

package/dist/agent/cua/pw-codegen/element-from-point.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"element-from-point.d.ts","sourceRoot":"","sources":["../../../../src/agent/cua/pw-codegen/element-from-point.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AA4DxD,qBAAa,uBAAwB,YAAW,qBAAqB;IACnE,OAAO,CAAC,IAAI,CAAmB;IACzB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAKrC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CA0DrD"}

package/dist/agent/cua/pw-codegen/element-from-point.js

@@ -0,0 +1,118 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ElementFromPointCodegen = void 0;
+const utils_1 = require("../../browsing/utils");
+async function getLocatorForClick(page, { x, y }) {
+    const locator = await page.evaluate(([x, y]) => {
+        const element = document.elementFromPoint(x, y);
+        const bbox = element?.getBoundingClientRect();
+        return {
+            locator: window.playwright.generateLocator(element),
+            isIframe: element?.tagName === "IFRAME",
+            x0: bbox?.x,
+            y0: bbox?.y,
+            src: element?.getAttribute("src"),
+            id: element?.getAttribute("id"),
+        };
+    }, [x, y]);
+    if (!locator.isIframe) {
+        return locator.locator;
+    }
+    await (0, utils_1.injectPwLocatorGenerator)(page);
+    const relativeX = x - locator.x0;
+    const relativeY = y - locator.y0;
+    // TODO: Reuse locator.locator for the frameLocator.
+    let frameEl = locator.src
+        ? page.locator(`[src="${locator.src}"]`)
+        : page.locator(`#${locator.id}`);
+    const frameLocator = locator.src
+        ? `locator('[src="${locator.src}"]')`
+        : `locator('#${locator.id}')`;
+    const elementLocatorInsideFrame = await frameEl
+        .contentFrame()
+        .locator(":root")
+        .evaluate((rootElement, coords) => {
+        const xPos = coords[0];
+        const yPos = coords[1];
+        if (xPos === undefined || yPos === undefined) {
+            throw new Error("Coordinates are undefined in evaluate call");
+        }
+        const element = document.elementFromPoint(xPos, yPos);
+        return window.playwright.generateLocator(element);
+    }, [relativeX, relativeY]);
+    return `${frameLocator}.contentFrame().${elementLocatorInsideFrame}`;
+}
+async function getLocatorForFill(page) {
+    const locator = await page.evaluate(() => {
+        const element = document.activeElement;
+        return window.playwright.generateLocator(element);
+    });
+    return locator;
+}
+class ElementFromPointCodegen {
+    page;
+    async initialize(page) {
+        this.page = page;
+        await (0, utils_1.injectPwLocatorGenerator)(page);
+    }
+    async codeForAction(action) {
+        try {
+            if (action.type === "goto") {
+                return `await page.goto("${action.url}");\n`;
+            }
+            if (action.type === "click") {
+                const { x, y } = action;
+                if (!this.page) {
+                    throw new Error("Page is not initialized");
+                }
+                const locator = await getLocatorForClick(this.page, { x, y });
+                return `await page.${locator}.click();\n`;
+            }
+            if (action.type === "type") {
+                if (!this.page) {
+                    throw new Error("Page is not initialized");
+                }
+                const locator = await getLocatorForFill(this.page);
+                return `await page.${locator}.fill("${action.text}");\n`;
+            }
+            if (action.type === "back") {
+                return `await page.goBack();\n`;
+            }
+            if (action.type === "forward") {
+                return `await page.goForward();\n`;
+            }
+            if (action.type === "doubleclick") {
+                const { x, y } = action;
+                if (!this.page) {
+                    throw new Error("Page is not initialized");
+                }
+                const locator = await getLocatorForClick(this.page, { x, y });
+                return `await page.${locator}.dblclick();\n`;
+            }
+            if (action.type === "move") {
+                const { x, y } = action;
+                return `await page.mouse.move(${x}, ${y});\n`;
+            }
+            if (action.type === "drag") {
+                // const { path } = action;
+                // TODO: This needs to be fixed
+                return `No code generated: drag action not supported by code generation`;
+            }
+            if (action.type === "scroll") {
+                return `No code to generate: will rely on Playwright's ability to auto-wait`;
+            }
+            if (action.type === "keypress") {
+                const { keys } = action;
+                return `await page.keyboard.press("${keys.join("+")}");\n`;
+            }
+            if (action.type === "wait") {
+                return `No code to generate: will rely on Playwright's ability to auto-wait`;
+            }
+            throw new Error(`Unsupported action: ${JSON.stringify(action)}`);
+        }
+        catch (e) {
+            return `Error generating code for action: ${e.message || e.toString()}`;
+        }
+    }
+}
+exports.ElementFromPointCodegen = ElementFromPointCodegen;

package/dist/agent/cua/pw-codegen/pw-pause/index.d.ts

@@ -0,0 +1,15 @@
+import type { Page } from "playwright";
+export { revertToOriginalPwCode } from "./utils";
+export declare const PW_PAUSE_IPC_PORT = 3039;
+export declare function preparePlaywrightForCodegen(repoDir: string): Promise<void>;
+export declare class PlaywrightPauseCodegen {
+    private port;
+    private page;
+    private server;
+    constructor();
+    initialize(page: Page): Promise<void>;
+    startPlaywrightCodegen(page: Page): Promise<void>;
+    recordAction(): Promise<void>;
+    getCodegenResult(): Promise<string>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/cua/pw-codegen/pw-pause/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/agent/cua/pw-codegen/pw-pause/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AAEjD,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAEtC,wBAAsB,2BAA2B,CAAC,OAAO,EAAE,MAAM,iBAoBhE;AAGD,qBAAa,sBAAsB;IACjC,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,IAAI,CAAmB;IAC/B,OAAO,CAAC,MAAM,CAA4C;;IAMpD,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBrC,sBAAsB,CAAC,IAAI,EAAE,IAAI;IAsBjC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAI1C"}

package/dist/agent/cua/pw-codegen/pw-pause/index.js

@@ -0,0 +1,84 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PlaywrightPauseCodegen = exports.PW_PAUSE_IPC_PORT = exports.revertToOriginalPwCode = void 0;
+exports.preparePlaywrightForCodegen = preparePlaywrightForCodegen;
+const express_1 = __importDefault(require("express"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const utils_1 = require("./utils");
+var utils_2 = require("./utils");
+Object.defineProperty(exports, "revertToOriginalPwCode", { enumerable: true, get: function () { return utils_2.revertToOriginalPwCode; } });
+exports.PW_PAUSE_IPC_PORT = 3039;
+async function preparePlaywrightForCodegen(repoDir) {
+    const npmListOutput = await (0, utils_1.runNpmList)(repoDir);
+    if (!npmListOutput) {
+        return;
+    }
+    const playwrightCorePath = (0, utils_1.getPlaywrightCoreFromNpmList)(npmListOutput);
+    const pathToRecorderApp = path_1.default.join(playwrightCorePath, "lib", "server", "recorder", "recorderApp.js");
+    if (!fs_1.default.existsSync(pathToRecorderApp)) {
+        const errMsg = `Cannot patch Playwright: ${pathToRecorderApp} does not exist`;
+        throw new Error(errMsg);
+    }
+    await (0, utils_1.createFileBackup)(pathToRecorderApp);
+    const port = exports.PW_PAUSE_IPC_PORT;
+    await (0, utils_1.patchPwCode)(pathToRecorderApp, port);
+}
+// TODO: Fix this to implement BasePlaywrightCodegen
+class PlaywrightPauseCodegen {
+    port = 0;
+    page;
+    server;
+    constructor() {
+        this.port = exports.PW_PAUSE_IPC_PORT;
+    }
+    async initialize(page) {
+        // Start server to receive generated code from patch
+        const app = (0, express_1.default)();
+        app.use(express_1.default.json());
+        app.post("/test", async (req, res) => {
+            console.log("--- Received request in PlaywrightPatchCodegen ---");
+            console.log(req.body);
+            return res.send({ success: true });
+        });
+        await new Promise((resolve) => {
+            this.server = app.listen(this.port, () => resolve());
+        });
+        console.log(`Server started on port ${this.port}`);
+        // Start page.pause experience in the page
+        this.page = page;
+        await this.startPlaywrightCodegen(page);
+    }
+    async startPlaywrightCodegen(page) {
+        // TODO: Glass pane needs to be removed on every page load
+        // We use bindings that Playwright exposes to the page
+        // Ref: https://github.com/microsoft/playwright/blob/e1c8e0f6b33923c95cc4b9416aefa6977b1d3c55/packages/playwright-core/src/server/recorder.ts#L191
+        await page.evaluate(() => {
+            setTimeout(() => {
+                // First, we start recording
+                // @ts-ignore
+                console.log(window["__pw_recorderSetMode"]("recording"));
+                // Then, we will resume the effect of pause()
+                // @ts-ignore
+                console.log(window["__pw_resume"]());
+                // Then, we remove highlights that Playwright shows on the screen
+                const glassPane = document.querySelector("x-pw-glass");
+                if (glassPane) {
+                    glassPane.remove();
+                }
+            }, 1000);
+        });
+        await page.pause();
+    }
+    async recordAction() {
+        console.log("Record action is no-op");
+    }
+    async getCodegenResult() {
+        console.log("Get codegen result is no-op");
+        return "";
+    }
+}
+exports.PlaywrightPauseCodegen = PlaywrightPauseCodegen;

package/dist/agent/cua/pw-codegen/pw-pause/utils.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Helpful commands for debugging:
+ *
+ * 1. Find playwright-core path
+ * cd `npm ls playwright-core --json --long | jq -r '.dependencies["@playwright/test"].dependencies.playwright.dependencies["playwright-core"].path'`
+ *
+ * 2. Find the path to the recorder app
+ * cat lib/server/recorder/recorderApp.js
+ *
+ */
+export declare function runNpmList(repoDir: string): Promise<string | undefined>;
+export declare function getPlaywrightCoreFromNpmList(output: string): any;
+export declare function revertToOriginalPwCode(repoDir: string): Promise<void>;
+export declare function createFileBackup(pathToFile: string): Promise<void>;
+export declare function patchPwCode(pathToRecorderApp: string, port: number): Promise<void>;
+//# sourceMappingURL=utils.d.ts.map

package/dist/agent/cua/pw-codegen/pw-pause/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../../src/agent/cua/pw-codegen/pw-pause/utils.ts"],"names":[],"mappings":"AAIA;;;;;;;;;GASG;AAEH,wBAAsB,UAAU,CAAC,OAAO,EAAE,MAAM,+BAiB/C;AAED,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,OAuB1D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,iBAoB3D;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,iBAGxD;AAED,wBAAsB,WAAW,CAAC,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,iBAkBxE"}

package/dist/agent/cua/pw-codegen/pw-pause/utils.js

@@ -0,0 +1,98 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runNpmList = runNpmList;
+exports.getPlaywrightCoreFromNpmList = getPlaywrightCoreFromNpmList;
+exports.revertToOriginalPwCode = revertToOriginalPwCode;
+exports.createFileBackup = createFileBackup;
+exports.patchPwCode = patchPwCode;
+const child_process_1 = require("child_process");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+/**
+ * Helpful commands for debugging:
+ *
+ * 1. Find playwright-core path
+ * cd `npm ls playwright-core --json --long | jq -r '.dependencies["@playwright/test"].dependencies.playwright.dependencies["playwright-core"].path'`
+ *
+ * 2. Find the path to the recorder app
+ * cat lib/server/recorder/recorderApp.js
+ *
+ */
+async function runNpmList(repoDir) {
+    try {
+        const output = (0, child_process_1.execSync)("npm ls playwright-core --json --long --production=false", {
+            cwd: repoDir,
+            encoding: "utf-8",
+        });
+        return output;
+    }
+    catch (error) {
+        console.error("Command failed:", error.message);
+        console.error("Error code:", error.status);
+        console.error("Signal:", error.signal);
+        console.error("Stdout:", error.stdout?.toString());
+        console.error("Stderr:", error.stderr?.toString());
+    }
+}
+function getPlaywrightCoreFromNpmList(output) {
+    const result = JSON.parse(output);
+    // We expect to find playwright-core as a dependency of playwright, which is
+    // a dependency of @playwright/test
+    const { dependencies } = result;
+    const playwrightTest = dependencies["@playwright/test"];
+    if (!playwrightTest) {
+        throw new Error("Could not find @playwright/test in npm dependencies");
+    }
+    const playwrightTestVersion = playwrightTest.version;
+    if (playwrightTestVersion !== "1.47.1") {
+        throw new Error("Unsupported playwright-test version: ${playwrightTestVersion}");
+    }
+    const playwrightDeps = playwrightTest.dependencies["playwright"];
+    const playwrightCoreDep = playwrightDeps.dependencies["playwright-core"];
+    if (!playwrightCoreDep) {
+        throw new Error("Could not find playwright-core in @playwright/test dependencies");
+    }
+    return playwrightCoreDep.path;
+}
+async function revertToOriginalPwCode(repoDir) {
+    try {
+        const npmListOutput = await runNpmList(repoDir);
+        if (!npmListOutput) {
+            return;
+        }
+        const playwrightCorePath = getPlaywrightCoreFromNpmList(npmListOutput);
+        const pathToRecorderApp = path_1.default.join(playwrightCorePath, "lib", "server", "recorder", "recorderApp.js");
+        const backupPath = `${pathToRecorderApp}.backup`;
+        fs_1.default.copyFileSync(backupPath, pathToRecorderApp);
+        fs_1.default.unlinkSync(backupPath);
+    }
+    catch (error) {
+        console.error("Error reverting to original Playwright code", error);
+    }
+}
+async function createFileBackup(pathToFile) {
+    const backupPath = `${pathToFile}.backup`;
+    fs_1.default.copyFileSync(pathToFile, backupPath);
+}
+async function patchPwCode(pathToRecorderApp, port) {
+    const currentFileContents = fs_1.default.readFileSync(pathToRecorderApp, "utf-8");
+    const original = `async setSources(sources) {}`;
+    // TODO: Can we have a request payload and parse it in our code
+    const replacement = `async setSources(sources) {
+    const payload = JSON.stringify(sources.filter(source => source.isRecorded).filter(source => source.id === "javascript").map(source => source.actions));
+    const body = JSON.stringify({ payload });
+    await fetch("http://localhost:${port}/test", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body,
+    });
+  }`;
+    const newCode = currentFileContents.replace(original, replacement);
+    fs_1.default.writeFileSync(pathToRecorderApp, newCode);
+    console.log(`Patched Playwright recorder app at ${pathToRecorderApp}`);
+}

package/dist/agent/cua/pw-codegen/types.d.ts

@@ -0,0 +1,46 @@
+import type { Page } from "playwright";
+export type Action = {
+    type: "click";
+    x: number;
+    y: number;
+} | {
+    type: "doubleclick";
+    x: number;
+    y: number;
+} | {
+    type: "back";
+} | {
+    type: "forward";
+} | {
+    type: "move";
+    x: number;
+    y: number;
+} | {
+    type: "drag";
+    path: {
+        x: number;
+        y: number;
+    }[];
+} | {
+    type: "scroll";
+    x: number;
+    y: number;
+    scroll_x: number;
+    scroll_y: number;
+} | {
+    type: "keypress";
+    keys: string[];
+} | {
+    type: "type";
+    text: string;
+} | {
+    type: "goto";
+    url: string;
+} | {
+    type: "wait";
+};
+export interface BasePlaywrightCodegen {
+    initialize(page: Page): Promise<void>;
+    codeForAction(action: Action): Promise<string>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/agent/cua/pw-codegen/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/agent/cua/pw-codegen/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,MAAM,MAAM,MAAM,GACd;IACE,IAAI,EAAE,OAAO,CAAC;IACd,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX,GACD;IACE,IAAI,EAAE,aAAa,CAAC;IACpB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX,GACD;IACE,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,SAAS,CAAC;CACjB,GACD;IACE,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX,GACD;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClC,GACD;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,GACD;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb,GACD;IACE,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAChD"}

package/dist/agent/cua/pw-codegen/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/agent/master/browser-tests/cua.spec.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const run_1 = require("../run");
 const fixtures_1 = require("./fixtures");
-(0, fixtures_1.test)("computer use agent generates code for iframes", async ({ page, server, }) => {
+fixtures_1.test.skip("computer use agent generates code for iframes", async ({ page, server, }) => {
     await page.goto(`${server.baseURL}/iframe-elements.html`);
     const result = await (0, run_1.createTestUsingComputerUseAgent)({
         page,
@@ -11,3 +11,15 @@ const fixtures_1 = require("./fixtures");
     (0, fixtures_1.expect)(result.code).toMatch(/page\.getByPlaceholder\(['"]Enter your name['"]?\)\.click/);
     (0, fixtures_1.expect)(result.code).toMatch(/locator\(['"]#nested-frame['"]?\)\.contentFrame\(\)\.getByPlaceholder\(['"]Enter your name['"]?\)/);
 });
+fixtures_1.test.skip("computer use agent generates code for google login", async ({ page, }) => {
+    await page.goto(`https://dash.empirical.run`);
+    const result = await (0, run_1.createTestUsingComputerUseAgent)({
+        page,
+        task: `Click on the "Sign in with Google" button and enter dpdzero-test-user@empirical.run
+as the email and click continue. then enter password as 1234556789`,
+    });
+    console.log(result.code);
+    (0, fixtures_1.expect)(result.code).toMatch(/page\.getByRole\('button', { name: 'Sign in with Google' }\)\.click/);
+    (0, fixtures_1.expect)(result.code).toContain("await page.getByLabel('Email or phone').fill(\"dpdzero-test-user@empirical.run\")");
+    (0, fixtures_1.expect)(result.code).toContain("await page.getByRole('button', { name: 'Next' }).click()");
+});

package/dist/artifacts/index.d.ts

@@ -0,0 +1,43 @@
+import { Artifact, ArtifactInput } from "@empiricalrun/shared-types";
+export declare function isArtifactCollectionEnabled(): string | undefined;
+/**
+ * Collects artifacts from the repository directory and returns their URLs.
+ *
+ * @param inputs - An array of artifact inputs.
+ * @param repoDir - The absolute path to the repository directory.
+ * @param toolCallId - Unique identifier for the tool call.
+ * @returns An array of artifacts with their URLs.
+ *
+ * @example
+ * // Collect artifacts from both file paths and in-memory data
+ * const artifacts = await collectArtifacts([
+ *   {
+ *     name: "screenshot.png",
+ *     contentType: "image/png",
+ *     path: "/Users/project/screenshots/test.png"
+ *   },
+ *   {
+ *     name: "log.txt",
+ *     contentType: "text/plain",
+ *     data: Buffer.from("test log content")
+ *   }
+ * ], "/Users/project", "tool-123");
+ *
+ * @example
+ * // Returns empty array when artifact collection is disabled
+ * const artifacts = await collectArtifacts([], "/Users/project", "tool-123");
+ * // Returns: []
+ *
+ * @example
+ * // Throws error for invalid file paths
+ * await collectArtifacts([
+ *   {
+ *     name: "test.png",
+ *     contentType: "image/png",
+ *     path: "relative/path/test.png" // Invalid - not absolute path
+ *   }
+ * ], "/Users/project", "tool-123");
+ * // Throws: "Invalid path: relative/path/test.png..."
+ */
+export declare function collectArtifacts(inputs: ArtifactInput[], repoDir: string, toolCallId: string): Promise<Artifact[]>;
+//# sourceMappingURL=index.d.ts.map

package/dist/artifacts/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,QAAQ,EACR,aAAa,EAGd,MAAM,4BAA4B,CAAC;AAkBpC,wBAAgB,2BAA2B,uBAM1C;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,EAAE,EACvB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6DrB"}