@empiricalrun/test-gen 0.53.12 → 0.54.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @empiricalrun/test-gen
 
+## 0.54.0
+
+### Minor Changes
+
+- 8324aa6: chore: update auth header for dashboard requests
+
+### Patch Changes
+
+- 769c3e7: feat: Implement environment-aware authentication with fallback
+- 4297ed5: chore: Move text editor tools from llm to test-gen package
+- 2ecdd64: fix: invalid tool calls should pipe back into the llm
+- a8f135e: fix: add package-lock.json to default exclude for grep tool
+- Updated dependencies [4297ed5]
+  - @empiricalrun/llm@0.14.8
+
+## 0.53.13
+
+### Patch Changes
+
+- b847558: feat: cua codegen works for clicks in iframes
+- bb402a4: feat: accept framelocators in test-gen tool calls
+- aa38bee: feat: add valid playwright project names to system prompt
+
 ## 0.53.12
 
 ### Patch Changes

package/dist/agent/chat/agent-loop.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAEL,UAAU,EAMX,MAAM,wBAAwB,CAAC;AAYhC,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAyChE,wBAAsB,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,QAAQ,EACR,KAAK,GACN,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IAC3B,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,iBA2CA"}
+{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,UAAU,EAIX,MAAM,wBAAwB,CAAC;AAiBhC,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAyChE,wBAAsB,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,QAAQ,EACR,KAAK,GACN,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IAC3B,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,iBAiDA"}

package/dist/agent/chat/agent-loop.js

@@ -7,6 +7,7 @@ const web_1 = require("../../bin/utils/platform/web");
 const commit_and_create_pr_1 = require("../../tools/commit-and-create-pr");
 const diagnosis_fetcher_1 = require("../../tools/diagnosis-fetcher");
 const grep_1 = require("../../tools/grep");
+const str_replace_editor_1 = require("../../tools/str_replace_editor");
 const test_gen_browser_1 = require("../../tools/test-gen-browser");
 const test_run_1 = require("../../tools/test-run");
 const test_run_fetcher_1 = require("../../tools/test-run-fetcher");
@@ -23,17 +24,17 @@ function getTools(selectedModel) {
     ];
     if (selectedModel.startsWith("gemini")) {
         // Claude will have its own built-in text editor tools
-        chat_1.textEditorTools.forEach((tool) => {
+        str_replace_editor_1.textEditorTools.forEach((tool) => {
             const originalExecute = tool.execute;
             tool.execute = (input) => originalExecute(input, web_1.validateTypescript);
         });
-        tools.push(...chat_1.textEditorTools);
+        tools.push(...str_replace_editor_1.textEditorTools);
     }
     const toolExecutors = {
         ...Object.fromEntries(tools.map((tool) => [tool.schema.name, tool.execute])),
     };
     if (selectedModel.startsWith("claude")) {
-        toolExecutors.str_replace_editor = (input) => (0, chat_1.strReplaceEditorExecutor)(input, web_1.validateTypescript);
+        toolExecutors.str_replace_editor = (input) => (0, str_replace_editor_1.strReplaceEditorExecutor)(input, web_1.validateTypescript);
     }
     return { tools, toolExecutors };
 }
@@ -49,6 +50,7 @@ const log = (...args) => {
 };
 async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, }) {
     const systemPrompt = await (0, prompt_1.buildSystemPrompt)();
+    trace?.update({ input: { systemPrompt } });
     const { tools, toolExecutors } = getTools(selectedModel);
     while (!chatModel.askUserForInput) {
         const toolCalls = chatModel.getPendingToolCalls();
@@ -58,15 +60,21 @@ async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, }) {
                 const args = JSON.stringify(call.input);
                 log(`Executing tool ${call.name} with args: ${args}`);
                 const toolExecutor = toolExecutors[call.name];
+                let callResponse;
                 if (!toolExecutor) {
-                    throw new Error(`Tool ${call.name} not found`);
-                }
-                const callResponse = await toolExecutor(call.input);
-                if (callResponse.isError) {
-                    log(`Tool ${call.name} failed: ${callResponse.result}`);
+                    callResponse = {
+                        isError: true,
+                        result: `Invalid function/tool call: ${call.name} not found`,
+                    };
                 }
                 else {
-                    log(`Tool ${call.name} completed`);
+                    callResponse = await toolExecutor(call.input);
+                    if (callResponse.isError) {
+                        log(`Tool ${call.name} failed: ${callResponse.result}`);
+                    }
+                    else {
+                        log(`Tool ${call.name} completed`);
+                    }
                 }
                 toolResults.push(callResponse);
             }
@@ -86,6 +94,6 @@ async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, }) {
         const latest = chatModel.getHumanReadableLatestMessage();
         await reporter((0, state_1.chatStateFromModel)(chatModel, selectedModel), latest);
     }
-    (0, chat_1.cleanupBackupFiles)(process.cwd());
+    (0, str_replace_editor_1.cleanupBackupFiles)(process.cwd());
 }
 exports.chatAgentLoop = chatAgentLoop;

package/dist/agent/chat/index.js

@@ -106,7 +106,7 @@ async function getChatSessionFromDashboard(chatSessionId) {
     const response = await fetch(`${DASHBOARD_DOMAIN}/api/chat-sessions/${chatSessionId}`, {
         headers: {
             "Content-Type": "application/json",
-            Authorization: `weQPMWKT`,
+            Authorization: `Bearer ${process.env.EMPIRICALRUN_API_KEY}`,
         },
     });
     const data = await response.json();
@@ -136,7 +136,7 @@ async function runChatAgentForDashboard({ chatSessionId, selectedModel, }) {
             }),
             headers: {
                 "Content-Type": "application/json",
-                Authorization: `weQPMWKT`,
+                Authorization: `Bearer ${process.env.EMPIRICALRUN_API_KEY}`,
             },
         });
         const data = await response.json();

package/dist/agent/chat/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/repo.ts"],"names":[],"mappings":"AAmCA,wBAAsB,cAAc,oBAwBnC"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/repo.ts"],"names":[],"mappings":"AAqCA,wBAAsB,cAAc,oBAyCnC"}

package/dist/agent/chat/repo.js

@@ -7,6 +7,7 @@ exports.getRepoContext = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 const repo_tree_1 = require("../../utils/repo-tree");
+const utils_1 = require("../browsing/utils");
 async function getAllMarkdownFiles() {
     const dir = path_1.default.join(process.cwd(), ".empiricalrun");
     if (!fs_extra_1.default.existsSync(dir)) {
@@ -48,6 +49,22 @@ While specifying paths to files, use relative paths from the current working dir
 - Correct path: "tests/lesson.spec.ts"
 - Incorrect path: "/repo/tests/lesson.spec.ts" or "${path_1.default.basename(process.cwd())}/tests/lesson.spec.ts"
 `;
+    try {
+        const playwrightConfig = await (0, utils_1.readPlaywrightConfig)(process.cwd());
+        const validProjectNames = await (0, utils_1.getValidProjectNames)(playwrightConfig);
+        if (validProjectNames.length > 0) {
+            REPO_CONTEXT_PROMPT += `
+    ## Playwright configuration
+    
+    This repo is configured with these Playwright projects (in the playwright.config.ts file):
+    ${validProjectNames.map((name) => `- ${name}`).join("\n")}
+    
+    `;
+        }
+    }
+    catch (error) {
+        console.warn("Failed to read playwright config", error);
+    }
     const knowledge = await knowledgeContext();
     if (knowledge.length > 0) {
         REPO_CONTEXT_PROMPT += `

package/dist/agent/cua/computer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,KAAK,cAAc,GACf,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAG7C;AAgCD,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAMD;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAqID"}
+{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIvC,KAAK,cAAc,GACf,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAG7C;AAgCD,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAMD;AA4DD,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAiID"}

package/dist/agent/cua/computer.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.handleModelAction = exports.handlePageGoto = exports.getScreenshot = void 0;
+const utils_1 = require("../browsing/utils");
 async function getScreenshot(page) {
     const screenshotBytes = await page.screenshot();
     return Buffer.from(screenshotBytes).toString("base64");
@@ -43,6 +44,53 @@ async function handlePageGoto(page, url) {
     };
 }
 exports.handlePageGoto = handlePageGoto;
+async function getLocatorForClick(page, { x, y }) {
+    const locator = await page.evaluate(([x, y]) => {
+        const element = document.elementFromPoint(x, y);
+        const bbox = element?.getBoundingClientRect();
+        return {
+            locator: window.playwright.generateLocator(element),
+            isIframe: element?.tagName === "IFRAME",
+            x0: bbox?.x,
+            y0: bbox?.y,
+            src: element?.getAttribute("src"),
+            id: element?.getAttribute("id"),
+        };
+    }, [x, y]);
+    if (!locator.isIframe) {
+        return locator.locator;
+    }
+    await (0, utils_1.injectPwLocatorGenerator)(page);
+    const relativeX = x - locator.x0;
+    const relativeY = y - locator.y0;
+    // TODO: Reuse locator.locator for the frameLocator.
+    let frameEl = locator.src
+        ? page.locator(`[src="${locator.src}"]`)
+        : page.locator(`#${locator.id}`);
+    const frameLocator = locator.src
+        ? `locator('[src="${locator.src}"]')`
+        : `locator('#${locator.id}')`;
+    const elementLocatorInsideFrame = await frameEl
+        .contentFrame()
+        .locator(":root")
+        .evaluate((rootElement, coords) => {
+        const xPos = coords[0];
+        const yPos = coords[1];
+        if (xPos === undefined || yPos === undefined) {
+            throw new Error("Coordinates are undefined in evaluate call");
+        }
+        const element = document.elementFromPoint(xPos, yPos);
+        return window.playwright.generateLocator(element);
+    }, [relativeX, relativeY]);
+    return `${frameLocator}.contentFrame().${elementLocatorInsideFrame}`;
+}
+async function getLocatorForFill(page) {
+    const locator = await page.evaluate(() => {
+        const element = document.activeElement;
+        return window.playwright.generateLocator(element);
+    });
+    return locator;
+}
 async function handleModelAction(page, action) {
     const actionType = action.type;
     let actionCode = "";
@@ -61,19 +109,18 @@ async function handleModelAction(page, action) {
                     pwButton = "middle";
                 }
                 if (pwButton) {
-                    const locator = await page.evaluate(([x, y]) => {
-                        const element = document.elementFromPoint(x, y);
-                        return window.playwright.generateLocator(element);
-                    }, [x, y]);
+                    const locator = await getLocatorForClick(page, { x, y });
                     actionCode = `await page.${locator}.click();\n`;
                     await page.mouse.click(x, y, { button: pwButton });
                 }
                 if (button === "back" || button === "forward") {
                     // Do page navigations, since there is no way to click on the back/forward buttons
                     if (button === "back") {
+                        actionCode = `await page.goBack();\n`;
                         await page.goBack();
                     }
                     else if (button === "forward") {
+                        actionCode = `await page.goForward();\n`;
                         await page.goForward();
                     }
                 }
@@ -83,6 +130,8 @@ async function handleModelAction(page, action) {
                 const { x, y } = action;
                 console.log(`Action: doubleclick at (${x}, ${y})`);
                 actionSummary = `Double click at (${x}, ${y})`;
+                const locator = await getLocatorForClick(page, { x, y });
+                actionCode = `await page.${locator}.dblclick();\n`;
                 await page.mouse.dblclick(x, y, { button: "left" });
                 break;
             }
@@ -96,6 +145,7 @@ async function handleModelAction(page, action) {
             case "drag": {
                 const { path } = action;
                 console.log(`Action: drag along path ${path}`);
+                // TODO: actionCode is not implemented
                 actionSummary = `Drag along path ${path}`;
                 if (!path || path.length === 0) {
                     break;
@@ -138,10 +188,7 @@ async function handleModelAction(page, action) {
                 console.log(`Action: type text '${text}'`);
                 actionSummary = `Type text '${text}'`;
                 await page.keyboard.type(text);
-                const locator = await page.evaluate(() => {
-                    const element = document.activeElement;
-                    return window.playwright.generateLocator(element);
-                });
+                const locator = await getLocatorForFill(page);
                 actionCode = `await page.${locator}.fill("${text}");\n`;
                 break;
             }

package/dist/agent/master/browser-tests/cua.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cua.spec.d.ts.map

package/dist/agent/master/browser-tests/cua.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cua.spec.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/browser-tests/cua.spec.ts"],"names":[],"mappings":""}

package/dist/agent/master/browser-tests/cua.spec.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const run_1 = require("../run");
+const fixtures_1 = require("./fixtures");
+(0, fixtures_1.test)("computer use agent generates code for iframes", async ({ page, server, }) => {
+    await page.goto(`${server.baseURL}/iframe-elements.html`);
+    const result = await (0, run_1.createTestUsingComputerUseAgent)({
+        page,
+        task: `Click on both of the "Enter your name" input fields`,
+    });
+    (0, fixtures_1.expect)(result.code).toMatch(/page\.getByPlaceholder\(['"]Enter your name['"]?\)\.click/);
+    (0, fixtures_1.expect)(result.code).toMatch(/locator\(['"]#nested-frame['"]?\)\.contentFrame\(\)\.getByPlaceholder\(['"]Enter your name['"]?\)/);
+});

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { Page } from "playwright";
+import { FrameLocator, Page } from "playwright";
 import { ScopeVars } from "./types";
-export declare function createTest(task: string, page: Page, scope?: ScopeVars): Promise<void>;
+export declare function createTest(task: string, pageRef: Page | FrameLocator, scope?: ScopeVars): Promise<void>;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAoBpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAsD3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQhD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAoBpC,wBAAsB,UAAU,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,IAAI,GAAG,YAAY,EAC5B,KAAK,CAAC,EAAE,SAAS,iBAwDlB"}

package/dist/index.js

@@ -21,11 +21,12 @@ function setupProcessListeners(cleanup) {
         events.forEach((event) => process.removeListener(event, cleanup));
     };
 }
-async function createTest(task, page, scope) {
+async function createTest(task, pageRef, scope) {
     const removeListeners = setupProcessListeners(flushEvents);
     try {
         const testConfigArg = process.env.TEST_GEN_TOKEN;
         const testGenConfig = (0, scenarios_1.loadTestConfigs)(testConfigArg);
+        const useComputerUseAgent = testGenConfig.options?.useComputerUseAgent;
         if (testGenConfig.options && testGenConfig.options.metadata) {
             (0, reporter_1.setReporterConfig)({
                 projectRepoName: testGenConfig.options?.metadata.projectRepoName,
@@ -39,8 +40,9 @@ async function createTest(task, page, scope) {
                 projectRepoName: testGenConfig.options?.metadata.projectRepoName,
             });
         }
+        // pageRef can be a FrameLocator, in which case we need to get the Page where the iframe is located
+        let page = "owner" in pageRef ? pageRef.owner().page() : pageRef;
         const fileServiceClient = new client_1.default();
-        const useComputerUseAgent = testGenConfig.options?.useComputerUseAgent;
         let agentResult;
         if (useComputerUseAgent) {
             agentResult = await (0, cua_1.createTestUsingComputerUseAgent)({

package/dist/reporter/lib.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lib.d.ts","sourceRoot":"","sources":["../../src/reporter/lib.ts"],"names":[],"mappings":"AAMA,KAAK,uBAAuB,GAAG;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EACP,SAAS,GACT,aAAa,GACb,kBAAkB,GAClB,OAAO,GACP,UAAU,GACV,SAAS,GACT,WAAW,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,qBAAa,cAAc;IAEvB,OAAO,CAAC,KAAK;gBAAL,KAAK,EAAE;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,uBAAuB,CAAC,aAAa,CAAC,CAAC;KAC9C;IAGH,YAAY,CAAC,gBAAgB,CAAC,EAAE;QAC9B,QAAQ,CAAC,EAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;KAC7C,GAAG,uBAAuB;CAO5B;AAED,qBAAa,QAAQ;IAEjB,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE;QACd,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;KACtB;IAGG,MAAM,CAAC,cAAc,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;YAc7C,sBAAsB;CAmCrC"}
+{"version":3,"file":"lib.d.ts","sourceRoot":"","sources":["../../src/reporter/lib.ts"],"names":[],"mappings":"AAMA,KAAK,uBAAuB,GAAG;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EACP,SAAS,GACT,aAAa,GACb,kBAAkB,GAClB,OAAO,GACP,UAAU,GACV,SAAS,GACT,WAAW,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,qBAAa,cAAc;IAEvB,OAAO,CAAC,KAAK;gBAAL,KAAK,EAAE;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,uBAAuB,CAAC,aAAa,CAAC,CAAC;KAC9C;IAGH,YAAY,CAAC,gBAAgB,CAAC,EAAE;QAC9B,QAAQ,CAAC,EAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;KAC7C,GAAG,uBAAuB;CAO5B;AAED,qBAAa,QAAQ;IAEjB,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE;QACd,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;KACtB;IAGG,MAAM,CAAC,cAAc,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;YAc7C,sBAAsB;CAkCrC"}

package/dist/reporter/lib.js

@@ -51,8 +51,7 @@ class Reporter {
                     method: "POST",
                     headers: {
                         "Content-Type": "application/json",
-                        // TODO: fix this with authentication of github updates api
-                        Authorization: "weQPMWKT",
+                        Authorization: `Bearer ${process.env.EMPIRICALRUN_API_KEY}`,
                     },
                     body,
                 });

package/dist/session/index.js

@@ -47,7 +47,7 @@ async function getSessionState() {
         method: "GET",
         headers: {
             "Content-Type": "application/json",
-            Authorization: "weQPMWKT",
+            Authorization: `Bearer ${process.env.EMPIRICALRUN_API_KEY}`,
         },
     });
     const generationStateData = (await response.json());
@@ -69,7 +69,7 @@ async function endSession() {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
-                Authorization: "weQPMWKT",
+                Authorization: `Bearer ${process.env.EMPIRICALRUN_API_KEY}`,
             },
             body: JSON.stringify({
                 state: {
@@ -93,7 +93,7 @@ async function updateSessionStatus(sessionId, payload) {
             method: "PATCH",
             headers: {
                 "Content-Type": "application/json",
-                Authorization: "weQPMWKT",
+                Authorization: `Bearer ${process.env.EMPIRICALRUN_API_KEY}`,
             },
             body,
         });

package/dist/tools/str_replace_editor.d.ts

@@ -0,0 +1,23 @@
+import { Tool, ToolResult } from "@empiricalrun/llm/chat";
+interface StrReplaceInput {
+    command: string;
+    path: string;
+    view_range?: [number, number];
+    old_str?: string;
+    new_str?: string;
+    file_text?: string;
+    insert_line?: number;
+}
+/**
+ * Cleans up any backup files that were created during the editing process
+ * @returns The number of backup files that were cleaned up
+ */
+export declare function cleanupBackupFiles(repoDir: string): number;
+/**
+ * Our implementation of Claude's built-in text editor tool
+ * https://docs.anthropic.com/en/docs/build-with-claude/tool-use/text-editor-tool
+ */
+export declare function strReplaceEditorExecutor(input: StrReplaceInput, typeChecker?: (filePath: string) => string[]): Promise<ToolResult>;
+export declare const textEditorTools: Tool[];
+export {};
+//# sourceMappingURL=str_replace_editor.d.ts.map

package/dist/tools/str_replace_editor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"str_replace_editor.d.ts","sourceRoot":"","sources":["../../src/tools/str_replace_editor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAyB1D,UAAU,eAAe;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwC1D;AAMD;;;GAGG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,eAAe,EACtB,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,EAAE,GAC3C,OAAO,CAAC,UAAU,CAAC,CA8IrB;AA6GD,eAAO,MAAM,eAAe,EAAE,IAAI,EAKjC,CAAC"}

package/dist/tools/str_replace_editor.js

@@ -0,0 +1,304 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.textEditorTools = exports.strReplaceEditorExecutor = exports.cleanupBackupFiles = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const zod_1 = require("zod");
+function createBackup(filePath) {
+    const backupPath = `${filePath}.bak`;
+    if (fs_1.default.existsSync(filePath)) {
+        fs_1.default.copyFileSync(filePath, backupPath);
+    }
+}
+function hasBackup(filePath) {
+    const backupPath = `${filePath}.bak`;
+    return fs_1.default.existsSync(backupPath);
+}
+function restoreBackup(filePath) {
+    const backupPath = `${filePath}.bak`;
+    if (fs_1.default.existsSync(backupPath)) {
+        fs_1.default.copyFileSync(backupPath, filePath);
+        fs_1.default.unlinkSync(backupPath);
+    }
+}
+/**
+ * Cleans up any backup files that were created during the editing process
+ * @returns The number of backup files that were cleaned up
+ */
+function cleanupBackupFiles(repoDir) {
+    let cleanedCount = 0;
+    const walkDir = (dir) => {
+        let files = [];
+        try {
+            files = fs_1.default.readdirSync(dir);
+        }
+        catch (readDirError) {
+            return;
+        }
+        for (const file of files) {
+            const fullPath = path_1.default.join(dir, file);
+            let stat;
+            try {
+                stat = fs_1.default.statSync(fullPath);
+            }
+            catch (statError) {
+                continue;
+            }
+            if (stat.isDirectory()) {
+                if (file !== "node_modules") {
+                    walkDir(fullPath);
+                }
+            }
+            else if (file.endsWith(".bak")) {
+                try {
+                    fs_1.default.unlinkSync(fullPath);
+                    cleanedCount++;
+                }
+                catch (unlinkError) {
+                    // Intentionally ignore errors during cleanup
+                }
+            }
+        }
+    };
+    try {
+        walkDir(repoDir);
+    }
+    catch (error) {
+        // Intentionally ignore errors during cleanup
+    }
+    return cleanedCount;
+}
+exports.cleanupBackupFiles = cleanupBackupFiles;
+function escapeRegExp(text) {
+    return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
+}
+/**
+ * Our implementation of Claude's built-in text editor tool
+ * https://docs.anthropic.com/en/docs/build-with-claude/tool-use/text-editor-tool
+ */
+async function strReplaceEditorExecutor(input, typeChecker) {
+    const { path: filePath } = input;
+    try {
+        let content;
+        let lines;
+        let newContent;
+        switch (input.command) {
+            case "view":
+                // TODO: This assumes repoDir is process.cwd()
+                if (!fs_1.default.existsSync(filePath)) {
+                    return {
+                        result: "Error: File not found",
+                        isError: true,
+                    };
+                }
+                // Handle directory view
+                if (fs_1.default.statSync(filePath).isDirectory()) {
+                    const files = fs_1.default.readdirSync(filePath);
+                    return {
+                        result: files.join("\n"),
+                        isError: false,
+                    };
+                }
+                // Handle file view
+                content = fs_1.default.readFileSync(filePath, "utf8");
+                lines = content.split("\n");
+                if (input.view_range) {
+                    const [start, end] = input.view_range;
+                    const endLine = end === -1 ? lines.length : end;
+                    return {
+                        result: lines
+                            .slice(start - 1, endLine)
+                            .map((line, idx) => `${start + idx}: ${line}`)
+                            .join("\n"),
+                        isError: false,
+                    };
+                }
+                return {
+                    result: lines.map((line, idx) => `${idx + 1}: ${line}`).join("\n"),
+                    isError: false,
+                };
+            case "create":
+                if (input.file_text === undefined || input.file_text === null) {
+                    throw new Error("file_text is required for create command");
+                }
+                fs_1.default.writeFileSync(filePath, input.file_text);
+                return {
+                    result: `Successfully created file ${filePath}`,
+                    isError: false,
+                };
+            case "str_replace":
+                if (!input.old_str) {
+                    throw new Error("old_str is required for str_replace command");
+                }
+                if (input.new_str === undefined || input.new_str === null) {
+                    // "" is valid as new_str, so we check for nullish -- not falsy
+                    throw new Error("new_str is required for str_replace command");
+                }
+                createBackup(filePath);
+                content = fs_1.default.readFileSync(filePath, "utf8");
+                if (!content.includes(input.old_str)) {
+                    return {
+                        result: `old_str not found in file: ${filePath}`,
+                        isError: true,
+                    };
+                }
+                else {
+                    const escapedOldStr = escapeRegExp(input.old_str);
+                    const occurences = content.match(new RegExp(escapedOldStr, "g"));
+                    if (occurences && occurences.length > 1) {
+                        return {
+                            result: `Error: old_str found ${occurences.length} times in file: ${filePath}. Please provide more context to make a unique match.`,
+                            isError: true,
+                        };
+                    }
+                    newContent = content.replace(input.old_str, input.new_str);
+                    fs_1.default.writeFileSync(filePath, newContent);
+                    if (typeChecker) {
+                        const errors = typeChecker(filePath);
+                        if (errors.length > 0) {
+                            return {
+                                result: `Edits to file ${filePath} have been applied. However, type checks are failing with errors:\n${errors.join("\n")}`,
+                                isError: true,
+                            };
+                        }
+                        else {
+                            return {
+                                result: `Edits to file ${filePath} have been applied. Type checks have also passed.`,
+                                isError: false,
+                            };
+                        }
+                    }
+                    return {
+                        result: `Edits to file ${filePath} have been applied.`,
+                        isError: false,
+                    };
+                }
+            case "insert":
+                if (input.insert_line === undefined || !input.new_str) {
+                    throw new Error("insert_line and new_str are required for insert command");
+                }
+                createBackup(filePath);
+                content = fs_1.default.readFileSync(filePath, "utf8");
+                lines = content.split("\n");
+                lines.splice(input.insert_line, 0, input.new_str);
+                fs_1.default.writeFileSync(filePath, lines.join("\n"));
+                return {
+                    result: "Success",
+                    isError: false,
+                };
+            case "undo_edit":
+                if (hasBackup(filePath)) {
+                    restoreBackup(filePath);
+                    return {
+                        result: "Success",
+                        isError: false,
+                    };
+                }
+                return {
+                    result: "No backup file found",
+                    isError: true,
+                };
+            default:
+                throw new Error(`Unknown command: ${input.command}`);
+        }
+    }
+    catch (error) {
+        return {
+            result: error instanceof Error ? error.message : "Unknown error occurred",
+            isError: true,
+        };
+    }
+}
+exports.strReplaceEditorExecutor = strReplaceEditorExecutor;
+const fileViewTool = {
+    schema: {
+        name: "fileViewTool",
+        description: `A tool to view the content of a file or directory. If the path points 
+to a directory, the tool will return a list of files in the directory, separated by line breaks.
+If the path points to a file, the tool will return the content of the file. File contents
+are returned with line numbers, starting from 1.
+
+1: line 1
+2: line 2
+...`,
+        parameters: zod_1.z.object({
+            // Does not support view_range for now
+            path: zod_1.z.string().describe("The path to the file or directory to view."),
+        }),
+    },
+    execute: async (input) => {
+        return strReplaceEditorExecutor({
+            command: "view",
+            path: input.path,
+        });
+    },
+};
+const fileCreateTool = {
+    schema: {
+        name: "fileCreateTool",
+        description: "A tool to create a new file with given contents.",
+        parameters: zod_1.z.object({
+            path: zod_1.z.string().describe("The path to the new file."),
+            file_text: zod_1.z.string().describe("The contents of the new file."),
+        }),
+    },
+    execute: async (input, typeChecker) => {
+        return strReplaceEditorExecutor({
+            command: "create",
+            path: input.path,
+            file_text: input.file_text,
+        }, typeChecker);
+    },
+};
+const stringReplaceTool = {
+    schema: {
+        name: "stringReplaceTool",
+        description: `A tool to replace a string in a file. This tool requires old_str to be unique
+in the file. If old_str is not unique, the tool will return an error.`,
+        parameters: zod_1.z.object({
+            path: zod_1.z.string().describe("The path to the file."),
+            old_str: zod_1.z.string().describe("The string to be replaced."),
+            new_str: zod_1.z.string().describe("The string to replace old_str with."),
+        }),
+    },
+    execute: async (input, typeChecker) => {
+        return strReplaceEditorExecutor({
+            command: "str_replace",
+            path: input.path,
+            old_str: input.old_str,
+            new_str: input.new_str,
+        }, typeChecker);
+    },
+};
+const stringInsertTool = {
+    schema: {
+        name: "stringInsertTool",
+        description: "A tool to insert a string at a specific line in a file.",
+        parameters: zod_1.z.object({
+            path: zod_1.z.string().describe("The path to the file."),
+            insert_line: zod_1.z
+                .number()
+                .int()
+                .min(0)
+                .describe("The line number after which to insert the text (0 for beginning of file)."),
+            new_str: zod_1.z.string().describe("The string to insert."),
+        }),
+    },
+    execute: async (input, typeChecker) => {
+        return strReplaceEditorExecutor({
+            command: "insert",
+            path: input.path,
+            insert_line: input.insert_line,
+            new_str: input.new_str,
+        }, typeChecker);
+    },
+};
+exports.textEditorTools = [
+    fileViewTool,
+    fileCreateTool,
+    stringReplaceTool,
+    stringInsertTool,
+];

package/dist/tools/test-gen-browser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AA8EnD,eAAO,MAAM,4BAA4B,EAAE,IA0E1C,CAAC"}
+{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAyFnD,eAAO,MAAM,4BAA4B,EAAE,IA0E1C,CAAC"}

package/dist/tools/test-gen-browser.js

@@ -66,6 +66,17 @@ await extPage
 // Instead, add the TODO before or after the multi-line statement
 \`\`\`
 
+The TODO comment must be inside a test block, not outside of it. For example, this is invalid:
+
+\`\`\`
+// --- BEGIN INVALID EXAMPLE ---
+test("Example test code", async ({ page }) => {
+  await page.goto("https://example.com");
+});
+// TODO(agent on page): Click on the login button
+// --- END INVALID EXAMPLE ---
+\`\`\`
+
 After execution, the browser agent will return a summary of actions that it took, and the generated Playwright code for them.
 You can then use the text editor tool to replace the TODO comment with the generated Playwright code.
 `;

package/dist/tools/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/index.ts"],"names":[],"mappings":"AAAA,wBAAsB,oBAAoB,CAAC,CAAC,EAAE,EAC5C,IAAI,EACJ,MAAc,EACd,IAAI,GACL,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,GAAG,OAAO,CAAC,CAAC,CAAC,CAoBb;AAED,wBAAsB,eAAe,CAAC,EACpC,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,oBAWA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/index.ts"],"names":[],"mappings":"AAAA,wBAAsB,oBAAoB,CAAC,CAAC,EAAE,EAC5C,IAAI,EACJ,MAAc,EACd,IAAI,GACL,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,GAAG,OAAO,CAAC,CAAC,CAAC,CAmBb;AAED,wBAAsB,eAAe,CAAC,EACpC,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,oBAWA"}

package/dist/tools/utils/index.js

@@ -4,8 +4,7 @@ exports.callGitHubProxy = exports.makeDashboardRequest = void 0;
 async function makeDashboardRequest({ path, method = "GET", body, }) {
     const requestHeaders = {
         "Content-Type": "application/json",
-        // TODO: Move to env variable for authentication
-        Authorization: "weQPMWKT",
+        Authorization: `Bearer ${process.env.EMPIRICALRUN_API_KEY}`,
         "User-Agent": "empiricalrun/test-gen",
     };
     const baseUrl = "https://dash.empirical.run";

package/dist/utils/repo-tree.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo-tree.d.ts","sourceRoot":"","sources":["../../src/utils/repo-tree.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,eAAe,qBAU3B,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,KAAK,UAsE9D"}
+{"version":3,"file":"repo-tree.d.ts","sourceRoot":"","sources":["../../src/utils/repo-tree.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,eAAe,qBAW3B,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,KAAK,UAsE9D"}

package/dist/utils/repo-tree.js

@@ -16,6 +16,7 @@ exports.DEFAULT_EXCLUDE = [
     "test-results",
     ".empiricalrun",
     "auth",
+    "package-lock.json",
 ];
 function generateAsciiTree(dirPath, options = {}) {
     const defaultOptions = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.53.12",
+  "version": "0.54.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -56,7 +56,7 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "zod": "^3.23.8",
-    "@empiricalrun/llm": "^0.14.7",
+    "@empiricalrun/llm": "^0.14.8",
     "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/test-run": "^0.7.7"
   },