@empiricalrun/test-gen 0.61.0 → 0.62.0

package/dist/tool-call-service/index.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ToolCallService = void 0;
 const chat_1 = require("@empiricalrun/llm/chat");
+const artifacts_1 = require("../artifacts");
 const commit_and_create_pr_1 = require("../tools/commit-and-create-pr");
 const diagnosis_fetcher_1 = require("../tools/diagnosis-fetcher");
 const download_build_1 = require("../tools/download-build");
@@ -19,10 +20,16 @@ class ToolCallService {
     chatSessionId;
     selectedModel;
     branchName;
-    constructor(chatSessionId, selectedModel, branchName) {
+    repoPath;
+    apiKey;
+    trace;
+    constructor({ chatSessionId, selectedModel, branchName, repoPath, apiKey, trace, }) {
         this.chatSessionId = chatSessionId;
         this.selectedModel = selectedModel;
         this.branchName = branchName;
+        this.trace = trace;
+        this.repoPath = repoPath;
+        this.apiKey = apiKey;
         this.tools = [
             grep_1.grepTool,
             test_run_1.runTestTool,
@@ -33,8 +40,6 @@ class ToolCallService {
             environment_crud_1.getEnvironmentTool,
             download_build_1.downloadBuildTool,
         ];
-    }
-    async getTools() {
         if ((0, chat_1.getProviderForModel)(this.selectedModel) !== "claude") {
             this.tools.push(...str_replace_editor_1.textEditorTools);
         }
@@ -42,9 +47,13 @@ class ToolCallService {
             this.toolExecutors[tool.schema.name] = tool.execute;
         });
         if ((0, chat_1.getProviderForModel)(this.selectedModel) === "claude") {
-            this.toolExecutors["str_replace_editor"] = str_replace_editor_1.strReplaceEditorExecutor;
+            this.toolExecutors = {
+                ...this.toolExecutors,
+                // Support for Claude 3x and 4: They use different tool names
+                str_replace_editor: str_replace_editor_1.strReplaceEditorExecutor,
+                str_replace_based_edit_tool: str_replace_editor_1.strReplaceEditorExecutor,
+            };
         }
-        return { tools: this.tools };
     }
     async sendToQueue(toolCalls) {
         const requestId = toolCalls[0]?.id;
@@ -59,8 +68,8 @@ class ToolCallService {
             branchName: this.branchName,
         });
     }
-    async execute(toolCalls, trace) {
-        const executeSpan = trace?.span({
+    async execute(toolCalls) {
+        const executeSpan = this.trace?.span({
             name: "execute_tools",
             input: { toolCalls: toolCalls.map((tc) => ({ name: tc.name })) },
         });
@@ -75,13 +84,14 @@ class ToolCallService {
                 const errorResult = {
                     isError: true,
                     result: `Invalid function/tool call: ${toolCall.name} not found`,
+                    artifacts: null,
                 };
                 toolResults.push(errorResult);
                 span?.end({ output: errorResult });
                 continue;
             }
             try {
-                const result = await toolExecutor(toolCall.input, trace);
+                const result = await toolExecutor(toolCall.input, this.repoPath, this.apiKey, this.trace, artifacts_1.collectArtifacts);
                 toolResults.push(result);
                 span?.end({ output: result });
             }
@@ -89,6 +99,7 @@ class ToolCallService {
                 const errorResult = {
                     isError: true,
                     result: error instanceof Error ? error.message : String(error),
+                    artifacts: null,
                 };
                 toolResults.push(errorResult);
                 span?.end({ output: errorResult });

package/dist/tools/download-build.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download-build.d.ts","sourceRoot":"","sources":["../../src/tools/download-build.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,uBAAuB;;;;;;EAElC,CAAC;AAEH,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAE7E,eAAO,MAAM,iBAAiB,EAAE,IA8B/B,CAAC"}
+{"version":3,"file":"download-build.d.ts","sourceRoot":"","sources":["../../src/tools/download-build.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,uBAAuB;;;;;;EAElC,CAAC;AAEH,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAE7E,eAAO,MAAM,iBAAiB,EAAE,IAkC/B,CAAC"}

package/dist/tools/download-build.js

@@ -14,8 +14,8 @@ have a build URL, you can try getting the environment details with the getEnviro
 Environment details will include the build URL.`,
         parameters: exports.downloadBuildToolSchema,
     },
-    execute: async (input) => {
-        if (!(await (0, test_build_1.hasDownloadScript)())) {
+    execute: async (input, repoPath, apiKey) => {
+        if (!(await (0, test_build_1.hasDownloadScript)(repoPath))) {
             return {
                 isError: true,
                 result: `This repo does not have a download script in package.json.
@@ -24,7 +24,7 @@ You probably don't need to worry about this, since it means this repo does not h
         }
         const { buildUrl } = input;
         try {
-            await (0, test_build_1.downloadBuild)(buildUrl);
+            await (0, test_build_1.downloadBuild)({ buildUrl, repoPath, apiKey });
             return {
                 isError: false,
                 result: "Build downloaded successfully",

package/dist/tools/grep/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/grep/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAc,MAAM,wBAAwB,CAAC;AAyH/D,eAAO,MAAM,QAAQ,EAAE,IAmBtB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/grep/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAc,MAAM,wBAAwB,CAAC;AA2H/D,eAAO,MAAM,QAAQ,EAAE,IAmBtB,CAAC"}

package/dist/tools/grep/index.js

@@ -74,8 +74,10 @@ async function usingSystemGrep(input) {
 async function usingRipgrep(input) {
     try {
         const dir = path_1.default.join(process.cwd(), input.directory || "");
-        const escapedPattern = input.pattern.replace(/\s+/g, "\\ ");
-        const results = await (0, ripgrep_1.ripgrep)(dir, {
+        const escapedPattern = input.pattern
+            .replace(/\s+/g, "\\ ")
+            .replace(/([[\]()])/g, "\\$1");
+        const results = (0, ripgrep_1.ripgrep)(dir, {
             string: escapedPattern,
             globs: input.filePattern ? [input.filePattern] : undefined,
         });

package/dist/tools/str_replace_editor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"str_replace_editor.d.ts","sourceRoot":"","sources":["../../src/tools/str_replace_editor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AA2B1D,UAAU,eAAe;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAqED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwC1D;AAMD;;;GAGG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,UAAU,CAAC,CAuMrB;AAiGD,eAAO,MAAM,eAAe,EAAE,IAAI,EAKjC,CAAC"}
+{"version":3,"file":"str_replace_editor.d.ts","sourceRoot":"","sources":["../../src/tools/str_replace_editor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AA2B1D,UAAU,eAAe;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAqED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwC1D;AAMD;;;GAGG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,UAAU,CAAC,CAiNrB;AAiGD,eAAO,MAAM,eAAe,EAAE,IAAI,EAKjC,CAAC"}

package/dist/tools/str_replace_editor.js

@@ -140,6 +140,7 @@ function escapeRegExp(text) {
  * https://docs.anthropic.com/en/docs/build-with-claude/tool-use/text-editor-tool
  */
 async function strReplaceEditorExecutor(input) {
+    const repoDir = process.cwd();
     const { path: filePath } = input;
     try {
         let content;
@@ -194,13 +195,19 @@ async function strReplaceEditorExecutor(input) {
                 if (input.file_text === undefined || input.file_text === null) {
                     throw new Error("file_text is required for create command");
                 }
+                if (filePath.endsWith("test.ts")) {
+                    throw new Error("Creating test.ts files is not allowed. Did you mean spec.ts?");
+                }
+                if (filePath.endsWith("spec.ts") && !filePath.startsWith("tests/")) {
+                    throw new Error("Creating spec.ts files is not allowed outside tests/ directory");
+                }
                 const parentDir = path_1.default.dirname(filePath);
                 if (parentDir !== "." && !fs_1.default.existsSync(parentDir)) {
                     // Ensure parent directory exists
                     fs_1.default.mkdirSync(parentDir, { recursive: true });
                 }
                 fs_1.default.writeFileSync(filePath, input.file_text);
-                let createTypescriptResult = await (0, web_1.runTypescriptCompiler)();
+                let createTypescriptResult = await (0, web_1.runTypescriptCompiler)(repoDir);
                 if (!createTypescriptResult.success) {
                     return {
                         result: `File ${filePath} has been created. However, type checks are failing with errors:\n${createTypescriptResult.errors.join("\n")}`,
@@ -253,7 +260,7 @@ async function strReplaceEditorExecutor(input) {
                     }
                     newContent = normalizedContent.replace(normalizedOldStr, input.new_str);
                     fs_1.default.writeFileSync(filePath, newContent);
-                    let strReplaceTypescriptResult = await (0, web_1.runTypescriptCompiler)();
+                    let strReplaceTypescriptResult = await (0, web_1.runTypescriptCompiler)(repoDir);
                     if (!strReplaceTypescriptResult.success) {
                         return {
                             result: `Edits to file ${filePath} have been applied. However, type checks are failing with errors:\n${strReplaceTypescriptResult.errors.join("\n")}`,
@@ -282,7 +289,7 @@ async function strReplaceEditorExecutor(input) {
                 }
                 lines.splice(input.insert_line - 1, 0, input.new_str);
                 fs_1.default.writeFileSync(filePath, lines.join("\n"));
-                let insertTypescriptResult = await (0, web_1.runTypescriptCompiler)();
+                let insertTypescriptResult = await (0, web_1.runTypescriptCompiler)(repoDir);
                 if (!insertTypescriptResult.success) {
                     return {
                         result: `Insertion in file ${filePath} was applied. However, type checks are failing with errors:\n${insertTypescriptResult.errors.join("\n")}`,

package/dist/tools/test-gen-browser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAsFnD,eAAO,MAAM,4BAA4B,EAAE,IAqF1C,CAAC"}
+{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AA4FnD,eAAO,MAAM,4BAA4B,EAAE,IAoH1C,CAAC"}

package/dist/tools/test-gen-browser.js

@@ -9,8 +9,10 @@ const promises_1 = __importDefault(require("fs/promises"));
 const zod_1 = require("zod");
 const run_1 = require("../agent/browsing/run");
 const utils_1 = require("../agent/browsing/utils");
+const pw_pause_1 = require("../agent/cua/pw-codegen/pw-pause");
 const web_1 = require("../bin/utils/platform/web");
 const scenarios_1 = require("../bin/utils/scenarios");
+const slug_1 = require("../utils/slug");
 const BrowserAgentSchema = zod_1.z.object({
     testName: zod_1.z.string().describe("The name of the test to create or modify"),
     testSuites: zod_1.z
@@ -87,8 +89,8 @@ exports.generateTestWithBrowserAgent = {
         description: BROWSER_AGENT_DESCRIPTION,
         parameters: BrowserAgentSchema,
     },
-    execute: async (input, trace) => {
-        const repoDir = process.cwd();
+    execute: async (input, repoPath, apiKey, trace, collectArtifacts) => {
+        const repoDir = repoPath;
         const { testName, testSuites, fileName, changeToMake, project } = input;
         try {
             const { projects } = await (0, test_run_1.getAllPlaywrightProjects)(repoDir);
@@ -113,6 +115,7 @@ exports.generateTestWithBrowserAgent = {
                 result: `Test block not found for test name: "${testName}" in file: "${fileName}" with describe blocks: "${testSuites.join(", ")}"`,
             };
         }
+        // Prepare the file for the browser agent
         const fileBackup = await promises_1.default.readFile(fileName, "utf-8");
         try {
             await (0, utils_1.replaceTodoWithCreateTest)(fileName);
@@ -125,6 +128,14 @@ exports.generateTestWithBrowserAgent = {
                 result: `Error running tool: ${error}`,
             };
         }
+        try {
+            // Prepare playwright for codegen
+            console.log("[generateTestWithBrowserAgent] Preparing playwright for codegen");
+            await (0, pw_pause_1.preparePlaywrightForCodegen)(repoDir);
+        }
+        catch (err) {
+            console.warn("[generateTestWithBrowserAgent] Error preparing playwright for codegen", err);
+        }
         const testGenToken = (0, scenarios_1.buildTokenFromOptions)({ name: testName, file: fileName, prompt: changeToMake }, { useComputerUseAgent: true });
         console.log("[generateTestWithBrowserAgent] Validations passed, starting agent");
         const toolResult = await (0, run_1.generateTestsUsingMasterAgent)({
@@ -138,9 +149,14 @@ exports.generateTestWithBrowserAgent = {
             repoDir,
             editFileWithGeneratedCode: false,
         });
-        // Undo the TODO -> createTest changes
+        // Cleanup: Undo the TODO -> createTest changes
+        await (0, pw_pause_1.revertToOriginalPwCode)(repoDir);
         await promises_1.default.writeFile(fileName, fileBackup, "utf-8");
-        const { isError, error, actionsSummary } = toolResult;
+        const { isError, error, actionsSummary, artifacts } = toolResult;
+        let collectedArtifacts;
+        if (artifacts) {
+            collectedArtifacts = await collectArtifacts?.(artifacts, repoDir, (0, slug_1.slugify)(testName));
+        }
         if (!isError) {
             return {
                 isError,
@@ -149,6 +165,7 @@ and the generated Playwright code:
 
 ${actionsSummary}
 `,
+                artifacts: collectedArtifacts,
             };
         }
         else {

package/dist/tools/test-run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAgDnD,eAAO,MAAM,WAAW,EAAE,IAiDzB,CAAC"}
+{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAc,MAAM,wBAAwB,CAAC;AA4C/D,eAAO,MAAM,WAAW,EAAE,IAsEzB,CAAC"}

package/dist/tools/test-run.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.runTestTool = void 0;
 const test_run_1 = require("@empiricalrun/test-run");
 const zod_1 = require("zod");
+const artifacts_1 = require("../artifacts");
+const utils_1 = require("./utils");
 const RunTestSchema = zod_1.z.object({
     testName: zod_1.z.string().describe("The name of the test to run"),
     suites: zod_1.z
@@ -13,11 +15,6 @@ const RunTestSchema = zod_1.z.object({
         .describe("The name of the file where the test is located. File name must end with .spec.ts"),
     project: zod_1.z.string().describe("The project to run the test on"),
 });
-function hasCloudflareCredentials() {
-    return (process.env.R2_ACCOUNT_ID &&
-        process.env.R2_ACCESS_KEY_ID &&
-        process.env.R2_SECRET_ACCESS_KEY);
-}
 function buildReportUrl(projectName, testRunId) {
     return `https://reports.empirical.run/${projectName}/${testRunId}/index.html`;
 }
@@ -37,10 +34,10 @@ exports.runTestTool = {
         description: "Run a test",
         parameters: RunTestSchema,
     },
-    execute: async (input) => {
+    execute: async (input, repoPath, apiKey, trace, collectArtifacts) => {
         let reportUrl = undefined;
         let envOverrides = undefined;
-        if (hasCloudflareCredentials()) {
+        if ((0, artifacts_1.isArtifactCollectionEnabled)()) {
             const projectName = "test-gen-chat-agent";
             const testRunId = Date.now().toString();
             reportUrl = buildReportUrl(projectName, testRunId);
@@ -53,6 +50,7 @@ exports.runTestTool = {
             console.warn("R2 credentials not found: report artifacts will not be uploaded");
         }
         const { testName, suites, fileName, project } = input;
+        const repoDir = process.cwd();
         try {
             const result = await (0, test_run_1.runSingleTest)({
                 testName,
@@ -60,20 +58,25 @@ exports.runTestTool = {
                 fileName,
                 projects: [project],
                 envOverrides,
+                repoDir,
             });
+            const attachments = (0, utils_1.extractAttachmentsFromPlaywrightJSONReport)(result.summaryJson);
+            const artifacts = await collectArtifacts?.(attachments, repoDir, Date.now().toString());
             return {
+                isError: false,
                 result: buildResult({
                     hasTestPassed: result.hasTestPassed,
                     summaryJson: result.summaryJson,
                     reportUrl: reportUrl,
                 }),
-                isError: false,
+                artifacts,
             };
         }
         catch (error) {
             // Ensure we capture the full error message regardless of error type
             const errorMessage = error instanceof Error ? error.message : String(error);
             return {
+                artifacts: null,
                 result: JSON.stringify({ error: errorMessage }),
                 isError: true,
             };

package/dist/tools/utils/index.d.ts

@@ -1,3 +1,4 @@
+import { JSONReport as PlaywrightJSONReport } from "@playwright/test/reporter";
 export declare function makeDashboardRequest<T>({ path, method, body, }: {
     path: string;
     method?: string;
@@ -8,4 +9,16 @@ export declare function callGitHubProxy({ method, url, body, }: {
     url: string;
     body?: any;
 }): Promise<unknown>;
+type AttachmentInfo = {
+    name: string;
+    path: string;
+    contentType: string;
+};
+/**
+ * Extracts attachment information from a Playwright JSON report.
+ * @param report The Playwright JSON report to extract attachments from
+ * @returns An array of objects containing path and contentType for each attachment
+ */
+export declare function extractAttachmentsFromPlaywrightJSONReport(report: PlaywrightJSONReport): AttachmentInfo[];
+export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/tools/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/index.ts"],"names":[],"mappings":"AAAA,wBAAsB,oBAAoB,CAAC,CAAC,EAAE,EAC5C,IAAI,EACJ,MAAc,EACd,IAAI,GACL,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,GAAG,OAAO,CAAC,CAAC,CAAC,CAuBb;AAED,wBAAsB,eAAe,CAAC,EACpC,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,oBAWA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,IAAI,oBAAoB,EAEnC,MAAM,2BAA2B,CAAC;AAEnC,wBAAsB,oBAAoB,CAAC,CAAC,EAAE,EAC5C,IAAI,EACJ,MAAc,EACd,IAAI,GACL,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,GAAG,OAAO,CAAC,CAAC,CAAC,CAuBb;AAED,wBAAsB,eAAe,CAAC,EACpC,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ,oBAWA;AAED,KAAK,cAAc,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,0CAA0C,CACxD,MAAM,EAAE,oBAAoB,GAC3B,cAAc,EAAE,CA4ClB"}

package/dist/tools/utils/index.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.makeDashboardRequest = makeDashboardRequest;
 exports.callGitHubProxy = callGitHubProxy;
+exports.extractAttachmentsFromPlaywrightJSONReport = extractAttachmentsFromPlaywrightJSONReport;
 async function makeDashboardRequest({ path, method = "GET", body, }) {
     const API_KEY = process.env.EMPIRICALRUN_API_KEY;
     if (!API_KEY) {
@@ -36,3 +37,49 @@ async function callGitHubProxy({ method, url, body, }) {
         },
     });
 }
+/**
+ * Extracts attachment information from a Playwright JSON report.
+ * @param report The Playwright JSON report to extract attachments from
+ * @returns An array of objects containing path and contentType for each attachment
+ */
+function extractAttachmentsFromPlaywrightJSONReport(report) {
+    const attachments = [];
+    if (!report) {
+        return attachments;
+    }
+    function processSpec(spec) {
+        const testTitle = spec.title;
+        for (const test of spec.tests) {
+            for (const result of test.results) {
+                if (!result.attachments)
+                    continue;
+                for (const attachment of result.attachments) {
+                    if (attachment.path && attachment.contentType) {
+                        attachments.push({
+                            name: `${testTitle} ${attachment.name}`,
+                            path: attachment.path,
+                            contentType: attachment.contentType,
+                        });
+                    }
+                }
+            }
+        }
+    }
+    for (const suite of report.suites) {
+        // Process nested suites
+        if (suite.suites) {
+            for (const nestedSuite of suite.suites) {
+                for (const spec of nestedSuite.specs) {
+                    processSpec(spec);
+                }
+            }
+        }
+        // Process specs in current suite
+        if (suite.specs) {
+            for (const spec of suite.specs) {
+                processSpec(spec);
+            }
+        }
+    }
+    return attachments;
+}

package/dist/utils/exec.d.ts

@@ -2,11 +2,13 @@ export declare class ProcessManager {
     private childProcess;
     execute(command: string, args: string[], options: {
         env?: Record<string, string>;
+        cwd?: string;
     }): Promise<number>;
     terminate(): void;
     isRunning(): boolean;
 }
 export declare function cmd(command: string, args: string[], options: {
     env?: Record<string, string>;
+    cwd?: string;
 }): Promise<number>;
 //# sourceMappingURL=exec.d.ts.map

package/dist/utils/exec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAmBA,qBAAa,cAAc;IACzB,OAAO,CAAC,YAAY,CAA6B;IAE3C,OAAO,CACX,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACxC,OAAO,CAAC,MAAM,CAAC;IAmDlB,SAAS,IAAI,IAAI;IASjB,SAAS,IAAI,OAAO;CAGrB;AAED,wBAAsB,GAAG,CACvB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GACxC,OAAO,CAAC,MAAM,CAAC,CAGjB"}
+{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAmBA,qBAAa,cAAc;IACzB,OAAO,CAAC,YAAY,CAA6B;IAE3C,OAAO,CACX,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EAEd,OAAO,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GACtD,OAAO,CAAC,MAAM,CAAC;IAoDlB,SAAS,IAAI,IAAI;IASjB,SAAS,IAAI,OAAO;CAGrB;AAED,wBAAsB,GAAG,CACvB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GACtD,OAAO,CAAC,MAAM,CAAC,CAGjB"}

package/dist/utils/exec.js

@@ -21,7 +21,9 @@ function isAcceptableExit(exitCode, signal) {
 }
 class ProcessManager {
     childProcess = null;
-    async execute(command, args, options) {
+    async execute(command, args, 
+    // TODO: Make these required when all tools use repoPath
+    options) {
         console.log(`Executing command: ${command} with args: ${args}`);
         if (this.childProcess) {
             throw new Error("Process is already running");
@@ -34,6 +36,7 @@ class ProcessManager {
             }
             const p = (0, child_process_1.spawn)(command, args, {
                 env: { ...process_1.default.env, ...options.env },
+                cwd: options.cwd,
                 detached: true, // Create process group so we can terminate all child processes
             });
             this.childProcess = p;

package/dist/utils/slug.d.ts

@@ -1,2 +1,18 @@
 export declare const asyncGenerateRandomSlug: (size: number) => Promise<string>;
+/**
+ * Converts a string into a URL-friendly slug by:
+ * 1. Converting to lowercase
+ * 2. Replacing non-alphanumeric characters with hyphens
+ * 3. Collapsing multiple hyphens into single hyphens
+ * 4. Removing empty segments
+ * 5. Removing leading and trailing hyphens
+ *
+ * @param str - The input string to convert into a slug
+ * @returns A URL-friendly slug string
+ * @example
+ * slugify("Hello World!") // returns "hello-world"
+ * slugify("Test & Case") // returns "test-case"
+ * slugify("Multiple---Hyphens") // returns "multiple-hyphens"
+ */
+export declare const slugify: (str: string) => string;
 //# sourceMappingURL=slug.d.ts.map

package/dist/utils/slug.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"slug.d.ts","sourceRoot":"","sources":["../../src/utils/slug.ts"],"names":[],"mappings":"AAQA,eAAO,MAAM,uBAAuB,GAAU,MAAM,MAAM,oBAOzD,CAAC"}
+{"version":3,"file":"slug.d.ts","sourceRoot":"","sources":["../../src/utils/slug.ts"],"names":[],"mappings":"AAQA,eAAO,MAAM,uBAAuB,GAAU,MAAM,MAAM,oBAOzD,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,OAAO,GAAI,KAAK,MAAM,KAAG,MASrC,CAAC"}

package/dist/utils/slug.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.asyncGenerateRandomSlug = void 0;
+exports.slugify = exports.asyncGenerateRandomSlug = void 0;
 // @ts-ignore
 let customAlphabet;
 const loadNanoid = async () => {
@@ -16,3 +16,29 @@ const asyncGenerateRandomSlug = async (size) => {
     return nanoid();
 };
 exports.asyncGenerateRandomSlug = asyncGenerateRandomSlug;
+/**
+ * Converts a string into a URL-friendly slug by:
+ * 1. Converting to lowercase
+ * 2. Replacing non-alphanumeric characters with hyphens
+ * 3. Collapsing multiple hyphens into single hyphens
+ * 4. Removing empty segments
+ * 5. Removing leading and trailing hyphens
+ *
+ * @param str - The input string to convert into a slug
+ * @returns A URL-friendly slug string
+ * @example
+ * slugify("Hello World!") // returns "hello-world"
+ * slugify("Test & Case") // returns "test-case"
+ * slugify("Multiple---Hyphens") // returns "multiple-hyphens"
+ */
+const slugify = (str) => {
+    const join = "-";
+    return str
+        .toLowerCase()
+        .replace(/[^a-z0-9-]+/g, join)
+        .replace(/-+/g, join)
+        .split(join)
+        .filter(Boolean)
+        .join(join);
+};
+exports.slugify = slugify;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.61.0",
+  "version": "0.62.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -50,6 +50,7 @@
     "inquirer": "^12.4.2",
     "istextorbinary": "^9.5.0",
     "jsdom": "^26.0.0",
+    "mime": "3.0.0",
     "minimatch": "^10.0.1",
     "nanoid": "^5.0.7",
     "openai": "4.87.3",
@@ -61,9 +62,9 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "zod": "^3.23.8",
-    "@empiricalrun/llm": "^0.16.1",
+    "@empiricalrun/llm": "^0.17.0",
     "@empiricalrun/r2-uploader": "^0.3.9",
-    "@empiricalrun/test-run": "^0.9.2"
+    "@empiricalrun/test-run": "^0.9.3"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",
@@ -72,12 +73,13 @@
     "@types/express": "^4.17.21",
     "@types/js-levenshtein": "^1.1.3",
     "@types/jsdom": "^21.1.7",
+    "@types/mime": "3.0.0",
     "@types/serve-handler": "^6.1.4",
     "js-levenshtein": "^1.1.6",
     "playwright": "1.47.1",
     "serve-handler": "^6.1.6",
     "ts-patch": "^3.3.0",
-    "@empiricalrun/shared-types": "0.2.0"
+    "@empiricalrun/shared-types": "0.3.0"
   },
   "scripts": {
     "dev": "tspc --build --watch",

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/index.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/skill.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/browsing/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/prompt.ts","./src/agent/chat/repo.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/codegen/create-test-block.ts","./src/agent/codegen/fix-ts-errors.ts","./src/agent/codegen/generate-code-apply-changes.ts","./src/agent/codegen/lexical-scoped-vars.ts","./src/agent/codegen/repo-edit.ts","./src/agent/codegen/run.ts","./src/agent/codegen/skills-retriever.ts","./src/agent/codegen/test-update-feedback.ts","./src/agent/codegen/types.ts","./src/agent/codegen/update-flow.ts","./src/agent/codegen/use-skill.ts","./src/agent/codegen/utils.ts","./src/agent/cua/computer.ts","./src/agent/cua/index.ts","./src/agent/cua/model.ts","./src/agent/diagnosis-agent/index.ts","./src/agent/diagnosis-agent/strict-mode-violation.ts","./src/agent/enrich-prompt/index.ts","./src/agent/enrich-prompt/utils.ts","./src/agent/infer-agent/index.ts","./src/agent/master/action-tool-calls.ts","./src/agent/master/element-annotation.ts","./src/agent/master/execute-browser-action.ts","./src/agent/master/execute-skill-action.ts","./src/agent/master/next-action.ts","./src/agent/master/planner.ts","./src/agent/master/run.ts","./src/agent/master/scroller.ts","./src/agent/master/with-hints.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/master/browser-tests/skills.spec.ts","./src/agent/master/icon-descriptor/index.ts","./src/agent/master/icon-descriptor/normalize-svg.ts","./src/agent/planner/run-time-planner.ts","./src/agent/planner/run.ts","./src/bin/index.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/browser-injected-scripts/annotate-elements.spec.ts","./src/constants/index.ts","./src/errors/index.ts","./src/evals/add-scenario-agent.evals.ts","./src/evals/append-create-test-agent.evals.ts","./src/evals/fetch-pom-skills-agent.evals.ts","./src/evals/infer-master-or-code-agent.evals.ts","./src/evals/master-agent.evals.ts","./src/evals/type.ts","./src/evals/update-scenario-agent.evals.ts","./src/file/client.ts","./src/file/server.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/reporter/index.ts","./src/reporter/lib.ts","./src/session/index.ts","./src/test-build/index.ts","./src/tool-call-service/index.ts","./src/tool-call-service/utils.ts","./src/tools/codegen-agent.ts","./src/tools/commit-and-create-pr.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/download-build.ts","./src/tools/environment-crud.ts","./src/tools/str_replace_editor.ts","./src/tools/test-gen-browser.ts","./src/tools/test-run.ts","./src/tools/grep/index.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/grep/ripgrep/types.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/test-run-fetcher/types.ts","./src/tools/utils/index.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/uploader/index.ts","./src/uploader/utils.ts","./src/utils/checkpoint.ts","./src/utils/env.ts","./src/utils/exec.ts","./src/utils/file-tree.ts","./src/utils/file.ts","./src/utils/git.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/pw-test.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts"],"version":"5.8.3"}
+{"root":["./src/index.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/skill.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/browsing/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/prompt.ts","./src/agent/chat/repo.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/codegen/create-test-block.ts","./src/agent/codegen/fix-ts-errors.ts","./src/agent/codegen/generate-code-apply-changes.ts","./src/agent/codegen/lexical-scoped-vars.ts","./src/agent/codegen/repo-edit.ts","./src/agent/codegen/run.ts","./src/agent/codegen/skills-retriever.ts","./src/agent/codegen/test-update-feedback.ts","./src/agent/codegen/types.ts","./src/agent/codegen/update-flow.ts","./src/agent/codegen/use-skill.ts","./src/agent/codegen/utils.ts","./src/agent/cua/computer.ts","./src/agent/cua/index.ts","./src/agent/cua/model.ts","./src/agent/cua/pw-codegen/element-from-point.ts","./src/agent/cua/pw-codegen/types.ts","./src/agent/cua/pw-codegen/pw-pause/index.ts","./src/agent/cua/pw-codegen/pw-pause/utils.ts","./src/agent/diagnosis-agent/index.ts","./src/agent/diagnosis-agent/strict-mode-violation.ts","./src/agent/enrich-prompt/index.ts","./src/agent/enrich-prompt/utils.ts","./src/agent/infer-agent/index.ts","./src/agent/master/action-tool-calls.ts","./src/agent/master/element-annotation.ts","./src/agent/master/execute-browser-action.ts","./src/agent/master/execute-skill-action.ts","./src/agent/master/next-action.ts","./src/agent/master/planner.ts","./src/agent/master/run.ts","./src/agent/master/scroller.ts","./src/agent/master/with-hints.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/master/browser-tests/skills.spec.ts","./src/agent/master/icon-descriptor/index.ts","./src/agent/master/icon-descriptor/normalize-svg.ts","./src/agent/planner/run-time-planner.ts","./src/agent/planner/run.ts","./src/artifacts/index.ts","./src/bin/index.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/browser-injected-scripts/annotate-elements.spec.ts","./src/constants/index.ts","./src/errors/index.ts","./src/evals/add-scenario-agent.evals.ts","./src/evals/append-create-test-agent.evals.ts","./src/evals/fetch-pom-skills-agent.evals.ts","./src/evals/infer-master-or-code-agent.evals.ts","./src/evals/master-agent.evals.ts","./src/evals/type.ts","./src/evals/update-scenario-agent.evals.ts","./src/file/client.ts","./src/file/server.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/reporter/index.ts","./src/reporter/lib.ts","./src/session/index.ts","./src/test-build/index.ts","./src/tool-call-service/index.ts","./src/tool-call-service/utils.ts","./src/tools/codegen-agent.ts","./src/tools/commit-and-create-pr.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/download-build.ts","./src/tools/environment-crud.ts","./src/tools/str_replace_editor.ts","./src/tools/test-gen-browser.ts","./src/tools/test-run.ts","./src/tools/grep/index.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/grep/ripgrep/types.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/test-run-fetcher/types.ts","./src/tools/utils/index.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/uploader/index.ts","./src/uploader/utils.ts","./src/utils/checkpoint.ts","./src/utils/env.ts","./src/utils/exec.ts","./src/utils/file-tree.ts","./src/utils/file.ts","./src/utils/git.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts"],"version":"5.8.3"}

package/dist/utils/pw-test.d.ts

@@ -1,2 +0,0 @@
-export declare function getTestFixtureModuleFromRepo(repoDir: string): Promise<any>;
-//# sourceMappingURL=pw-test.d.ts.map

package/dist/utils/pw-test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pw-test.d.ts","sourceRoot":"","sources":["../../src/utils/pw-test.ts"],"names":[],"mappings":"AAEA,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,GAAG,CAAC,CAKd"}

package/dist/utils/pw-test.js

@@ -1,13 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTestFixtureModuleFromRepo = getTestFixtureModuleFromRepo;
-const api_1 = __importDefault(require("tsx/cjs/api"));
-async function getTestFixtureModuleFromRepo(repoDir) {
-    const [lastDir] = repoDir.split("/").reverse();
-    const dir = `${repoDir}/${lastDir}`;
-    const { test } = await api_1.default.require(`./tests/fixtures.ts`, dir);
-    return test;
-}