@empiricalrun/test-gen 0.49.0 → 0.50.1

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @empiricalrun/test-gen
 
+## 0.50.1
+
+### Patch Changes
+
+- b070af3: fix: error handling in test run tool
+- a94ef14: fix: chat agent system prompt for proactiveness
+- Updated dependencies [b070af3]
+  - @empiricalrun/test-run@0.7.4
+
+## 0.50.0
+
+### Minor Changes
+
+- 0eeff70: feat: add test-run-fetcher tool call, response body changes
+
+### Patch Changes
+
+- b14d5bf: feat: support headed executions of test run tool
+- Updated dependencies [b14d5bf]
+  - @empiricalrun/test-run@0.7.3
+
 ## 0.49.0
 
 ### Minor Changes

package/dist/agent/chat.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AA6D1D,wBAAsB,SAAS,CAAC,EAC9B,MAAM,GACP,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,8CAqEA"}
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AA8E1D,wBAAsB,SAAS,CAAC,EAC9B,MAAM,GACP,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,8CAqEA"}

package/dist/agent/chat.js

@@ -11,6 +11,7 @@ const browser_agent_1 = require("../tools/browser-agent");
 const diagnosis_fetcher_1 = require("../tools/diagnosis-fetcher");
 const grep_1 = require("../tools/grep");
 const test_run_1 = require("../tools/test-run");
+const test_run_fetcher_1 = require("../tools/test-run-fetcher");
 const zod_schema_1 = require("../tools/zod-schema");
 const repo_tree_1 = require("../utils/repo-tree");
 const systemPrompt = `
@@ -25,6 +26,9 @@ Summarize the results in a few sentences.
 If the user provides a diagnosis URL, you can use the fetchDiagnosisDetails tool
 to get more information about the test case and its results.
 
+If the user provides a test run URL, you can use the fetchTestRunDetails tool
+to get detailed information about a specific test run.
+
 Or if the user asks you to modify a test, you could use the generateTestWithBrowserAgent tool. If you suspect
 that a UI selector needs to be updated, using the browser agent is a good idea.
 
@@ -40,6 +44,7 @@ The position of the comment is important: the browser agent will look for this c
 the actual code to click on the login button. If you are fixing a failing test, your comment should be 
 around the failing line of code, so that it can be replaced/modified.
 
+# Repo context
 You are running as a CLI tool inside the directory of the repo where this test file is located. Here is
 the repo directory structure:
 
@@ -48,8 +53,20 @@ ${(0, repo_tree_1.generateAsciiTree)(process.cwd())}
 While specifying paths to files, use relative paths from the current working directory. For example:
 - Correct path: "tests/lesson.spec.ts"
 - Incorrect path: "/repo/tests/lesson.spec.ts" or "${path_1.default.basename(process.cwd())}/tests/lesson.spec.ts"
+
+# Proactiveness
+You are allowed to be proactive, but only when the user asks you to do something. You should strive to
+strike a balance between:
+1. Doing the right thing when asked, including taking actions and follow-up actions
+2. Not surprising the user with actions you take without asking
 `;
-const tools = [test_run_1.runTestTool, browser_agent_1.browserAgentTool, diagnosis_fetcher_1.diagnosisTool, grep_1.grepTool];
+const tools = [
+    test_run_1.runTestTool,
+    browser_agent_1.browserAgentTool,
+    diagnosis_fetcher_1.diagnosisTool,
+    grep_1.grepTool,
+    test_run_fetcher_1.testRunTool,
+];
 const toolExecutors = {
     ...Object.fromEntries(tools.map((tool) => [tool.schema.name, tool.execute])),
     str_replace_editor: claude_1.strReplaceEditorTool,

package/dist/tools/diagnosis-fetcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"diagnosis-fetcher.d.ts","sourceRoot":"","sources":["../../src/tools/diagnosis-fetcher.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAUpC,eAAO,MAAM,aAAa,EAAE,IA+E3B,CAAC"}
+{"version":3,"file":"diagnosis-fetcher.d.ts","sourceRoot":"","sources":["../../src/tools/diagnosis-fetcher.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAUpC,eAAO,MAAM,aAAa,EAAE,IAgF3B,CAAC"}

package/dist/tools/diagnosis-fetcher.js

@@ -25,11 +25,10 @@ exports.diagnosisTool = {
         if (!slug) {
             throw new Error("Invalid diagnosis URL - could not extract slug");
         }
-        // Make the API call to fetch diagnosis details
         const response = await fetch(`https://dash.empirical.run/api/diagnosis/${slug}/detailed`, {
             method: "GET",
             headers: {
-                Authorization: "weQPMWKT", // Using the auth token from test-endpoint.mdc
+                Authorization: "weQPMWKT",
             },
         });
         if (!response.ok) {
@@ -40,8 +39,10 @@ exports.diagnosisTool = {
         }
         const data = await response.json();
         const { test_case, diagnosis } = data.data;
-        const project = diagnosis[0]?.test_project || "unknown";
+        const project = diagnosis?.test_project || "unknown";
         const sourceContext = await promises_1.default.readFile(path_1.default.join("tests", test_case.file_path), "utf-8");
+        const repoName = path_1.default.basename(process.cwd());
+        const cleanErrorStack = diagnosis?.failed_run_metadata?.stack?.replace(`"/runner/_work/${repoName}/${repoName}/source-repo/"`, "");
         // Format the response as markdown
         const markdownResponse = `
 # Test Case Diagnosis
@@ -58,21 +59,21 @@ ${sourceContext}
 ## What Happened in the Test Run
 
 ### Failure Details
-- **Failing Line**: ${diagnosis[0]?.failing_line || "No failing line available"}
+- **Failing Line**: ${diagnosis?.failing_line || "No failing line available"}
 
 #### Error Stack
 \`\`\`
-${diagnosis[0]?.failed_run_metadata?.stack?.replace("/runner/_work/shopflo-tests/shopflo-tests/source-repo/", "") || "No error stack available"}
+${cleanErrorStack || "No error stack available"}
 \`\`\`
 
 #### Error Summary
-${diagnosis[0]?.error_stack_summary?.content || "No error summary available"}
+${diagnosis?.error_stack_summary?.content || "No error summary available"}
 
 #### Visual Analysis
-${diagnosis[0]?.visual_diff_summary?.summary || "No visual analysis available"}
+${diagnosis?.visual_diff_summary?.summary || "No visual analysis available"}
 
 #### Merged Summary
-${diagnosis[0]?.merged_summary?.content || "No merged summary available"}
+${diagnosis?.merged_summary?.content || "No merged summary available"}
 `;
         return {
             result: markdownResponse,

package/dist/tools/test-run-fetcher.d.ts

@@ -0,0 +1,3 @@
+import type { Tool } from "./types";
+export declare const testRunTool: Tool;
+//# sourceMappingURL=test-run-fetcher.d.ts.map

package/dist/tools/test-run-fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-run-fetcher.d.ts","sourceRoot":"","sources":["../../src/tools/test-run-fetcher.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAUpC,eAAO,MAAM,WAAW,EAAE,IA0DzB,CAAC"}

package/dist/tools/test-run-fetcher.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.testRunTool = void 0;
+const zod_1 = require("zod");
+const TestRunSchema = zod_1.z.object({
+    testRunUrl: zod_1.z
+        .string()
+        .describe("The full URL of the test run (e.g. https://dash.empirical.run/sortment-tests/test-runs/20269 or with query params like ?status=failed)"),
+});
+exports.testRunTool = {
+    schema: {
+        name: "fetchTestRunDetails",
+        description: "Fetch details about a test run using its URL",
+        parameters: TestRunSchema,
+    },
+    execute: async (input) => {
+        const { testRunUrl } = input;
+        // Remove query parameters if they exist
+        const urlWithoutParams = testRunUrl.split("?")[0] || testRunUrl;
+        // Extract the run ID and repo name from the URL
+        const urlParts = urlWithoutParams.split("/");
+        const runId = urlParts.pop(); // Last part is the run ID
+        const repoName = urlParts[urlParts.length - 2]; // Second to last part is the repo name
+        if (!runId || !repoName) {
+            throw new Error("Invalid test run URL - could not extract run ID or repo name");
+        }
+        // Make the API call to fetch test run details
+        const response = await fetch(`https://dash.empirical.run/api/test-runs/${runId}?repo_name=${repoName}`, {
+            method: "GET",
+            headers: {
+                Authorization: "weQPMWKT",
+            },
+        });
+        if (!response.ok) {
+            return {
+                result: `Failed to fetch test run details: ${response.statusText}`,
+                isError: true,
+            };
+        }
+        const data = await response.json();
+        // Format the response as markdown
+        const markdownResponse = `
+# Test Run Details
+
+## Run Information
+- **Run ID**: ${runId}
+- **Repository**: ${repoName}
+
+## Test Run Data
+\`\`\`json
+${JSON.stringify(data, null, 2)}
+\`\`\`
+`;
+        return {
+            result: markdownResponse,
+            isError: false,
+        };
+    },
+};

package/dist/tools/test-run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAepC,eAAO,MAAM,WAAW,EAAE,IAmBzB,CAAC"}
+{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAsBpC,eAAO,MAAM,WAAW,EAAE,IA8BzB,CAAC"}

package/dist/tools/test-run.js

@@ -12,6 +12,11 @@ const RunTestSchema = zod_1.z.object({
         .string()
         .describe("The name of the file where the test is located. File name must end with .spec.ts"),
     project: zod_1.z.string().describe("The project to run the test on"),
+    headed: zod_1.z
+        .boolean()
+        .describe("Whether to run the test in headed mode (default is false, which is headless)")
+        .optional()
+        .default(false),
 });
 exports.runTestTool = {
     schema: {
@@ -20,16 +25,27 @@ exports.runTestTool = {
         parameters: RunTestSchema,
     },
     execute: async (input) => {
-        const { testName, suites, fileName, project } = input;
-        const result = await (0, test_run_1.runSingleTest)({
-            testName,
-            suites,
-            fileName,
-            projects: [project],
-        });
-        return {
-            result: JSON.stringify(result),
-            isError: false,
-        };
+        const { testName, suites, fileName, project, headed } = input;
+        try {
+            const result = await (0, test_run_1.runSingleTest)({
+                testName,
+                suites,
+                fileName,
+                projects: [project],
+                headed,
+            });
+            return {
+                result: JSON.stringify(result),
+                isError: false,
+            };
+        }
+        catch (error) {
+            // Ensure we capture the full error message regardless of error type
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                result: JSON.stringify({ error: errorMessage }),
+                isError: true,
+            };
+        }
     },
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.49.0",
+  "version": "0.50.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -74,10 +74,10 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "zod": "^3.23.8",
-    "@empiricalrun/llm": "^0.10.1",
     "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/reporter": "^0.23.1",
-    "@empiricalrun/test-run": "^0.7.2"
+    "@empiricalrun/test-run": "^0.7.4",
+    "@empiricalrun/llm": "^0.10.1"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",