@empiricalrun/test-gen 0.51.0 → 0.51.2

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @empiricalrun/test-gen
 
+## 0.51.2
+
+### Patch Changes
+
+- ecd3c30: fix: show loader when claude is working
+- ad6c96f: fix: prune test-run tool call response for input tokens
+
+## 0.51.1
+
+### Patch Changes
+
+- d04190f: fix: remove process.on listeners to avoid leaks
+- 75c7921: fix: show chat usage summary on ctrl+C
+- Updated dependencies [d04190f]
+  - @empiricalrun/test-run@0.7.6
+  - @empiricalrun/llm@0.11.1
+
 ## 0.51.0
 
 ### Minor Changes

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB;;;GA8EnB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB;;;GAgFnB"}

package/dist/agent/browsing/run.js

@@ -32,13 +32,14 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
     const testsDirectory = `${repoDir}/tests`;
     const isTestRunTriggeredForTeardown = teardownFileRegex.test(testFilePath);
     const teardowns = new utils_1.TeardownManager(testsDirectory);
-    if (!isTestRunTriggeredForTeardown) {
-        await teardowns.skipAll();
-    }
+    let removeListeners;
     const command = `npx playwright test ${testFilePath} --retries 0 --project ${project} --timeout 0 --headed`;
     let isError = false;
     let error = "";
     try {
+        if (!isTestRunTriggeredForTeardown) {
+            removeListeners = await teardowns.skipAll();
+        }
         await (0, exec_1.cmd)(command.split(" "), {
             env: {
                 APP_PORT: port.toString(),
@@ -55,7 +56,9 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
         console.error(error);
         isError = true;
     }
-    if (!isTestRunTriggeredForTeardown) {
+    finally {
+        // Remove process listeners before unskipping files
+        removeListeners?.();
         teardowns.unskipAll();
     }
     // clean up the file if there is any error

package/dist/agent/browsing/utils.d.ts

@@ -39,7 +39,7 @@ export declare class TeardownManager {
     private teardownFiles;
     private getAllTeardownFiles;
     private skipTeardownFile;
-    skipAll(): Promise<void>;
+    skipAll(): Promise<() => void>;
     unskipAll(): void;
 }
 //# sourceMappingURL=utils.d.ts.map

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAIxE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAiFD,wBAAsB,yBAAyB,CAAC,EAC9C,YAAY,EACZ,YAAY,EACZ,cAAc,GACf,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B,iBAyBA;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CAM/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAIxE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAiFD,wBAAsB,yBAAyB,CAAC,EAC9C,YAAY,EACZ,YAAY,EACZ,cAAc,GACf,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B,iBAyBA;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CAM/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAoBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -400,18 +400,14 @@ class TeardownManager {
     async skipAll() {
         this.teardownFiles = await this.getAllTeardownFiles();
         await Promise.all(this.teardownFiles.map(async ({ filePath }) => await this.skipTeardownFile(filePath)));
-        process.on("beforeExit", () => {
-            this.unskipAll();
-        });
-        process.on("exit", () => {
-            this.unskipAll();
-        });
-        process.on("SIGINT", () => {
-            this.unskipAll();
-        });
-        process.on("SIGTERM", () => {
-            this.unskipAll();
-        });
+        const setupProcessListeners = (cleanup) => {
+            const events = ["beforeExit", "exit", "SIGINT", "SIGTERM"];
+            events.forEach((event) => process.on(event, cleanup));
+            return () => {
+                events.forEach((event) => process.removeListener(event, cleanup));
+            };
+        };
+        return setupProcessListeners(this.unskipAll.bind(this));
     }
     unskipAll() {
         this.teardownFiles.forEach(({ filePath, content }) => {

package/dist/agent/chat.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AA8EA,wBAAsB,SAAS,CAAC,EAC9B,SAAwC,EACxC,mBAAmB,GACpB,EAAE;IACD,SAAS,CAAC,EAAE,4BAA4B,GAAG,4BAA4B,CAAC;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;;;;GAoFA"}
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AA8EA,wBAAsB,SAAS,CAAC,EAC9B,SAAwC,EACxC,mBAAmB,GACpB,EAAE;IACD,SAAS,CAAC,EAAE,4BAA4B,GAAG,4BAA4B,CAAC;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;;;;GA0GA"}

package/dist/agent/chat.js

@@ -72,8 +72,15 @@ const toolExecutors = {
     str_replace_editor: claude_1.strReplaceEditorTool,
 };
 async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForChatState, }) {
+    const ora = (await import("ora")).default;
     let userPrompt = undefined;
     let chatState = useDiskForChatState ? claude_1.ChatState.load() : new claude_1.ChatState(false);
+    const handleSigInt = () => {
+        console.log("\nExiting. Usage summary:", chatState.getUsageSummary());
+        process.exit(0);
+    };
+    process.once("SIGINT", handleSigInt);
+    process.once("SIGTERM", handleSigInt);
     if (chatState.askUserForInput) {
         // Show last message to the user for context when we loaded from disk
         const messages = chatState.messages;
@@ -89,9 +96,19 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
     while (!userPrompt?.toLowerCase().includes("stop")) {
         chatState.saveToDisk();
         if (chatState.askUserForInput) {
-            userPrompt = await human_in_the_loop_1.humanLoop.getFeedback({
-                message: "User:",
-            });
+            try {
+                userPrompt = await human_in_the_loop_1.humanLoop.getFeedback({
+                    message: "User:",
+                });
+            }
+            catch (e) {
+                // https://github.com/SBoudrias/Inquirer.js/issues/1502#issuecomment-2275991680
+                if (e instanceof Error && e.name === "ExitPromptError") {
+                    console.log("Exiting. Usage summary:", chatState.getUsageSummary());
+                    process.exit(0);
+                }
+                throw e;
+            }
             chatState.pushMessage({
                 role: "user",
                 content: [
@@ -105,12 +122,13 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
         }
         const toolUse = chatState.getPendingToolCall();
         if (toolUse) {
-            console.log("Executing tool:", toolUse.name, "with args:", toolUse.input);
+            const spinner = ora(`Executing tool ${toolUse.name} with args: ${JSON.stringify(toolUse.input)}`).start();
             const toolExecutor = toolExecutors[toolUse.name];
             if (!toolExecutor) {
                 throw new Error(`Tool ${toolUse.name} not found`);
             }
             const toolResult = await toolExecutor(toolUse.input);
+            spinner.succeed(`Tool ${toolUse.name} completed`);
             chatState.pushMessage({
                 role: "user",
                 content: [
@@ -124,6 +142,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
             });
             continue;
         }
+        const spinner = ora("Claude is working...").start();
         const response = await (0, claude_1.createClaudeMessage)({
             systemPrompt,
             messages: chatState.getMessagesForCreateCompletion(),
@@ -131,6 +150,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
             model: chatModel,
             withStrReplaceEditor: true,
         });
+        spinner.stop();
         if (!response) {
             throw new Error("No response from LLM");
         }

package/dist/bin/index.js

@@ -28,10 +28,13 @@ dotenv_1.default.config({
 const flushEvents = async () => {
     await (0, llm_1.flushAllTraces)();
 };
-process.on("beforeExit", async () => await flushEvents());
-process.on("exit", async () => await flushEvents());
-process.on("SIGINT", async () => await flushEvents());
-process.on("SIGTERM", async () => await flushEvents());
+function setupProcessListeners(cleanup) {
+    const events = ["beforeExit", "exit", "SIGINT", "SIGTERM"];
+    events.forEach((event) => process.on(event, cleanup));
+    return () => {
+        events.forEach((event) => process.removeListener(event, cleanup));
+    };
+}
 async function runChatAgent(modelInput, useDiskForChatState) {
     const MODEL_MAPPING = {
         "claude-3-7": "claude-3-7-sonnet-20250219",
@@ -172,6 +175,7 @@ async function runAgentsWorkflow(testGenConfig, testGenToken) {
     return agent;
 }
 (async function main() {
+    const removeListeners = setupProcessListeners(flushEvents);
     console.log(`Running test-gen v${require("../../package.json").version} from ${__dirname}`);
     const program = new commander_1.Command();
     program
@@ -228,7 +232,7 @@ async function runAgentsWorkflow(testGenConfig, testGenToken) {
             testName: testGenConfig.testCase.name,
         });
     }
-    // TODO: move these reporters to a better lifecycle
+    removeListeners();
     await (0, llm_1.flushAllTraces)();
     await (0, logger_1.waitForLogsToFlush)();
     await (0, session_1.endSession)();

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAYpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAiC3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAepC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAyC3E"}

package/dist/index.js

@@ -14,42 +14,54 @@ const pw_test_1 = require("./utils/pw-test");
 const flushEvents = async () => {
     await (0, llm_1.flushAllTraces)();
 };
-process.on("beforeExit", async () => await flushEvents());
-process.on("exit", async () => await flushEvents());
-process.on("SIGINT", async () => await flushEvents());
-process.on("SIGTERM", async () => await flushEvents());
+function setupProcessListeners(cleanup) {
+    const events = ["beforeExit", "exit", "SIGINT", "SIGTERM"];
+    events.forEach((event) => process.on(event, cleanup));
+    return () => {
+        events.forEach((event) => process.removeListener(event, cleanup));
+    };
+}
 async function createTest(task, page, scope) {
-    const testConfigArg = process.env.TEST_GEN_TOKEN;
-    const testGenConfig = (0, scenarios_1.loadTestConfigs)(testConfigArg);
-    (0, reporter_1.setReporterConfig)({
-        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-        testSessionId: testGenConfig.options?.metadata.testSessionId,
-        generationId: testGenConfig.options?.metadata.generationId,
-    });
-    (0, session_1.setSessionDetails)({
-        sessionId: testGenConfig.options?.metadata.testSessionId,
-        generationId: testGenConfig.options?.metadata.generationId,
-        testCaseId: testGenConfig.testCase.id,
-        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-    });
-    const fileService = new client_1.default();
-    const { testCase, specPath } = testGenConfig;
-    const { code, importPaths } = await (0, run_1.createTestUsingMasterAgent)({
-        testCase,
-        specPath,
-        page,
-        task,
-        options: {
-            ...testGenConfig.options,
-        },
-        scopeVars: scope,
-    });
-    await fileService.updateTest({
-        task,
-        generatedCode: code,
-        importPaths,
-    });
-    // skip the rest of the test once generation is over
-    await (0, pw_test_1.skipTest)();
+    const removeListeners = setupProcessListeners(flushEvents);
+    try {
+        const testConfigArg = process.env.TEST_GEN_TOKEN;
+        const testGenConfig = (0, scenarios_1.loadTestConfigs)(testConfigArg);
+        (0, reporter_1.setReporterConfig)({
+            projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+            testSessionId: testGenConfig.options?.metadata.testSessionId,
+            generationId: testGenConfig.options?.metadata.generationId,
+        });
+        (0, session_1.setSessionDetails)({
+            sessionId: testGenConfig.options?.metadata.testSessionId,
+            generationId: testGenConfig.options?.metadata.generationId,
+            testCaseId: testGenConfig.testCase.id,
+            projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+        });
+        const fileService = new client_1.default();
+        const { testCase, specPath } = testGenConfig;
+        const { code, importPaths } = await (0, run_1.createTestUsingMasterAgent)({
+            testCase,
+            specPath,
+            page,
+            task,
+            options: {
+                ...testGenConfig.options,
+            },
+            scopeVars: scope,
+        });
+        await fileService.updateTest({
+            task,
+            generatedCode: code,
+            importPaths,
+        });
+        // skip the rest of the test once generation is over
+        await (0, pw_test_1.skipTest)();
+    }
+    finally {
+        // Ensure listeners are removed even if an error occurs
+        removeListeners();
+        // Flush events one final time before removing listeners
+        await flushEvents();
+    }
 }
 exports.createTest = createTest;

package/dist/tools/test-run-fetcher/index.d.ts

@@ -0,0 +1,4 @@
+import type { Tool } from "../types";
+export declare function extractPathAfterSourceRepo(fullPath: string): string;
+export declare const testRunTool: Tool;
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/test-run-fetcher/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAWrC,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOnE;AAED,eAAO,MAAM,WAAW,EAAE,IAwHzB,CAAC"}

package/dist/tools/test-run-fetcher/index.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.testRunTool = exports.extractPathAfterSourceRepo = void 0;
+const zod_1 = require("zod");
+const TestRunSchema = zod_1.z.object({
+    testRunUrl: zod_1.z
+        .string()
+        .describe("The full URL of the test run (e.g. https://dash.empirical.run/sortment-tests/test-runs/20269 or with query params like ?status=failed)"),
+});
+function extractPathAfterSourceRepo(fullPath) {
+    const parts = fullPath.split("source-repo");
+    if (parts.length <= 1) {
+        return fullPath;
+    }
+    else {
+        return parts[1].replace(/^\/|\/$/g, "");
+    }
+}
+exports.extractPathAfterSourceRepo = extractPathAfterSourceRepo;
+exports.testRunTool = {
+    schema: {
+        name: "fetchTestRunDetails",
+        description: "Fetch details about a test run using its URL",
+        parameters: TestRunSchema,
+    },
+    execute: async (input) => {
+        const { testRunUrl } = input;
+        // Remove query parameters if they exist
+        const urlWithoutParams = testRunUrl.split("?")[0] || testRunUrl;
+        // Extract the run ID and repo name from the URL
+        const urlParts = urlWithoutParams.split("/");
+        const runId = urlParts.pop(); // Last part is the run ID
+        const repoName = urlParts[urlParts.length - 2]; // Second to last part is the repo name
+        if (!runId || !repoName) {
+            throw new Error("Invalid test run URL - could not extract run ID or repo name");
+        }
+        // Make the API call to fetch test run details
+        const response = await fetch(`https://dash.empirical.run/api/test-runs/${runId}?repo_name=${repoName}`, {
+            method: "GET",
+            headers: {
+                Authorization: "weQPMWKT",
+            },
+        });
+        if (!response.ok) {
+            return {
+                result: `Failed to fetch test run details: ${response.statusText}`,
+                isError: true,
+            };
+        }
+        const data = (await response.json());
+        // To efficiently use input_tokens, we
+        //   1. Truncate stack trace to last 300 characters
+        //   2. Remove request/response headers from network metadata
+        //   3. Focus on failed tests only
+        if (data.data?.test_run?.flattenedSummaryDetails) {
+            data.data.test_run.flattenedSummaryDetails =
+                data.data.test_run.flattenedSummaryDetails.map((detail) => {
+                    // Truncate stack trace to last 300 characters
+                    if (detail.failed_run_metadata?.stack) {
+                        const stack = detail.failed_run_metadata.stack;
+                        // TODO: change to last 10 lines
+                        detail.failed_run_metadata.stack =
+                            stack.length > 300 ? `...${stack.slice(-300)}` : stack;
+                    }
+                    // Remove headers from network metadata
+                    if (detail.network_metadata?.failed_calls) {
+                        detail.network_metadata.failed_calls =
+                            detail.network_metadata.failed_calls.map((call) => ({
+                                ...call,
+                                request: { ...call.request, headers: [] },
+                                response: { ...call.response, headers: [] },
+                            }));
+                    }
+                    if (detail.network_metadata?.failed_calls_within_time_range) {
+                        detail.network_metadata.failed_calls_within_time_range =
+                            detail.network_metadata.failed_calls_within_time_range.map((call) => ({
+                                ...call,
+                                request: { ...call.request, headers: [] },
+                                response: { ...call.response, headers: [] },
+                            }));
+                    }
+                    return detail;
+                });
+        }
+        const failedTests = data.data?.test_run?.flattenedSummaryDetails.filter((detail) => detail.status === "failed");
+        const failedTestInMarkdown = failedTests
+            ?.map((detail) => {
+            const { nesting } = detail.failed_run_metadata;
+            return {
+                testName: nesting.slice(1).join(" > "),
+                fileName: extractPathAfterSourceRepo(detail.failed_run_metadata.location.file),
+                detail,
+            };
+        })
+            .map((test) => {
+            return `
+### Test name: ${test.testName}
+File: ${test.fileName}
+
+Failure data below
+
+\`\`\`json
+${JSON.stringify(test.detail, null, 2)}
+\`\`\`
+`;
+        })
+            .join("\n\n");
+        const markdownResponse = `
+# Test run details
+
+## Run info
+- Run ID: ${runId}
+- Repository: ${repoName}
+- Started at: ${data.data?.test_run?.testRun.run_started_at}
+- Ended at: ${data.data?.test_run?.testRun.run_ended_at}
+- Duration: ${data.data?.test_run?.testRun.duration} seconds
+- Environment name: ${data.data?.test_run?.testRun.environment_name}
+
+## Failed tests
+${failedTestInMarkdown}
+`;
+        return {
+            result: markdownResponse,
+            isError: false,
+        };
+    },
+};

package/dist/tools/test-run-fetcher/types.d.ts

@@ -0,0 +1,120 @@
+type Project = {
+    id: number;
+    org_id: number;
+    name: string;
+    repo_name: string;
+};
+type TestRun = {
+    id: number;
+    run_id: number;
+    project_id: number;
+    total_count: number;
+    unique_count: number;
+    success_count: number;
+    failed_count: number;
+    skipped_count: number;
+    flaky_count: number;
+    run_ended_at: string;
+    run_started_at: string;
+    created_at: string;
+    duration: number;
+    test_run_branch: string;
+    summary_url: string;
+    trigger_type: string;
+    state: string;
+    build_id: number;
+    environment_id: number;
+    metadata: Record<string, any>;
+    test_run_head_sha: string;
+};
+type TestRunEnriched = TestRun & {
+    environment_name: string | null;
+    environment_slug: string | null;
+    commit: string | null;
+    commit_url: string | null;
+    build_branch: string | null;
+    build_url: string | null;
+};
+type FailedRunMetadata = {
+    stack: string;
+    trace: string;
+    videos: string[];
+    nesting: string[];
+    location: {
+        file: string;
+        line: number;
+        column: number;
+    };
+    start_time: string;
+    pw_test_id: string;
+    varying_retries: boolean;
+};
+type CapturedNetworkFailure = {
+    endpoint: string;
+    status: number;
+    timestamp: string;
+    distanceFromFailureInMilliseconds: number;
+    method: string;
+    request: {
+        headers: {
+            [any: string]: string;
+        }[];
+    };
+    response: {
+        headers: {
+            [any: string]: string;
+        }[];
+    };
+};
+type TestCaseSummaryDetail = {
+    id: number;
+    test_case_id: number;
+    test_run_id: number;
+    status: string;
+    test_project: string;
+    failure_type: string;
+    failure_type_description: string;
+    last_commit_before_run_started_at: string;
+    merged_summary?: {
+        content: string;
+    } | null;
+    successful_run_metadata: {
+        videos: string[];
+        test_id?: string;
+        test_run_id?: string;
+    } | null;
+    failed_run_metadata: FailedRunMetadata;
+    visual_diff_summary: {
+        failedStep: string;
+        reason: string;
+        summary: string;
+        videos_picked_for_comparison: {
+            failure: string;
+            success: string;
+        };
+    } | null;
+    error_stack_summary?: {
+        content: string;
+    } | null;
+    network_metadata: {
+        failed_calls: CapturedNetworkFailure[];
+        failed_calls_within_time_range: CapturedNetworkFailure[];
+    } | null;
+    created_at: string;
+    slug: string;
+    failing_line: string;
+};
+export type GetTestRunApiResponse = {
+    data: {
+        test_run: {
+            project: Project;
+            testRun: TestRunEnriched;
+            flattenedSummaryDetails: TestCaseSummaryDetail[];
+        };
+    } | null;
+    error?: {
+        message: string;
+    };
+};
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/tools/test-run-fetcher/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/types.ts"],"names":[],"mappings":"AACA,KAAK,OAAO,GAAG;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,OAAO,GAAG;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,KAAK,eAAe,GAAG,OAAO,GAAG;IAC/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC,EAAE,MAAM,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QACP,OAAO,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,EAAE,CAAC;KACtC,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,EAAE,CAAC;KACtC,CAAC;CACH,CAAC;AAEF,KAAK,qBAAqB,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,MAAM,CAAC;IACjC,iCAAiC,EAAE,MAAM,CAAC;IAC1C,cAAc,CAAC,EAAE;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,uBAAuB,EAAE;QACvB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,IAAI,CAAC;IACT,mBAAmB,EAAE,iBAAiB,CAAC;IACvC,mBAAmB,EAAE;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,4BAA4B,EAAE;YAC5B,OAAO,EAAE,MAAM,CAAC;YAChB,OAAO,EAAE,MAAM,CAAC;SACjB,CAAC;KACH,GAAG,IAAI,CAAC;IACT,mBAAmB,CAAC,EAAE;QACpB,OAAO,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,gBAAgB,EAAE;QAChB,YAAY,EAAE,sBAAsB,EAAE,CAAC;QACvC,8BAA8B,EAAE,sBAAsB,EAAE,CAAC;KAC1D,GAAG,IAAI,CAAC;IACT,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE;QACJ,QAAQ,EAAE;YACR,OAAO,EAAE,OAAO,CAAC;YACjB,OAAO,EAAE,eAAe,CAAC;YACzB,uBAAuB,EAAE,qBAAqB,EAAE,CAAC;SAClD,CAAC;KACH,GAAG,IAAI,CAAC;IACT,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC"}

package/dist/tools/test-run-fetcher/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.51.0",
+  "version": "0.51.2",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -64,6 +64,7 @@
     "minimatch": "^10.0.1",
     "nanoid": "^5.0.7",
     "openai": "4.87.3",
+    "ora": "^8.1.0",
     "picocolors": "^1.0.1",
     "prettier": "^3.2.5",
     "remove-markdown": "^0.5.5",
@@ -73,10 +74,10 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "zod": "^3.23.8",
-    "@empiricalrun/llm": "^0.11.0",
+    "@empiricalrun/llm": "^0.11.1",
     "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/reporter": "^0.23.2",
-    "@empiricalrun/test-run": "^0.7.5"
+    "@empiricalrun/test-run": "^0.7.6"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",

package/dist/tools/test-run-fetcher.d.ts

@@ -1,3 +0,0 @@
-import type { Tool } from "./types";
-export declare const testRunTool: Tool;
-//# sourceMappingURL=test-run-fetcher.d.ts.map

package/dist/tools/test-run-fetcher.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"test-run-fetcher.d.ts","sourceRoot":"","sources":["../../src/tools/test-run-fetcher.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAUpC,eAAO,MAAM,WAAW,EAAE,IA0DzB,CAAC"}

package/dist/tools/test-run-fetcher.js

@@ -1,59 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.testRunTool = void 0;
-const zod_1 = require("zod");
-const TestRunSchema = zod_1.z.object({
-    testRunUrl: zod_1.z
-        .string()
-        .describe("The full URL of the test run (e.g. https://dash.empirical.run/sortment-tests/test-runs/20269 or with query params like ?status=failed)"),
-});
-exports.testRunTool = {
-    schema: {
-        name: "fetchTestRunDetails",
-        description: "Fetch details about a test run using its URL",
-        parameters: TestRunSchema,
-    },
-    execute: async (input) => {
-        const { testRunUrl } = input;
-        // Remove query parameters if they exist
-        const urlWithoutParams = testRunUrl.split("?")[0] || testRunUrl;
-        // Extract the run ID and repo name from the URL
-        const urlParts = urlWithoutParams.split("/");
-        const runId = urlParts.pop(); // Last part is the run ID
-        const repoName = urlParts[urlParts.length - 2]; // Second to last part is the repo name
-        if (!runId || !repoName) {
-            throw new Error("Invalid test run URL - could not extract run ID or repo name");
-        }
-        // Make the API call to fetch test run details
-        const response = await fetch(`https://dash.empirical.run/api/test-runs/${runId}?repo_name=${repoName}`, {
-            method: "GET",
-            headers: {
-                Authorization: "weQPMWKT",
-            },
-        });
-        if (!response.ok) {
-            return {
-                result: `Failed to fetch test run details: ${response.statusText}`,
-                isError: true,
-            };
-        }
-        const data = await response.json();
-        // Format the response as markdown
-        const markdownResponse = `
-# Test Run Details
-
-## Run Information
-- **Run ID**: ${runId}
-- **Repository**: ${repoName}
-
-## Test Run Data
-\`\`\`json
-${JSON.stringify(data, null, 2)}
-\`\`\`
-`;
-        return {
-            result: markdownResponse,
-            isError: false,
-        };
-    },
-};