@empiricalrun/test-gen 0.51.1 → 0.51.2

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @empiricalrun/test-gen
 
+## 0.51.2
+
+### Patch Changes
+
+- ecd3c30: fix: show loader when claude is working
+- ad6c96f: fix: prune test-run tool call response for input tokens
+
 ## 0.51.1
 
 ### Patch Changes

package/dist/agent/chat.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AA8EA,wBAAsB,SAAS,CAAC,EAC9B,SAAwC,EACxC,mBAAmB,GACpB,EAAE;IACD,SAAS,CAAC,EAAE,4BAA4B,GAAG,4BAA4B,CAAC;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;;;;GA6FA"}
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AA8EA,wBAAsB,SAAS,CAAC,EAC9B,SAAwC,EACxC,mBAAmB,GACpB,EAAE;IACD,SAAS,CAAC,EAAE,4BAA4B,GAAG,4BAA4B,CAAC;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;;;;GA0GA"}

package/dist/agent/chat.js

@@ -72,8 +72,15 @@ const toolExecutors = {
     str_replace_editor: claude_1.strReplaceEditorTool,
 };
 async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForChatState, }) {
+    const ora = (await import("ora")).default;
     let userPrompt = undefined;
     let chatState = useDiskForChatState ? claude_1.ChatState.load() : new claude_1.ChatState(false);
+    const handleSigInt = () => {
+        console.log("\nExiting. Usage summary:", chatState.getUsageSummary());
+        process.exit(0);
+    };
+    process.once("SIGINT", handleSigInt);
+    process.once("SIGTERM", handleSigInt);
     if (chatState.askUserForInput) {
         // Show last message to the user for context when we loaded from disk
         const messages = chatState.messages;
@@ -115,12 +122,13 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
         }
         const toolUse = chatState.getPendingToolCall();
         if (toolUse) {
-            console.log("Executing tool:", toolUse.name, "with args:", toolUse.input);
+            const spinner = ora(`Executing tool ${toolUse.name} with args: ${JSON.stringify(toolUse.input)}`).start();
             const toolExecutor = toolExecutors[toolUse.name];
             if (!toolExecutor) {
                 throw new Error(`Tool ${toolUse.name} not found`);
             }
             const toolResult = await toolExecutor(toolUse.input);
+            spinner.succeed(`Tool ${toolUse.name} completed`);
             chatState.pushMessage({
                 role: "user",
                 content: [
@@ -134,6 +142,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
             });
             continue;
         }
+        const spinner = ora("Claude is working...").start();
         const response = await (0, claude_1.createClaudeMessage)({
             systemPrompt,
             messages: chatState.getMessagesForCreateCompletion(),
@@ -141,6 +150,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
             model: chatModel,
             withStrReplaceEditor: true,
         });
+        spinner.stop();
         if (!response) {
             throw new Error("No response from LLM");
         }

package/dist/tools/test-run-fetcher/index.d.ts

@@ -0,0 +1,4 @@
+import type { Tool } from "../types";
+export declare function extractPathAfterSourceRepo(fullPath: string): string;
+export declare const testRunTool: Tool;
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/test-run-fetcher/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAWrC,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOnE;AAED,eAAO,MAAM,WAAW,EAAE,IAwHzB,CAAC"}

package/dist/tools/test-run-fetcher/index.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.testRunTool = exports.extractPathAfterSourceRepo = void 0;
+const zod_1 = require("zod");
+const TestRunSchema = zod_1.z.object({
+    testRunUrl: zod_1.z
+        .string()
+        .describe("The full URL of the test run (e.g. https://dash.empirical.run/sortment-tests/test-runs/20269 or with query params like ?status=failed)"),
+});
+function extractPathAfterSourceRepo(fullPath) {
+    const parts = fullPath.split("source-repo");
+    if (parts.length <= 1) {
+        return fullPath;
+    }
+    else {
+        return parts[1].replace(/^\/|\/$/g, "");
+    }
+}
+exports.extractPathAfterSourceRepo = extractPathAfterSourceRepo;
+exports.testRunTool = {
+    schema: {
+        name: "fetchTestRunDetails",
+        description: "Fetch details about a test run using its URL",
+        parameters: TestRunSchema,
+    },
+    execute: async (input) => {
+        const { testRunUrl } = input;
+        // Remove query parameters if they exist
+        const urlWithoutParams = testRunUrl.split("?")[0] || testRunUrl;
+        // Extract the run ID and repo name from the URL
+        const urlParts = urlWithoutParams.split("/");
+        const runId = urlParts.pop(); // Last part is the run ID
+        const repoName = urlParts[urlParts.length - 2]; // Second to last part is the repo name
+        if (!runId || !repoName) {
+            throw new Error("Invalid test run URL - could not extract run ID or repo name");
+        }
+        // Make the API call to fetch test run details
+        const response = await fetch(`https://dash.empirical.run/api/test-runs/${runId}?repo_name=${repoName}`, {
+            method: "GET",
+            headers: {
+                Authorization: "weQPMWKT",
+            },
+        });
+        if (!response.ok) {
+            return {
+                result: `Failed to fetch test run details: ${response.statusText}`,
+                isError: true,
+            };
+        }
+        const data = (await response.json());
+        // To efficiently use input_tokens, we
+        //   1. Truncate stack trace to last 300 characters
+        //   2. Remove request/response headers from network metadata
+        //   3. Focus on failed tests only
+        if (data.data?.test_run?.flattenedSummaryDetails) {
+            data.data.test_run.flattenedSummaryDetails =
+                data.data.test_run.flattenedSummaryDetails.map((detail) => {
+                    // Truncate stack trace to last 300 characters
+                    if (detail.failed_run_metadata?.stack) {
+                        const stack = detail.failed_run_metadata.stack;
+                        // TODO: change to last 10 lines
+                        detail.failed_run_metadata.stack =
+                            stack.length > 300 ? `...${stack.slice(-300)}` : stack;
+                    }
+                    // Remove headers from network metadata
+                    if (detail.network_metadata?.failed_calls) {
+                        detail.network_metadata.failed_calls =
+                            detail.network_metadata.failed_calls.map((call) => ({
+                                ...call,
+                                request: { ...call.request, headers: [] },
+                                response: { ...call.response, headers: [] },
+                            }));
+                    }
+                    if (detail.network_metadata?.failed_calls_within_time_range) {
+                        detail.network_metadata.failed_calls_within_time_range =
+                            detail.network_metadata.failed_calls_within_time_range.map((call) => ({
+                                ...call,
+                                request: { ...call.request, headers: [] },
+                                response: { ...call.response, headers: [] },
+                            }));
+                    }
+                    return detail;
+                });
+        }
+        const failedTests = data.data?.test_run?.flattenedSummaryDetails.filter((detail) => detail.status === "failed");
+        const failedTestInMarkdown = failedTests
+            ?.map((detail) => {
+            const { nesting } = detail.failed_run_metadata;
+            return {
+                testName: nesting.slice(1).join(" > "),
+                fileName: extractPathAfterSourceRepo(detail.failed_run_metadata.location.file),
+                detail,
+            };
+        })
+            .map((test) => {
+            return `
+### Test name: ${test.testName}
+File: ${test.fileName}
+
+Failure data below
+
+\`\`\`json
+${JSON.stringify(test.detail, null, 2)}
+\`\`\`
+`;
+        })
+            .join("\n\n");
+        const markdownResponse = `
+# Test run details
+
+## Run info
+- Run ID: ${runId}
+- Repository: ${repoName}
+- Started at: ${data.data?.test_run?.testRun.run_started_at}
+- Ended at: ${data.data?.test_run?.testRun.run_ended_at}
+- Duration: ${data.data?.test_run?.testRun.duration} seconds
+- Environment name: ${data.data?.test_run?.testRun.environment_name}
+
+## Failed tests
+${failedTestInMarkdown}
+`;
+        return {
+            result: markdownResponse,
+            isError: false,
+        };
+    },
+};

package/dist/tools/test-run-fetcher/types.d.ts

@@ -0,0 +1,120 @@
+type Project = {
+    id: number;
+    org_id: number;
+    name: string;
+    repo_name: string;
+};
+type TestRun = {
+    id: number;
+    run_id: number;
+    project_id: number;
+    total_count: number;
+    unique_count: number;
+    success_count: number;
+    failed_count: number;
+    skipped_count: number;
+    flaky_count: number;
+    run_ended_at: string;
+    run_started_at: string;
+    created_at: string;
+    duration: number;
+    test_run_branch: string;
+    summary_url: string;
+    trigger_type: string;
+    state: string;
+    build_id: number;
+    environment_id: number;
+    metadata: Record<string, any>;
+    test_run_head_sha: string;
+};
+type TestRunEnriched = TestRun & {
+    environment_name: string | null;
+    environment_slug: string | null;
+    commit: string | null;
+    commit_url: string | null;
+    build_branch: string | null;
+    build_url: string | null;
+};
+type FailedRunMetadata = {
+    stack: string;
+    trace: string;
+    videos: string[];
+    nesting: string[];
+    location: {
+        file: string;
+        line: number;
+        column: number;
+    };
+    start_time: string;
+    pw_test_id: string;
+    varying_retries: boolean;
+};
+type CapturedNetworkFailure = {
+    endpoint: string;
+    status: number;
+    timestamp: string;
+    distanceFromFailureInMilliseconds: number;
+    method: string;
+    request: {
+        headers: {
+            [any: string]: string;
+        }[];
+    };
+    response: {
+        headers: {
+            [any: string]: string;
+        }[];
+    };
+};
+type TestCaseSummaryDetail = {
+    id: number;
+    test_case_id: number;
+    test_run_id: number;
+    status: string;
+    test_project: string;
+    failure_type: string;
+    failure_type_description: string;
+    last_commit_before_run_started_at: string;
+    merged_summary?: {
+        content: string;
+    } | null;
+    successful_run_metadata: {
+        videos: string[];
+        test_id?: string;
+        test_run_id?: string;
+    } | null;
+    failed_run_metadata: FailedRunMetadata;
+    visual_diff_summary: {
+        failedStep: string;
+        reason: string;
+        summary: string;
+        videos_picked_for_comparison: {
+            failure: string;
+            success: string;
+        };
+    } | null;
+    error_stack_summary?: {
+        content: string;
+    } | null;
+    network_metadata: {
+        failed_calls: CapturedNetworkFailure[];
+        failed_calls_within_time_range: CapturedNetworkFailure[];
+    } | null;
+    created_at: string;
+    slug: string;
+    failing_line: string;
+};
+export type GetTestRunApiResponse = {
+    data: {
+        test_run: {
+            project: Project;
+            testRun: TestRunEnriched;
+            flattenedSummaryDetails: TestCaseSummaryDetail[];
+        };
+    } | null;
+    error?: {
+        message: string;
+    };
+};
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/tools/test-run-fetcher/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/types.ts"],"names":[],"mappings":"AACA,KAAK,OAAO,GAAG;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,OAAO,GAAG;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,KAAK,eAAe,GAAG,OAAO,GAAG;IAC/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC,EAAE,MAAM,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QACP,OAAO,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,EAAE,CAAC;KACtC,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,EAAE,CAAC;KACtC,CAAC;CACH,CAAC;AAEF,KAAK,qBAAqB,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,MAAM,CAAC;IACjC,iCAAiC,EAAE,MAAM,CAAC;IAC1C,cAAc,CAAC,EAAE;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,uBAAuB,EAAE;QACvB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,IAAI,CAAC;IACT,mBAAmB,EAAE,iBAAiB,CAAC;IACvC,mBAAmB,EAAE;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,4BAA4B,EAAE;YAC5B,OAAO,EAAE,MAAM,CAAC;YAChB,OAAO,EAAE,MAAM,CAAC;SACjB,CAAC;KACH,GAAG,IAAI,CAAC;IACT,mBAAmB,CAAC,EAAE;QACpB,OAAO,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,gBAAgB,EAAE;QAChB,YAAY,EAAE,sBAAsB,EAAE,CAAC;QACvC,8BAA8B,EAAE,sBAAsB,EAAE,CAAC;KAC1D,GAAG,IAAI,CAAC;IACT,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE;QACJ,QAAQ,EAAE;YACR,OAAO,EAAE,OAAO,CAAC;YACjB,OAAO,EAAE,eAAe,CAAC;YACzB,uBAAuB,EAAE,qBAAqB,EAAE,CAAC;SAClD,CAAC;KACH,GAAG,IAAI,CAAC;IACT,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC"}

package/dist/tools/test-run-fetcher/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.51.1",
+  "version": "0.51.2",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -64,6 +64,7 @@
     "minimatch": "^10.0.1",
     "nanoid": "^5.0.7",
     "openai": "4.87.3",
+    "ora": "^8.1.0",
     "picocolors": "^1.0.1",
     "prettier": "^3.2.5",
     "remove-markdown": "^0.5.5",

package/dist/tools/test-run-fetcher.d.ts

@@ -1,3 +0,0 @@
-import type { Tool } from "./types";
-export declare const testRunTool: Tool;
-//# sourceMappingURL=test-run-fetcher.d.ts.map

package/dist/tools/test-run-fetcher.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"test-run-fetcher.d.ts","sourceRoot":"","sources":["../../src/tools/test-run-fetcher.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAUpC,eAAO,MAAM,WAAW,EAAE,IA0DzB,CAAC"}

package/dist/tools/test-run-fetcher.js

@@ -1,59 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.testRunTool = void 0;
-const zod_1 = require("zod");
-const TestRunSchema = zod_1.z.object({
-    testRunUrl: zod_1.z
-        .string()
-        .describe("The full URL of the test run (e.g. https://dash.empirical.run/sortment-tests/test-runs/20269 or with query params like ?status=failed)"),
-});
-exports.testRunTool = {
-    schema: {
-        name: "fetchTestRunDetails",
-        description: "Fetch details about a test run using its URL",
-        parameters: TestRunSchema,
-    },
-    execute: async (input) => {
-        const { testRunUrl } = input;
-        // Remove query parameters if they exist
-        const urlWithoutParams = testRunUrl.split("?")[0] || testRunUrl;
-        // Extract the run ID and repo name from the URL
-        const urlParts = urlWithoutParams.split("/");
-        const runId = urlParts.pop(); // Last part is the run ID
-        const repoName = urlParts[urlParts.length - 2]; // Second to last part is the repo name
-        if (!runId || !repoName) {
-            throw new Error("Invalid test run URL - could not extract run ID or repo name");
-        }
-        // Make the API call to fetch test run details
-        const response = await fetch(`https://dash.empirical.run/api/test-runs/${runId}?repo_name=${repoName}`, {
-            method: "GET",
-            headers: {
-                Authorization: "weQPMWKT",
-            },
-        });
-        if (!response.ok) {
-            return {
-                result: `Failed to fetch test run details: ${response.statusText}`,
-                isError: true,
-            };
-        }
-        const data = await response.json();
-        // Format the response as markdown
-        const markdownResponse = `
-# Test Run Details
-
-## Run Information
-- **Run ID**: ${runId}
-- **Repository**: ${repoName}
-
-## Test Run Data
-\`\`\`json
-${JSON.stringify(data, null, 2)}
-\`\`\`
-`;
-        return {
-            result: markdownResponse,
-            isError: false,
-        };
-    },
-};