@empiricalrun/test-gen 0.47.3 → 0.48.0

package/dist/bin/utils/index.js

@@ -6,12 +6,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.validateAndCompleteCliOptions = void 0;
 const inquirer_1 = __importDefault(require("inquirer"));
 async function validateAndCompleteCliOptions(options) {
+    // For existing flow between dashboard <> test-gen (via ci-worker)
     const hasToken = !!options.token;
     if (hasToken) {
         return options;
     }
+    let requiredFields = ["name", "file", "prompt"];
+    // For new chat flow in local CLI usage, only prompt is required
+    if (options.useChat) {
+        requiredFields = ["prompt"];
+    }
     const questions = [];
-    if (!options.name) {
+    if (!options.name && requiredFields.includes("name")) {
         questions.push({
             type: "input",
             name: "name",
@@ -19,7 +25,7 @@ async function validateAndCompleteCliOptions(options) {
             validate: (input) => input.trim().length > 0 || "Test name is required",
         });
     }
-    if (!options.file) {
+    if (!options.file && requiredFields.includes("file")) {
         questions.push({
             type: "input",
             name: "file",
@@ -27,7 +33,7 @@ async function validateAndCompleteCliOptions(options) {
             validate: (input) => input.trim().length > 0 || "Test file path is required",
         });
     }
-    if (!options.prompt) {
+    if (!options.prompt && requiredFields.includes("prompt")) {
         questions.push({
             type: "editor",
             name: "prompt",

package/dist/file/server.d.ts

@@ -2,12 +2,14 @@ export declare class FileService {
     private port;
     private filePath;
     private repoDir;
+    private server;
     constructor({ port, repoDir }: {
         port: number;
         repoDir: string;
     });
     setFilePath(filePath: string): void;
     startFileService(): Promise<number>;
+    stop(): Promise<void>;
 }
 export declare function startFileService(): Promise<void>;
 //# sourceMappingURL=server.d.ts.map

package/dist/file/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAWA,qBAAa,WAAW;IACtB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;gBAEjB,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAKhE,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAyC1C;AAED,wBAAsB,gBAAgB,kBAAK"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAWA,qBAAa,WAAW;IACtB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAA4C;gBAE9C,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAKhE,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IA0CnC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB5B;AAED,wBAAsB,gBAAgB,kBAAK"}

package/dist/file/server.js

@@ -13,6 +13,7 @@ class FileService {
     port = 0;
     filePath = "";
     repoDir = "";
+    server;
     constructor({ port, repoDir }) {
         this.port = port;
         this.repoDir = repoDir;
@@ -43,7 +44,23 @@ class FileService {
             return res.send({ success: false });
         });
         return new Promise((resolve) => {
-            app.listen(this.port, () => resolve(this.port));
+            this.server = app.listen(this.port, () => resolve(this.port));
+        });
+    }
+    async stop() {
+        return new Promise((resolve, reject) => {
+            if (!this.server) {
+                resolve();
+                return;
+            }
+            this.server.close((err) => {
+                if (err) {
+                    reject(err);
+                    return;
+                }
+                this.server = undefined;
+                resolve();
+            });
         });
     }
 }

package/dist/tools/browser-agent.d.ts

@@ -0,0 +1,3 @@
+import type { Tool } from "./types";
+export declare const browserAgentTool: Tool;
+//# sourceMappingURL=browser-agent.d.ts.map

package/dist/tools/browser-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-agent.d.ts","sourceRoot":"","sources":["../../src/tools/browser-agent.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAUpC,eAAO,MAAM,gBAAgB,EAAE,IAkH9B,CAAC"}

package/dist/tools/browser-agent.js

@@ -0,0 +1,120 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.browserAgentTool = void 0;
+const run_1 = require("../agent/browsing/run");
+const utils_1 = require("../agent/browsing/utils");
+const scenarios_1 = require("../bin/utils/scenarios");
+const git_1 = require("../utils/git");
+exports.browserAgentTool = {
+    schema: {
+        type: "function",
+        function: {
+            name: "generateTestWithBrowserAgent",
+            description: `
+Create or modify a test case with browser agent. The browser agent can take user interactions in a web browser
+and generate Playwright code for that actions. This is a useful tool when the modifications require knowing the 
+locator/selector for an element on the page.
+
+IMPORTANT: Before you invoke this tool, you need to ensure that the test code is correctly prepared for this
+agent. Preparation involves adding a TODO comment that describes the change that needs to be made. A good
+comment calls out the element and browser interactions sto take on them. The TODO comment also has (agent) next to it, to
+clearly label that the change is for the agent to make.
+
+For example: This is a good TODO comment
+
+\`\`\`
+test("Example test code", async ({ page }) => {
+  await page.goto("https://example.com");
+  // TODO(agent): Click on the login button
+});
+\`\`\`
+
+The browser agent will execute the steps before the TODO comment and replace the TODO comment with the Playwright
+code that performs the actions described in the comment. For instance, on the running the tool, the agent will
+output the following final code:
+
+\`\`\`
+test("Example test code", async ({ page }) => {
+  await page.goto("https://example.com");
+  await page.getByRole("button", { name: "Login" }).click();
+});
+\`\`\`
+      `,
+            parameters: {
+                type: "object",
+                properties: {
+                    testName: {
+                        type: "string",
+                        description: "The name of the test to create or modify",
+                    },
+                    testSuites: {
+                        type: "array",
+                        description: "The suites (describe blocks) where the test is located",
+                        items: {
+                            type: "string",
+                        },
+                    },
+                    fileName: {
+                        type: "string",
+                        description: "The name of the file where the test is located. File name must end with .spec.ts",
+                    },
+                    project: {
+                        type: "string",
+                        description: "The Playwright project to run tests against (e.g. 'chromium' or 'firefox')",
+                    },
+                    changeToMake: {
+                        type: "string",
+                        description: "The change to make to the test",
+                    },
+                },
+                required: [
+                    "testName",
+                    "testSuites",
+                    "fileName",
+                    "changeToMake",
+                    "project",
+                ],
+            },
+        },
+    },
+    execute: async (input) => {
+        const { testName, testSuites, fileName, changeToMake, project } = input;
+        await (0, utils_1.replaceTodoWithCreateTest)({
+            testCaseName: testName,
+            testCaseSuites: testSuites,
+            testFilePath: fileName,
+        });
+        const { isError, error } = await (0, run_1.generateTestsUsingMasterAgent)({
+            testFilePath: fileName,
+            filePathToUpdate: fileName,
+            pwProjectsFilter: [project],
+            testGenToken: (0, scenarios_1.buildTokenFromOptions)({
+                name: testName,
+                file: fileName,
+                prompt: changeToMake,
+            }),
+            repoDir: process.cwd(),
+        });
+        if (!isError) {
+            const gitPatch = (0, git_1.getGitDiff)(fileName);
+            return {
+                isError,
+                result: `Test was generated successfully. Here is the git patch:
+\`\`\`
+${gitPatch}
+\`\`\`
+`,
+            };
+        }
+        else {
+            return {
+                isError,
+                result: `Test was not generated successfully. Here is the error:
+\`\`\`
+${error}
+\`\`\`
+`,
+            };
+        }
+    },
+};

package/dist/tools/codegen-agent.d.ts

@@ -0,0 +1,3 @@
+import type { Tool } from "./types";
+export declare const codegenTool: Tool;
+//# sourceMappingURL=codegen-agent.d.ts.map

package/dist/tools/codegen-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codegen-agent.d.ts","sourceRoot":"","sources":["../../src/tools/codegen-agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AASpC,eAAO,MAAM,WAAW,EAAE,IAsDzB,CAAC"}

package/dist/tools/codegen-agent.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.codegenTool = void 0;
+const run_1 = require("../agent/codegen/run");
+exports.codegenTool = {
+    schema: {
+        type: "function",
+        function: {
+            name: "generateTestWithCodegen",
+            description: "Create or modify a test case with code generation. This is useful when modifications can be done with TypeScript only, and don't require any browser interactions or element selectors.",
+            parameters: {
+                type: "object",
+                properties: {
+                    testName: {
+                        type: "string",
+                        description: "The name of the test to create or modify",
+                    },
+                    testSuites: {
+                        type: "array",
+                        description: "The suites (describe blocks) where the test is located",
+                        items: {
+                            type: "string",
+                        },
+                    },
+                    fileName: {
+                        type: "string",
+                        description: "The name of the file where the test is located. File name must end with .spec.ts",
+                    },
+                    changeToMake: {
+                        type: "string",
+                        description: "The change to make to the test",
+                    },
+                },
+                required: ["testName", "testSuites", "fileName", "changeToMake"],
+            },
+        },
+    },
+    execute: async (input) => {
+        const { testName, testSuites, fileName, changeToMake } = input;
+        const testCase = {
+            id: 0,
+            name: testName,
+            filePath: fileName,
+            suites: testSuites,
+            steps: [changeToMake],
+        };
+        const result = await (0, run_1.generateTestWithCodegen)({
+            testCase,
+            file: fileName,
+        });
+        return {
+            result: JSON.stringify(result),
+            isError: false,
+        };
+    },
+};

package/dist/tools/diagnosis-fetcher.d.ts

@@ -0,0 +1,3 @@
+import type { Tool } from "./types";
+export declare const diagnosisTool: Tool;
+//# sourceMappingURL=diagnosis-fetcher.d.ts.map

package/dist/tools/diagnosis-fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diagnosis-fetcher.d.ts","sourceRoot":"","sources":["../../src/tools/diagnosis-fetcher.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAMpC,eAAO,MAAM,aAAa,EAAE,IA4F3B,CAAC"}

package/dist/tools/diagnosis-fetcher.js

@@ -0,0 +1,88 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.diagnosisTool = void 0;
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+exports.diagnosisTool = {
+    schema: {
+        type: "function",
+        function: {
+            name: "fetchDiagnosisDetails",
+            description: "Fetch details about a test case diagnosis using its URL or slug",
+            parameters: {
+                type: "object",
+                properties: {
+                    diagnosisUrl: {
+                        type: "string",
+                        description: "The full URL of the diagnosis (e.g. https://dash.empirical.run/shopflo-tests/diagnosis/split-cod-place-the-order--byynrPjCml57)",
+                    },
+                },
+                required: ["diagnosisUrl"],
+            },
+        },
+    },
+    execute: async (input) => {
+        const { diagnosisUrl } = input;
+        // Extract the slug from the URL - it's the part after the last '--'
+        const slug = diagnosisUrl.split("--").pop();
+        if (!slug) {
+            throw new Error("Invalid diagnosis URL - could not extract slug");
+        }
+        // Make the API call to fetch diagnosis details
+        const response = await fetch(`https://dash.empirical.run/api/diagnosis/${slug}/detailed`, {
+            method: "GET",
+            headers: {
+                Authorization: "weQPMWKT", // Using the auth token from test-endpoint.mdc
+            },
+        });
+        if (!response.ok) {
+            return {
+                result: `Failed to fetch diagnosis details: ${response.statusText}`,
+                isError: true,
+            };
+        }
+        const data = await response.json();
+        const { test_case, diagnosis } = data.data;
+        const project = diagnosis[0]?.test_project || "unknown";
+        const sourceContext = await promises_1.default.readFile(path_1.default.join("tests", test_case.file_path), "utf-8");
+        // Format the response as markdown
+        const markdownResponse = `
+# Test Case Diagnosis
+
+## Test Case Information
+- **Test Case Name**: ${test_case.name}
+- **Test Suite**: ${test_case.suites}
+- **File Name**: tests/${test_case.file_path}
+- **Project**: ${project}
+
+## Source Context
+${sourceContext}
+
+## What Happened in the Test Run
+
+### Failure Details
+- **Failing Line**: ${diagnosis[0]?.failing_line || "No failing line available"}
+
+#### Error Stack
+\`\`\`
+${diagnosis[0]?.failed_run_metadata?.stack?.replace("/runner/_work/shopflo-tests/shopflo-tests/source-repo/", "") || "No error stack available"}
+\`\`\`
+
+#### Error Summary
+${diagnosis[0]?.error_stack_summary?.content || "No error summary available"}
+
+#### Visual Analysis
+${diagnosis[0]?.visual_diff_summary?.summary || "No visual analysis available"}
+
+#### Merged Summary
+${diagnosis[0]?.merged_summary?.content || "No merged summary available"}
+`;
+        return {
+            result: markdownResponse,
+            isError: false,
+        };
+    },
+};

package/dist/tools/test-run.d.ts

@@ -0,0 +1,3 @@
+import type { Tool } from "./types";
+export declare const runTestTool: Tool;
+//# sourceMappingURL=test-run.d.ts.map

package/dist/tools/test-run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AASpC,eAAO,MAAM,WAAW,EAAE,IA8CzB,CAAC"}

package/dist/tools/test-run.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runTestTool = void 0;
+const test_run_1 = require("@empiricalrun/test-run");
+exports.runTestTool = {
+    schema: {
+        type: "function",
+        function: {
+            name: "runTest",
+            description: "Run a test",
+            parameters: {
+                type: "object",
+                properties: {
+                    testName: {
+                        type: "string",
+                        description: "The name of the test to run",
+                    },
+                    suites: {
+                        type: "array",
+                        description: "The suites (describe blocks) where the test is located.",
+                        items: { type: "string" },
+                    },
+                    fileName: {
+                        type: "string",
+                        description: "The name of the file where the test is located. File name must end with .spec.ts",
+                    },
+                    project: {
+                        type: "string",
+                        description: "The project to run the test on",
+                    },
+                },
+                required: ["testName", "suites", "fileName", "project"],
+            },
+        },
+    },
+    execute: async (input) => {
+        const { testName, suites, fileName, project } = input;
+        const result = await (0, test_run_1.runSingleTest)({
+            testName,
+            suites,
+            fileName,
+            projects: [project],
+        });
+        return {
+            result: JSON.stringify(result),
+            isError: false,
+        };
+    },
+};

package/dist/tools/types.d.ts

@@ -0,0 +1,11 @@
+import type { OpenAI } from "openai";
+export type ToolSchema = OpenAI.Chat.Completions.ChatCompletionTool;
+export type ToolResult = {
+    result: string;
+    isError: boolean;
+};
+export type Tool = {
+    schema: ToolSchema;
+    execute: (input: any) => Promise<ToolResult>;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/tools/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/tools/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEpE,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,IAAI,GAAG;IAIjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CAC9C,CAAC"}

package/dist/tools/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/git.d.ts

@@ -0,0 +1,2 @@
+export declare function getGitDiff(filepath: string): string;
+//# sourceMappingURL=git.d.ts.map

package/dist/utils/git.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/utils/git.ts"],"names":[],"mappings":"AAEA,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKnD"}

package/dist/utils/git.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getGitDiff = void 0;
+const child_process_1 = require("child_process");
+function getGitDiff(filepath) {
+    const diff = (0, child_process_1.execSync)(`git diff ${filepath}`, {
+        encoding: "utf-8",
+    });
+    return diff;
+}
+exports.getGitDiff = getGitDiff;

package/dist/utils/repo-tree.d.ts

@@ -0,0 +1,2 @@
+export declare function generateAsciiTree(dirPath: string, options?: {}): string;
+//# sourceMappingURL=repo-tree.d.ts.map

package/dist/utils/repo-tree.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"repo-tree.d.ts","sourceRoot":"","sources":["../../src/utils/repo-tree.ts"],"names":[],"mappings":"AAYA,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,KAAK,UAsE9D"}

package/dist/utils/repo-tree.js

@@ -0,0 +1,75 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateAsciiTree = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const DEFAULT_EXCLUDE = [
+    "node_modules",
+    "dist",
+    "build",
+    /\.git/,
+    ".DS_Store",
+    "playwright-report",
+];
+function generateAsciiTree(dirPath, options = {}) {
+    const defaultOptions = {
+        showHidden: false,
+        exclude: DEFAULT_EXCLUDE,
+        maxDepth: 10,
+    };
+    const opts = { ...defaultOptions, ...options };
+    // Make sure the path exists and is a directory
+    if (!fs_1.default.existsSync(dirPath) || !fs_1.default.statSync(dirPath).isDirectory()) {
+        throw new Error(`"${dirPath}" is not a valid directory path`);
+    }
+    // Start with the root directory name
+    const rootName = path_1.default.basename(dirPath);
+    let result = rootName + "\n";
+    function processDirectory(currentPath, prefix = "", depth = 1) {
+        if (depth > opts.maxDepth)
+            return;
+        const items = fs_1.default.readdirSync(currentPath);
+        // Sort items: directories first, then files
+        const sortedItems = items.sort((a, b) => {
+            const aIsDir = fs_1.default.statSync(path_1.default.join(currentPath, a)).isDirectory();
+            const bIsDir = fs_1.default.statSync(path_1.default.join(currentPath, b)).isDirectory();
+            if (aIsDir && !bIsDir)
+                return -1;
+            if (!aIsDir && bIsDir)
+                return 1;
+            return a.localeCompare(b);
+        });
+        // Process each item
+        sortedItems.forEach((item, index) => {
+            // Skip hidden files if not showing hidden
+            if (!opts.showHidden && item.startsWith("."))
+                return;
+            // Skip excluded patterns
+            if (opts.exclude.some((pattern) => typeof pattern === "string"
+                ? item === pattern
+                : pattern instanceof RegExp
+                    ? pattern.test(item)
+                    : false))
+                return;
+            const itemPath = path_1.default.join(currentPath, item);
+            const isDirectory = fs_1.default.statSync(itemPath).isDirectory();
+            const isLast = index === sortedItems.length - 1;
+            // Current item symbols
+            const symbol = isLast ? "└── " : "├── ";
+            const nextPrefix = isLast ? "    " : "│   ";
+            // Add the current item to the result
+            result += `${prefix}${symbol}${item}${isDirectory ? "/" : ""}\n`;
+            // Process subdirectories
+            if (isDirectory) {
+                processDirectory(itemPath, prefix + nextPrefix, depth + 1);
+            }
+        });
+    }
+    // Start the recursive processing
+    processDirectory(dirPath);
+    return result;
+}
+exports.generateAsciiTree = generateAsciiTree;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.47.3",
+  "version": "0.48.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -73,9 +73,10 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.36",
+    "@empiricalrun/llm": "^0.10.0",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/reporter": "^0.23.1"
+    "@empiricalrun/reporter": "^0.23.1",
+    "@empiricalrun/test-run": "^0.7.2"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",