@empiricalrun/test-gen 0.39.0 → 0.40.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @empiricalrun/test-gen
 
+## 0.40.0
+
+### Minor Changes
+
+- 451c840: feat: pro 1165 adding interactible live stream of test generation
+
+### Patch Changes
+
+- 7989789: fix: replaced implementation for create test with repo edit
+
 ## 0.39.0
 
 ### Minor Changes

package/dist/agent/codegen/repo-edit.d.ts

@@ -1,6 +1,30 @@
 import { TraceClient } from "@empiricalrun/llm";
+import { ChatCompletionMessageParam } from "openai/resources/index.mjs";
+export declare function generateCodeUsingRepoAgent({ task, trace, repoFiles, }: {
+    trace?: TraceClient;
+    task: string;
+    repoFiles?: string;
+}): Promise<{
+    prompt: ChatCompletionMessageParam[];
+    agentResponse: string;
+    fileChanges: {
+        filePath: string | undefined;
+        oldCode: string | undefined;
+        newCode: string | undefined;
+        reason: string | undefined;
+    }[];
+}>;
 export declare function repoEditAgent({ trace, task, }: {
     trace?: TraceClient;
     task: string;
-}): Promise<void>;
+}): Promise<{
+    prompt: ChatCompletionMessageParam[];
+    agentResponse: string;
+    fileChanges: {
+        filePath: string | undefined;
+        oldCode: string | undefined;
+        newCode: string | undefined;
+        reason: string | undefined;
+    }[];
+}>;
 //# sourceMappingURL=repo-edit.d.ts.map

package/dist/agent/codegen/repo-edit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAcrD,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,iBA+JA"}
+{"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAWxE,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,KAAK,EACL,SAAS,GACV,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CA6GD;AAED,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CAgED"}

package/dist/agent/codegen/repo-edit.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.repoEditAgent = void 0;
+exports.repoEditAgent = exports.generateCodeUsingRepoAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fsSync = __importStar(require("fs"));
 const promises_1 = __importDefault(require("fs/promises"));
@@ -37,15 +37,12 @@ const constants_1 = require("../../constants");
 const reporter_1 = require("../../reporter");
 const test_update_feedback_1 = require("./test-update-feedback");
 const utils_1 = require("./utils");
-async function repoEditAgent({ trace, task, }) {
-    const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
-    void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
+async function generateCodeUsingRepoAgent({ task, trace, repoFiles, }) {
     const repoEditSpan = trace?.span({
         name: "repo-edit",
         input: { task },
     });
     // TODO: add support for playwright config and other files
-    const { prompt: repoPrompt } = await (0, context_1.generateTxtForRepository)();
     const prompt = [
         {
             role: "system",
@@ -95,7 +92,7 @@ Coding principles and guidelines:
 - Use the "function" keyword for pure functions to benefit from hoisting and clarity.
 
 Here is the list of files:
-${repoPrompt}  
+${repoFiles}  
 `,
         },
         {
@@ -142,6 +139,23 @@ Task: ${task}
             task,
         },
     });
+    return {
+        prompt,
+        agentResponse: updatedUsageExampleMessage?.content,
+        fileChanges: updates,
+    };
+}
+exports.generateCodeUsingRepoAgent = generateCodeUsingRepoAgent;
+async function repoEditAgent({ trace, task, }) {
+    const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
+    void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
+    const { prompt: repoFiles } = await (0, context_1.generateTxtForRepository)();
+    const repoAgentOutput = await generateCodeUsingRepoAgent({
+        task,
+        trace,
+        repoFiles,
+    });
+    const updates = repoAgentOutput.fileChanges;
     const fileUpdateResponses = await (0, utils_1.applyFileChanges)({
         validateTypes: false,
         trace,
@@ -152,8 +166,8 @@ Task: ${task}
     if (errorResponses.length > 0) {
         const updatedFileChanges = await (0, test_update_feedback_1.applyTestUpdateFeedbacks)({
             trace,
-            oldPrompt: prompt,
-            agentResponse: updatedUsageExampleMessage?.content,
+            oldPrompt: repoAgentOutput.prompt,
+            agentResponse: repoAgentOutput.agentResponse,
             feedbacks: errorResponses.map((e) => ({
                 filePath: e?.filePath,
                 errorMessage: e?.errorMessage,
@@ -182,5 +196,6 @@ Task: ${task}
         })();
     }));
     await testgenUpdatesReporter.sendMessage(`Successfully generated code for the given task. \n View [trace](${trace?.getTraceUrl()})`);
+    return repoAgentOutput;
 }
 exports.repoEditAgent = repoEditAgent;

package/dist/agent/codegen/run.d.ts

@@ -1,12 +1,4 @@
 import { TraceClient } from "@empiricalrun/llm";
 import { TestCase, TestGenConfigOptions } from "../../types";
-export declare function getAddScenarioCompletion({ testCase, testFiles, pageFiles, testFilePath, trace, options, }: {
-    testCase: TestCase;
-    testFiles: string;
-    pageFiles: string;
-    testFilePath: string;
-    trace?: TraceClient;
-    options?: TestGenConfigOptions;
-}): Promise<string>;
-export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[]>;
+export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[] | void>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkC,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAkBhF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,mBA4BA;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgFrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAsC5B"}

package/dist/agent/codegen/run.js

@@ -3,44 +3,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTest = exports.getAddScenarioCompletion = void 0;
+exports.generateTest = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../../bin/logger");
 const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
-const constants_1 = require("../../constants");
-const fix_ts_errors_1 = require("./fix-ts-errors");
+const repo_edit_1 = require("./repo-edit");
 const update_flow_1 = require("./update-flow");
-async function getAddScenarioCompletion({ testCase, testFiles, pageFiles, testFilePath, trace, options, }) {
-    const promptSpan = trace?.span({
-        name: "add-scenario-prompt",
-    });
-    const instruction = await (0, llm_1.getPrompt)("add-scenario", {
-        testFiles: testFiles,
-        pageFiles: pageFiles,
-        scenarioName: testCase.name,
-        scenarioSteps: testCase.steps.join("\n"),
-        scenarioFile: testFilePath,
-    });
-    promptSpan?.end({ output: { instruction } });
-    const llm = new llm_1.LLM({
-        trace,
-        provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-        defaultModel: options?.model || constants_1.DEFAULT_MODEL,
-        providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
-    });
-    const firstShotMessage = await llm.createChatCompletion({
-        messages: instruction,
-        modelParameters: {
-            ...constants_1.DEFAULT_MODEL_PARAMETERS,
-            ...options?.modelParameters,
-        },
-    });
-    let response = firstShotMessage?.content || "";
-    return response;
-}
-exports.getAddScenarioCompletion = getAddScenarioCompletion;
 async function generateTest(testCase, file, options, trace) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(file)) {
@@ -48,7 +18,7 @@ async function generateTest(testCase, file, options, trace) {
         fs_extra_1.default.createFileSync(file);
     }
     const context = await (0, context_1.contextForGeneration)(file);
-    const { codePrompt, pomPrompt, nonSpecFilePrompt, testFileContent } = context;
+    const { codePrompt, pomPrompt, testFileContent } = context;
     const { testBlock } = (0, web_1.getTypescriptTestBlock)({
         scenarioName: testCase?.name,
         content: testFileContent,
@@ -58,8 +28,6 @@ async function generateTest(testCase, file, options, trace) {
     if (isUpdate) {
         return await (0, update_flow_1.updateTest)(testCase, file, options, true, true, trace);
     }
-    const generatedTestCases = [];
-    logger.logEmptyLine();
     const createTestSpan = trace?.span({
         name: "create-test",
         input: {
@@ -69,47 +37,15 @@ async function generateTest(testCase, file, options, trace) {
             testFilePath: file,
         },
     });
-    const response = await getAddScenarioCompletion({
-        testCase,
-        testFiles: codePrompt,
-        pageFiles: pomPrompt,
-        testFilePath: file,
-        trace: createTestSpan,
-        options,
-    });
-    logger.success("Test generated successfully!");
-    const readWriteFileSpan = trace?.span({ name: "write-to-file" });
-    let contents = fs_extra_1.default.readFileSync(file, "utf-8");
-    const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response, testCase?.name);
-    let updatedContent = prependContent +
-        (0, web_1.injectCodeSnippetBySuiteChain)({
-            testFileContent: contents,
-            suites: testCase?.suites,
-            codeSnippet: `\n\n${strippedContent}`,
-        });
-    await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
-    readWriteFileSpan?.end({ output: { updatedContent } });
-    logger.log("Linting generated code...");
-    createTestSpan?.event({ name: "lint-file" });
-    await (0, web_1.lintErrors)(file);
-    await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
-        trace,
-        logger: new logger_1.CustomLogger({ useReporter: false }),
-        file,
-        pomCode: pomPrompt,
-        nonSpecFileCode: nonSpecFilePrompt,
-        testCase: testCase,
-        options,
-    });
-    createTestSpan?.event({ name: "format-file" });
-    await (0, web_1.formatCode)(file);
-    logger.success("File formatted successfully!");
-    if (trace) {
-        logger.log(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
-    }
-    generatedTestCases.push(testCase);
-    createTestSpan?.end({ output: { response } });
+    const task = `
+  Create a new test at the given test file path and perform the relevant changes required:\n
+    Scenario name: ${testCase.name}
+    Test file path: ${file}
+    Test suite: ${testCase.suites?.join("->") || "-"}
+    Scenario: ${testCase.steps.join("\n")}
+  `;
+    const repoAgentResponse = await (0, repo_edit_1.repoEditAgent)({ trace, task });
+    createTestSpan?.end({ output: repoAgentResponse.fileChanges });
     await (0, llm_1.flushAllTraces)();
-    return generatedTestCases;
 }
 exports.generateTest = generateTest;

package/dist/evals/add-scenario-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,QAAA,MAAM,4BAA4B,EAAE,UAkBnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
+{"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AA+BpC,QAAA,MAAM,4BAA4B,EAAE,UA6BnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/add-scenario-agent.evals.js

@@ -1,23 +1,42 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const run_1 = require("../agent/codegen/run");
+const repo_edit_1 = require("../agent/codegen/repo-edit");
+function evaluateEqualityScore({ currentOutput, expectedOutput, }) {
+    for (const expectedFile of expectedOutput) {
+        if (!currentOutput.find((output) => {
+            return output.filePath === expectedFile.filePath;
+        })) {
+            return 0;
+        }
+    }
+    return 1;
+}
 const addScenarioCodeAgentEvaluate = async ({ item, trace }) => {
-    const { testCase, testFiles, pageFiles, testFilePath } = item.input;
-    const response = await (0, run_1.getAddScenarioCompletion)({
-        testCase,
-        testFiles,
-        pageFiles,
-        testFilePath,
+    const { testCase, testFilePath, pageFiles, testFiles } = item.input;
+    const task = `
+  Create a new test at the given test file path and perform the relevant changes required:\n
+  Scenario name: ${testCase.name}
+  Test file path: ${testFilePath}
+  Test suite: ${testCase.suites?.join("->") || "-"}
+  Scenario: ${testCase.steps.join("\n")}
+`;
+    const repoFiles = pageFiles + testFiles;
+    const repoAgentOutput = await (0, repo_edit_1.generateCodeUsingRepoAgent)({
+        task,
         trace,
+        repoFiles,
     });
     return {
         scores: [
             {
                 name: "equality",
-                value: item.expectedOutput === response ? 1 : 0,
+                value: evaluateEqualityScore({
+                    currentOutput: repoAgentOutput.fileChanges,
+                    expectedOutput: item.expectedOutput,
+                }),
             },
         ],
-        output: response,
+        output: repoAgentOutput.fileChanges,
     };
 };
 exports.default = addScenarioCodeAgentEvaluate;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.39.0",
+  "version": "0.40.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -70,9 +70,9 @@
     "ts-morph": "^24.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
+    "@empiricalrun/reporter": "^0.21.6",
     "@empiricalrun/llm": "^0.9.29",
-    "@empiricalrun/r2-uploader": "^0.3.7",
-    "@empiricalrun/reporter": "^0.21.6"
+    "@empiricalrun/r2-uploader": "^0.3.7"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",