@empiricalrun/test-gen 0.41.1 → 0.41.2

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @empiricalrun/test-gen
 
+## 0.41.2
+
+### Patch Changes
+
+- 8d6f8be: fix: enable type check in repo agent
+- 6573a58: fix: move generate requested change method to test-gen package
+- 54e7e03: fix: avoid uploading screenshots if reporter config is not set
+- 3eff6a8: fix: updated prompt for create test
+
 ## 0.41.1
 
 ### Patch Changes

package/dist/agent/codegen/repo-edit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAexE,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,KAAK,EACL,SAAS,GACV,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CA6GD;AAED,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CAwED"}
+{"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAexE,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,KAAK,EACL,SAAS,GACV,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CA6GD;AAED,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CAuED"}

package/dist/agent/codegen/repo-edit.js

@@ -29,7 +29,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.repoEditAgent = exports.generateCodeUsingRepoAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fsSync = __importStar(require("fs"));
-const promises_1 = __importDefault(require("fs/promises"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = require("path");
 const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
@@ -147,6 +147,7 @@ Task: ${task}
 }
 exports.generateCodeUsingRepoAgent = generateCodeUsingRepoAgent;
 async function repoEditAgent({ trace, task, }) {
+    console.log("mock test", fs_extra_1.default.readdir(""));
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
     const { prompt: repoFiles } = await (0, context_1.generateTxtForRepository)();
@@ -178,23 +179,22 @@ async function repoEditAgent({ trace, task, }) {
             fileChanges: updatedFileChanges,
         });
     }
-    await (0, utils_1.validateTypesAndFormatCode)({
-        validateTypes: false,
-        trace,
-        testCase: {},
-        fileChanges: updates.filter((f) => f.filePath && fsSync.existsSync(f.filePath)),
-    });
     const newFileUpdates = updates.filter((f) => !fsSync.existsSync(f.filePath));
     await Promise.all(newFileUpdates.map((f) => {
         return (async () => {
-            await promises_1.default.mkdir((0, path_1.dirname)(f.filePath), { recursive: true });
-            await promises_1.default.writeFile(f.filePath, f.newCode, "utf-8");
+            await fs_extra_1.default.mkdir((0, path_1.dirname)(f.filePath), { recursive: true });
+            await fs_extra_1.default.writeFile(f.filePath, f.newCode, "utf-8");
         })();
     }));
+    await (0, utils_1.validateTypesAndFormatCode)({
+        validateTypes: true,
+        trace,
+        testCase: {},
+        fileChanges: updates,
+    });
     await Promise.all(updates.map((f) => {
         return (async () => {
             if (fsSync.existsSync(f.filePath)) {
-                await (0, web_1.formatCode)(f.filePath);
                 await (0, web_1.lintErrors)(f.filePath);
             }
         })();

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAsC5B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAK7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAkC5B"}

package/dist/agent/codegen/run.js

@@ -11,6 +11,7 @@ const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const repo_edit_1 = require("./repo-edit");
 const update_flow_1 = require("./update-flow");
+const utils_1 = require("./utils");
 async function generateTest(testCase, file, options, trace) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(file)) {
@@ -37,14 +38,10 @@ async function generateTest(testCase, file, options, trace) {
             testFilePath: file,
         },
     });
-    const task = `
-  Create a new test at the given test file path and perform the relevant changes required:\n
-    Scenario name: ${testCase.name}
-    Test file path: ${file}
-    Test suite: ${testCase.suites?.join("->") || "-"}
-    Scenario: ${testCase.steps.join("\n")}
-  `;
-    const repoAgentResponse = await (0, repo_edit_1.repoEditAgent)({ trace, task });
+    const repoAgentResponse = await (0, repo_edit_1.repoEditAgent)({
+        trace,
+        task: (0, utils_1.getTaskForCreateTest)({ testCase, file }),
+    });
     createTestSpan?.end({ output: repoAgentResponse.fileChanges });
     await (0, llm_1.flushAllTraces)();
 }

package/dist/agent/codegen/utils.d.ts

@@ -84,4 +84,8 @@ export declare function validateTypesAndFormatCode({ validateTypes, trace, testC
     pomPrompt?: string;
     nonSpecFilePrompt?: string;
 }): Promise<void>;
+export declare function getTaskForCreateTest({ testCase, file, }: {
+    testCase: TestCase;
+    file: string;
+}): string;
 //# sourceMappingURL=utils.d.ts.map

package/dist/agent/codegen/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAwIxE;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA6BA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAwIxE;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA6BA;AAED,wBAAgB,oBAAoB,CAAC,EACnC,QAAQ,EACR,IAAI,GACL,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd,UAaA"}

package/dist/agent/codegen/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateTypesAndFormatCode = exports.applyFileChanges = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractTestUpdates = void 0;
+exports.getTaskForCreateTest = exports.validateTypesAndFormatCode = exports.applyFileChanges = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractTestUpdates = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const ts_morph_1 = require("ts-morph");
@@ -229,3 +229,16 @@ async function validateTypesAndFormatCode({ validateTypes = true, trace, testCas
     await (0, llm_1.flushAllTraces)();
 }
 exports.validateTypesAndFormatCode = validateTypesAndFormatCode;
+function getTaskForCreateTest({ testCase, file, }) {
+    return `
+Create a new test case with name '${testCase.name}' at file path '${file}'
+${testCase.suites.length
+        ? `
+Put this test case inside describe blocks named: ${testCase.suites.join(" > ")}`
+        : ""}
+
+In the above test, write code that performs these actions and ONLY these actions:
+${testCase.steps.join("\n")}
+  `;
+}
+exports.getTaskForCreateTest = getTaskForCreateTest;

package/dist/agent/enrich-prompt/index.d.ts

@@ -0,0 +1,14 @@
+import { TraceClient } from "@empiricalrun/llm";
+type ArgsT = {
+    testBlock: string;
+    testFilePath: string;
+    suggestionForFix: string;
+    trace?: TraceClient;
+};
+export declare const enrichPromptWithFailingLine: ({ trace, testBlock, testFilePath, suggestionForFix, }: ArgsT) => Promise<{
+    output: string;
+    is_user_message_enriched: boolean;
+    reason_for_output: string;
+}>;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/enrich-prompt/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/enrich-prompt/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAKhE,KAAK,KAAK,GAAG;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,CAAC;AAgCF,eAAO,MAAM,2BAA2B,0DAKrC,KAAK;YACE,MAAM;8BACY,OAAO;uBACd,MAAM;EA+D1B,CAAC"}

package/dist/agent/enrich-prompt/index.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.enrichPromptWithFailingLine = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const constants_1 = require("../../constants");
+const utils_1 = require("./utils");
+const promptName = "generate-self-heal-requested-change";
+const responseFormat = {
+    type: "json_schema",
+    json_schema: {
+        name: "analyze_user_message_and_conditionally_enrich_it",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                reason_for_output: {
+                    type: "string",
+                    description: "provide reason in both cases whether the user message was enriched or not",
+                },
+                is_user_message_enriched: {
+                    type: "boolean",
+                    description: "whether the user message should be enriched or not",
+                },
+                output: {
+                    type: "string",
+                    description: "This is the final output after analyzing the user message, and optionally enriching it.",
+                },
+            },
+            required: ["is_user_message_enriched", "output", "reason_for_output"],
+            additionalProperties: false,
+        },
+    },
+};
+// TODO: fix the format
+const enrichPromptWithFailingLine = async ({ trace, testBlock, testFilePath, suggestionForFix, }) => {
+    let output = {
+        output: suggestionForFix,
+        is_user_message_enriched: false,
+        reason_for_output: "",
+    };
+    try {
+        const instructions = await (0, llm_1.getPrompt)(promptName, {
+            testBlock,
+            testFilePath,
+            userMessage: suggestionForFix,
+        }, 26);
+        trace?.event({
+            name: "get prompt",
+            input: {
+                promptName,
+                testBlock,
+                testFilePath,
+                userMessage: suggestionForFix,
+            },
+            output: { instructions },
+        });
+        const llm = new llm_1.LLM({
+            trace,
+            providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
+            provider: "anthropic",
+            defaultModel: "claude-3-5-sonnet-latest",
+        });
+        const llmResponse = await llm.createChatCompletion({
+            messages: instructions,
+            trace,
+            modelParameters: {
+                temperature: 0.5,
+            },
+            responseFormat,
+        });
+        const generateRequestedChangeResponse = (0, utils_1.extractGenerateRequestedChange)(llmResponse?.content);
+        output = {
+            output: generateRequestedChangeResponse.output,
+            is_user_message_enriched: generateRequestedChangeResponse.is_user_message_enriched,
+            reason_for_output: generateRequestedChangeResponse.reason_for_output,
+        };
+    }
+    catch (e) {
+        console.error(`Failed to generate requested change for test - "${testBlock}", suggestion - "${suggestionForFix}"`, e);
+    }
+    return output;
+};
+exports.enrichPromptWithFailingLine = enrichPromptWithFailingLine;

package/dist/agent/enrich-prompt/utils.d.ts

@@ -0,0 +1,6 @@
+export declare function extractGenerateRequestedChange(input: string): {
+    reason_for_output: string;
+    is_user_message_enriched: boolean;
+    output: string;
+};
+//# sourceMappingURL=utils.d.ts.map

package/dist/agent/enrich-prompt/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/enrich-prompt/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC7D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,wBAAwB,EAAE,OAAO,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;CAChB,CAQA"}

package/dist/agent/enrich-prompt/utils.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractGenerateRequestedChange = void 0;
+function extractGenerateRequestedChange(input) {
+    const regex = /<reason_for_output>([\s\S]*?)<\/reason_for_output>[\s\S]*?<is_user_message_enriched>([\s\S]*?)<\/is_user_message_enriched>[\s\S]*?<output>([\s\S]*?)<\/output>/g;
+    const match = regex.exec(input);
+    const reason_for_output = match[1]?.trim() || "";
+    const is_user_message_enriched = match[2]?.trim() === "true";
+    const output = match[3]?.trim() || "";
+    return { reason_for_output, is_user_message_enriched, output };
+}
+exports.extractGenerateRequestedChange = extractGenerateRequestedChange;

package/dist/evals/add-scenario-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AA+BpC,QAAA,MAAM,4BAA4B,EAAE,UA6BnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
+{"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAqCpC,QAAA,MAAM,4BAA4B,EAAE,UAqBnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/add-scenario-agent.evals.js

@@ -1,10 +1,19 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = __importDefault(require("path"));
 const repo_edit_1 = require("../agent/codegen/repo-edit");
+const utils_1 = require("../agent/codegen/utils");
 function evaluateEqualityScore({ currentOutput, expectedOutput, }) {
     for (const expectedFile of expectedOutput) {
         if (!currentOutput.find((output) => {
-            return output.filePath === expectedFile.filePath;
+            if (output.filePath && expectedFile.filePath) {
+                return (path_1.default.basename(output.filePath) ===
+                    path_1.default.basename(expectedFile.filePath));
+            }
+            return false;
         })) {
             return 0;
         }
@@ -13,16 +22,9 @@ function evaluateEqualityScore({ currentOutput, expectedOutput, }) {
 }
 const addScenarioCodeAgentEvaluate = async ({ item, trace }) => {
     const { testCase, testFilePath, pageFiles, testFiles } = item.input;
-    const task = `
-  Create a new test at the given test file path and perform the relevant changes required:\n
-  Scenario name: ${testCase.name}
-  Test file path: ${testFilePath}
-  Test suite: ${testCase.suites?.join("->") || "-"}
-  Scenario: ${testCase.steps.join("\n")}
-`;
     const repoFiles = pageFiles + testFiles;
     const repoAgentOutput = await (0, repo_edit_1.generateCodeUsingRepoAgent)({
-        task,
+        task: (0, utils_1.getTaskForCreateTest)({ testCase, file: testFilePath }),
         trace,
         repoFiles,
     });

package/dist/reporter/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAa5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAgDK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3C,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1C,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAWxD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAa5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAgDK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+C9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3C,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1C,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAWxD"}

package/dist/reporter/index.js

@@ -95,6 +95,10 @@ class TestGenUpdatesReporter {
             logger.log("Skipped uploading current view screenshot");
             return;
         }
+        if (!reporterConfig) {
+            logger.log("Reporter config is not set skipping screenshot uploads.");
+            return;
+        }
         try {
             // upload current screenshot to r2 and report it to reporter
             if (!fs_extra_1.default.existsSync(path_1.default.join(process.cwd(), "gen-assets"))) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.41.1",
+  "version": "0.41.2",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -22,6 +22,10 @@
       "types": "./dist/agent/master/planner.d.ts",
       "default": "./dist/agent/master/planner.js"
     },
+    "./agent/enrich-prompt": {
+      "types": "./dist/agent/enrich-prompt/index.d.ts",
+      "default": "./dist/agent/enrich-prompt/index.js"
+    },
     "./types": {
       "types": "./dist/types/index.d.ts",
       "default": "./dist/types/index.js"