@empiricalrun/test-gen 0.39.0 → 0.40.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/dist/agent/codegen/repo-edit.d.ts +25 -1
  3. package/dist/agent/codegen/repo-edit.d.ts.map +1 -1
  4. package/dist/agent/codegen/repo-edit.js +29 -10
  5. package/dist/agent/codegen/run.d.ts +1 -9
  6. package/dist/agent/codegen/run.d.ts.map +1 -1
  7. package/dist/agent/codegen/run.js +12 -76
  8. package/dist/agent/codegen/update-flow.d.ts.map +1 -1
  9. package/dist/agent/codegen/update-flow.js +19 -2
  10. package/dist/agent/codegen/utils.d.ts +18 -4
  11. package/dist/agent/codegen/utils.d.ts.map +1 -1
  12. package/dist/agent/codegen/utils.js +44 -31
  13. package/dist/bin/utils/platform/web/index.d.ts.map +1 -1
  14. package/dist/bin/utils/platform/web/index.js +7 -0
  15. package/dist/evals/add-scenario-agent.evals.d.ts.map +1 -1
  16. package/dist/evals/add-scenario-agent.evals.js +28 -9
  17. package/package.json +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,22 @@
1
1
  # @empiricalrun/test-gen
2
2
 
3
+ ## 0.40.1
4
+
5
+ ### Patch Changes
6
+
7
+ - b45c035: fix: sequentially execute file updates and validation in codegen
8
+ - b3069f1: fix: handle empty files while applying prettier format
9
+
10
+ ## 0.40.0
11
+
12
+ ### Minor Changes
13
+
14
+ - 451c840: feat: pro 1165 adding interactible live stream of test generation
15
+
16
+ ### Patch Changes
17
+
18
+ - 7989789: fix: replaced implementation for create test with repo edit
19
+
3
20
  ## 0.39.0
4
21
 
5
22
  ### Minor Changes
package/dist/agent/codegen/repo-edit.d.ts CHANGED
@@ -1,6 +1,30 @@
1
1
  import { TraceClient } from "@empiricalrun/llm";
2
+ import { ChatCompletionMessageParam } from "openai/resources/index.mjs";
3
+ export declare function generateCodeUsingRepoAgent({ task, trace, repoFiles, }: {
4
+ trace?: TraceClient;
5
+ task: string;
6
+ repoFiles?: string;
7
+ }): Promise<{
8
+ prompt: ChatCompletionMessageParam[];
9
+ agentResponse: string;
10
+ fileChanges: {
11
+ filePath: string | undefined;
12
+ oldCode: string | undefined;
13
+ newCode: string | undefined;
14
+ reason: string | undefined;
15
+ }[];
16
+ }>;
2
17
  export declare function repoEditAgent({ trace, task, }: {
3
18
  trace?: TraceClient;
4
19
  task: string;
5
- }): Promise<void>;
20
+ }): Promise<{
21
+ prompt: ChatCompletionMessageParam[];
22
+ agentResponse: string;
23
+ fileChanges: {
24
+ filePath: string | undefined;
25
+ oldCode: string | undefined;
26
+ newCode: string | undefined;
27
+ reason: string | undefined;
28
+ }[];
29
+ }>;
6
30
  //# sourceMappingURL=repo-edit.d.ts.map
package/dist/agent/codegen/repo-edit.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAcrD,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,iBA+JA"}
1
+ {"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAexE,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,KAAK,EACL,SAAS,GACV,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CA6GD;AAED,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CAwED"}
package/dist/agent/codegen/repo-edit.js CHANGED
@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
26
26
  return (mod && mod.__esModule) ? mod : { "default": mod };
27
27
  };
28
28
  Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.repoEditAgent = void 0;
29
+ exports.repoEditAgent = exports.generateCodeUsingRepoAgent = void 0;
30
30
  const llm_1 = require("@empiricalrun/llm");
31
31
  const fsSync = __importStar(require("fs"));
32
32
  const promises_1 = __importDefault(require("fs/promises"));
@@ -37,15 +37,12 @@ const constants_1 = require("../../constants");
37
37
  const reporter_1 = require("../../reporter");
38
38
  const test_update_feedback_1 = require("./test-update-feedback");
39
39
  const utils_1 = require("./utils");
40
- async function repoEditAgent({ trace, task, }) {
41
- const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
42
- void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
40
+ async function generateCodeUsingRepoAgent({ task, trace, repoFiles, }) {
43
41
  const repoEditSpan = trace?.span({
44
42
  name: "repo-edit",
45
43
  input: { task },
46
44
  });
47
45
  // TODO: add support for playwright config and other files
48
- const { prompt: repoPrompt } = await (0, context_1.generateTxtForRepository)();
49
46
  const prompt = [
50
47
  {
51
48
  role: "system",
@@ -95,7 +92,7 @@ Coding principles and guidelines:
95
92
  - Use the "function" keyword for pure functions to benefit from hoisting and clarity.
96
93
 
97
94
  Here is the list of files:
98
- ${repoPrompt}
95
+ ${repoFiles}
99
96
  `,
100
97
  },
101
98
  {
@@ -142,8 +139,24 @@ Task: ${task}
142
139
  task,
143
140
  },
144
141
  });
142
+ return {
143
+ prompt,
144
+ agentResponse: updatedUsageExampleMessage?.content,
145
+ fileChanges: updates,
146
+ };
147
+ }
148
+ exports.generateCodeUsingRepoAgent = generateCodeUsingRepoAgent;
149
+ async function repoEditAgent({ trace, task, }) {
150
+ const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
151
+ void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
152
+ const { prompt: repoFiles } = await (0, context_1.generateTxtForRepository)();
153
+ const repoAgentOutput = await generateCodeUsingRepoAgent({
154
+ task,
155
+ trace,
156
+ repoFiles,
157
+ });
158
+ const updates = repoAgentOutput.fileChanges;
145
159
  const fileUpdateResponses = await (0, utils_1.applyFileChanges)({
146
- validateTypes: false,
147
160
  trace,
148
161
  testCase: {},
149
162
  fileChanges: updates.filter((f) => f.filePath && fsSync.existsSync(f.filePath)),
@@ -152,20 +165,25 @@ Task: ${task}
152
165
  if (errorResponses.length > 0) {
153
166
  const updatedFileChanges = await (0, test_update_feedback_1.applyTestUpdateFeedbacks)({
154
167
  trace,
155
- oldPrompt: prompt,
156
- agentResponse: updatedUsageExampleMessage?.content,
168
+ oldPrompt: repoAgentOutput.prompt,
169
+ agentResponse: repoAgentOutput.agentResponse,
157
170
  feedbacks: errorResponses.map((e) => ({
158
171
  filePath: e?.filePath,
159
172
  errorMessage: e?.errorMessage,
160
173
  })),
161
174
  });
162
175
  await (0, utils_1.applyFileChanges)({
163
- validateTypes: false,
164
176
  trace,
165
177
  testCase: {},
166
178
  fileChanges: updatedFileChanges,
167
179
  });
168
180
  }
181
+ await (0, utils_1.validateTypesAndFormatCode)({
182
+ validateTypes: false,
183
+ trace,
184
+ testCase: {},
185
+ fileChanges: updates.filter((f) => f.filePath && fsSync.existsSync(f.filePath)),
186
+ });
169
187
  const newFileUpdates = updates.filter((f) => !fsSync.existsSync(f.filePath));
170
188
  await Promise.all(newFileUpdates.map((f) => {
171
189
  return (async () => {
@@ -182,5 +200,6 @@ Task: ${task}
182
200
  })();
183
201
  }));
184
202
  await testgenUpdatesReporter.sendMessage(`Successfully generated code for the given task. \n View [trace](${trace?.getTraceUrl()})`);
203
+ return repoAgentOutput;
185
204
  }
186
205
  exports.repoEditAgent = repoEditAgent;
package/dist/agent/codegen/run.d.ts CHANGED
@@ -1,12 +1,4 @@
1
1
  import { TraceClient } from "@empiricalrun/llm";
2
2
  import { TestCase, TestGenConfigOptions } from "../../types";
3
- export declare function getAddScenarioCompletion({ testCase, testFiles, pageFiles, testFilePath, trace, options, }: {
4
- testCase: TestCase;
5
- testFiles: string;
6
- pageFiles: string;
7
- testFilePath: string;
8
- trace?: TraceClient;
9
- options?: TestGenConfigOptions;
10
- }): Promise<string>;
11
- export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[]>;
3
+ export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[] | void>;
12
4
  //# sourceMappingURL=run.d.ts.map
package/dist/agent/codegen/run.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkC,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAkBhF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,mBA4BA;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgFrB"}
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAsC5B"}
package/dist/agent/codegen/run.js CHANGED
@@ -3,44 +3,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.generateTest = exports.getAddScenarioCompletion = void 0;
6
+ exports.generateTest = void 0;
7
7
  const llm_1 = require("@empiricalrun/llm");
8
8
  const fs_extra_1 = __importDefault(require("fs-extra"));
9
9
  const logger_1 = require("../../bin/logger");
10
10
  const context_1 = require("../../bin/utils/context");
11
11
  const web_1 = require("../../bin/utils/platform/web");
12
- const constants_1 = require("../../constants");
13
- const fix_ts_errors_1 = require("./fix-ts-errors");
12
+ const repo_edit_1 = require("./repo-edit");
14
13
  const update_flow_1 = require("./update-flow");
15
- async function getAddScenarioCompletion({ testCase, testFiles, pageFiles, testFilePath, trace, options, }) {
16
- const promptSpan = trace?.span({
17
- name: "add-scenario-prompt",
18
- });
19
- const instruction = await (0, llm_1.getPrompt)("add-scenario", {
20
- testFiles: testFiles,
21
- pageFiles: pageFiles,
22
- scenarioName: testCase.name,
23
- scenarioSteps: testCase.steps.join("\n"),
24
- scenarioFile: testFilePath,
25
- });
26
- promptSpan?.end({ output: { instruction } });
27
- const llm = new llm_1.LLM({
28
- trace,
29
- provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
30
- defaultModel: options?.model || constants_1.DEFAULT_MODEL,
31
- providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
32
- });
33
- const firstShotMessage = await llm.createChatCompletion({
34
- messages: instruction,
35
- modelParameters: {
36
- ...constants_1.DEFAULT_MODEL_PARAMETERS,
37
- ...options?.modelParameters,
38
- },
39
- });
40
- let response = firstShotMessage?.content || "";
41
- return response;
42
- }
43
- exports.getAddScenarioCompletion = getAddScenarioCompletion;
44
14
  async function generateTest(testCase, file, options, trace) {
45
15
  const logger = new logger_1.CustomLogger();
46
16
  if (!fs_extra_1.default.existsSync(file)) {
@@ -48,7 +18,7 @@ async function generateTest(testCase, file, options, trace) {
48
18
  fs_extra_1.default.createFileSync(file);
49
19
  }
50
20
  const context = await (0, context_1.contextForGeneration)(file);
51
- const { codePrompt, pomPrompt, nonSpecFilePrompt, testFileContent } = context;
21
+ const { codePrompt, pomPrompt, testFileContent } = context;
52
22
  const { testBlock } = (0, web_1.getTypescriptTestBlock)({
53
23
  scenarioName: testCase?.name,
54
24
  content: testFileContent,
@@ -58,8 +28,6 @@ async function generateTest(testCase, file, options, trace) {
58
28
  if (isUpdate) {
59
29
  return await (0, update_flow_1.updateTest)(testCase, file, options, true, true, trace);
60
30
  }
61
- const generatedTestCases = [];
62
- logger.logEmptyLine();
63
31
  const createTestSpan = trace?.span({
64
32
  name: "create-test",
65
33
  input: {
@@ -69,47 +37,15 @@ async function generateTest(testCase, file, options, trace) {
69
37
  testFilePath: file,
70
38
  },
71
39
  });
72
- const response = await getAddScenarioCompletion({
73
- testCase,
74
- testFiles: codePrompt,
75
- pageFiles: pomPrompt,
76
- testFilePath: file,
77
- trace: createTestSpan,
78
- options,
79
- });
80
- logger.success("Test generated successfully!");
81
- const readWriteFileSpan = trace?.span({ name: "write-to-file" });
82
- let contents = fs_extra_1.default.readFileSync(file, "utf-8");
83
- const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response, testCase?.name);
84
- let updatedContent = prependContent +
85
- (0, web_1.injectCodeSnippetBySuiteChain)({
86
- testFileContent: contents,
87
- suites: testCase?.suites,
88
- codeSnippet: `\n\n${strippedContent}`,
89
- });
90
- await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
91
- readWriteFileSpan?.end({ output: { updatedContent } });
92
- logger.log("Linting generated code...");
93
- createTestSpan?.event({ name: "lint-file" });
94
- await (0, web_1.lintErrors)(file);
95
- await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
96
- trace,
97
- logger: new logger_1.CustomLogger({ useReporter: false }),
98
- file,
99
- pomCode: pomPrompt,
100
- nonSpecFileCode: nonSpecFilePrompt,
101
- testCase: testCase,
102
- options,
103
- });
104
- createTestSpan?.event({ name: "format-file" });
105
- await (0, web_1.formatCode)(file);
106
- logger.success("File formatted successfully!");
107
- if (trace) {
108
- logger.log(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
109
- }
110
- generatedTestCases.push(testCase);
111
- createTestSpan?.end({ output: { response } });
40
+ const task = `
41
+ Create a new test at the given test file path and perform the relevant changes required:\n
42
+ Scenario name: ${testCase.name}
43
+ Test file path: ${file}
44
+ Test suite: ${testCase.suites?.join("->") || "-"}
45
+ Scenario: ${testCase.steps.join("\n")}
46
+ `;
47
+ const repoAgentResponse = await (0, repo_edit_1.repoEditAgent)({ trace, task });
48
+ createTestSpan?.end({ output: repoAgentResponse.fileChanges });
112
49
  await (0, llm_1.flushAllTraces)();
113
- return generatedTestCases;
114
50
  }
115
51
  exports.generateTest = generateTest;
package/dist/agent/codegen/update-flow.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAYxE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,OAAO,EAIL,eAAe,EAChB,MAAM,SAAS,CAAC;AAEjB,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CAmDD;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CAqF5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBAuGA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA4C7B"}
1
+ {"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAYxE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,OAAO,EAIL,eAAe,EAEhB,MAAM,SAAS,CAAC;AAEjB,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CAmDD;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA+F5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBAuGA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAqD7B"}
package/dist/agent/codegen/update-flow.js CHANGED
@@ -100,7 +100,6 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
100
100
  });
101
101
  console.log(`fileChanges ${testCase.steps.join("")}`, fileChanges);
102
102
  const fileUpdateResponses = await (0, utils_1.applyFileChanges)({
103
- validateTypes: validate,
104
103
  trace: updateTestSpan,
105
104
  testCase,
106
105
  fileChanges,
@@ -121,7 +120,6 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
121
120
  })),
122
121
  });
123
122
  await (0, utils_1.applyFileChanges)({
124
- validateTypes: validate,
125
123
  trace: updateTestSpan,
126
124
  testCase,
127
125
  fileChanges: filteredFileChanges,
@@ -131,6 +129,16 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
131
129
  nonSpecFilePrompt: nonSpecFilePrompt,
132
130
  });
133
131
  }
132
+ await (0, utils_1.validateTypesAndFormatCode)({
133
+ validateTypes: validate,
134
+ trace: updateTestSpan,
135
+ testCase,
136
+ fileChanges,
137
+ logger,
138
+ testGenOptions: options,
139
+ pomPrompt: pomPrompt,
140
+ nonSpecFilePrompt: nonSpecFilePrompt,
141
+ });
134
142
  logger.success("Test generated successfully!");
135
143
  if (trace) {
136
144
  logger.log(`View [trace](${trace.getTraceUrl()})`);
@@ -268,6 +276,15 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
268
276
  testGenOptions: options,
269
277
  pomPrompt: pomPrompt,
270
278
  nonSpecFilePrompt: nonSpecFilePrompt,
279
+ });
280
+ await (0, utils_1.validateTypesAndFormatCode)({
281
+ trace,
282
+ testCase,
283
+ fileChanges,
284
+ logger,
285
+ testGenOptions: options,
286
+ pomPrompt: pomPrompt,
287
+ nonSpecFilePrompt: nonSpecFilePrompt,
271
288
  validateTypes,
272
289
  });
273
290
  if (trace) {
package/dist/agent/codegen/utils.d.ts CHANGED
@@ -51,8 +51,7 @@ export declare function extractTestStepsSuggestions(input: string): {
51
51
  export type UpdatedTestCase = TestCase & {
52
52
  updatedFiles: string[];
53
53
  };
54
- export declare function applyFileChanges({ validateTypes, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }: {
55
- validateTypes?: boolean;
54
+ export declare function applyFileChanges({ trace, testCase, fileChanges, logger, }: {
56
55
  trace?: TraceClient;
57
56
  testCase: TestCase;
58
57
  fileChanges: {
@@ -65,9 +64,24 @@ export declare function applyFileChanges({ validateTypes, trace, testCase, fileC
65
64
  testGenOptions?: TestGenConfigOptions;
66
65
  pomPrompt?: string;
67
66
  nonSpecFilePrompt?: string;
68
- }): Promise<({
67
+ }): Promise<{
69
68
  error: boolean;
70
69
  errorMessage: string;
71
70
  filePath: string;
72
- } | undefined)[]>;
71
+ }[]>;
72
+ export declare function validateTypesAndFormatCode({ validateTypes, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }: {
73
+ validateTypes?: boolean;
74
+ trace?: TraceClient;
75
+ testCase: TestCase;
76
+ fileChanges: {
77
+ filePath: string | undefined;
78
+ oldCode: string | undefined;
79
+ newCode: string | undefined;
80
+ reason: string | undefined;
81
+ }[];
82
+ logger?: CustomLogger;
83
+ testGenOptions?: TestGenConfigOptions;
84
+ pomPrompt?: string;
85
+ nonSpecFilePrompt?: string;
86
+ }): Promise<void>;
73
87
  //# sourceMappingURL=utils.d.ts.map
package/dist/agent/codegen/utils.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,EACrC,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CACT,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAAC,EAAE,CAC3E,CAmJA"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAwIxE;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA6BA"}
package/dist/agent/codegen/utils.js CHANGED
@@ -3,7 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.applyFileChanges = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractTestUpdates = void 0;
6
+ exports.validateTypesAndFormatCode = exports.applyFileChanges = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractTestUpdates = void 0;
7
+ const llm_1 = require("@empiricalrun/llm");
7
8
  const fs_extra_1 = __importDefault(require("fs-extra"));
8
9
  const ts_morph_1 = require("ts-morph");
9
10
  const web_1 = require("../../bin/utils/platform/web");
@@ -88,10 +89,11 @@ function extractTestStepsSuggestions(input) {
88
89
  return result.filter((r) => !!r.filePath && !!r.usageExample);
89
90
  }
90
91
  exports.extractTestStepsSuggestions = extractTestStepsSuggestions;
91
- async function applyFileChanges({ validateTypes = true, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }) {
92
- return await Promise.all(fileChanges.map(async (fileChange) => {
92
+ async function applyFileChanges({ trace, testCase, fileChanges, logger, }) {
93
+ const results = [];
94
+ for (const fileChange of fileChanges) {
93
95
  if (!fileChange.filePath) {
94
- return;
96
+ continue;
95
97
  }
96
98
  try {
97
99
  const hasTestCaseAsUpdateContext = !!testCase?.name;
@@ -103,7 +105,7 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
103
105
  const block = (0, web_1.getTypescriptTestBlock)({
104
106
  scenarioName: testCase?.name,
105
107
  content: fileChange.newCode || "",
106
- suites: [], // suites should be empty here since we ask LLM to send immediate parent AST node for the code update. so there won't be any nesting here, just the test block
108
+ suites: [], // // suites should be empty here since we ask LLM to send immediate parent AST node for the code update. so there won't be any nesting here, just the test block
107
109
  });
108
110
  testBlockUpdate = block.testBlock;
109
111
  applyFileChangesSpan?.end({ output: { testBlock: testBlockUpdate } });
@@ -132,8 +134,8 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
132
134
  const sourceFile = project.createSourceFile("updated-code.ts", fileChange.newCode);
133
135
  const functions = sourceFile.getFunctions();
134
136
  const checkForMethodSrc = project.createSourceFile("check-method.ts", `class A {
135
- ${fileChange.newCode}
136
- }`);
137
+ ${fileChange.newCode}
138
+ }`);
137
139
  const methods = checkForMethodSrc.getDescendantsOfKind(ts_morph_1.SyntaxKind.MethodDeclaration);
138
140
  const originalSource = project.createSourceFile("current-code.ts", contents);
139
141
  // if there is a single function update in the file
@@ -160,9 +162,6 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
160
162
  contents = originalSource.getFullText();
161
163
  }
162
164
  else {
163
- // since we dont know what is getting updated,
164
- // we believe that the patch is correct and contains few before and after lines
165
- // to make the change unique for search & replace
166
165
  // since we dont know what is getting updated,
167
166
  // we believe that the patch is correct and contains few before and after lines
168
167
  // to make the change unique for search & replace
@@ -171,36 +170,22 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
171
170
  }
172
171
  else {
173
172
  logger?.error(`Unable to find the code to update in ${fileChange.filePath}`);
174
- return {
173
+ results.push({
175
174
  filePath: fileChange.filePath,
176
175
  error: true,
177
176
  errorMessage: `The content of "old_code_block" corresponding to file path "${fileChange.filePath}" did not match the current content of the file "${fileChange.filePath}"`,
178
- };
177
+ });
178
+ continue;
179
179
  }
180
180
  }
181
181
  await fs_extra_1.default.writeFile(fileChange.filePath, contents, "utf-8");
182
182
  readWriteFileSpan?.end({ output: { contents } });
183
183
  }
184
- // format and validate file change
185
- if (validateTypes) {
186
- await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
187
- trace,
188
- logger,
189
- file: fileChange.filePath,
190
- pomCode: pomPrompt,
191
- nonSpecFileCode: nonSpecFilePrompt,
192
- testCase: testCase,
193
- options: testGenOptions,
194
- });
195
- }
196
- trace?.event({ name: "format-file" });
197
- await (0, web_1.formatCode)(fileChange.filePath, trace);
198
- logger?.success(`${fileChange.filePath} file formatted successfully!`);
199
- return {
184
+ results.push({
200
185
  filePath: fileChange.filePath,
201
186
  error: false,
202
187
  errorMessage: "",
203
- };
188
+ });
204
189
  }
205
190
  catch (e) {
206
191
  trace?.event({
@@ -211,8 +196,36 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
211
196
  },
212
197
  });
213
198
  console.error(`Error while applying changes to file ${fileChange.filePath}`, e);
214
- return;
215
199
  }
216
- }));
200
+ }
201
+ return results;
217
202
  }
218
203
  exports.applyFileChanges = applyFileChanges;
204
+ async function validateTypesAndFormatCode({ validateTypes = true, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }) {
205
+ for (let fileChange of fileChanges) {
206
+ if (!fileChange.filePath) {
207
+ continue;
208
+ }
209
+ try {
210
+ if (validateTypes) {
211
+ await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
212
+ trace,
213
+ logger,
214
+ file: fileChange.filePath,
215
+ pomCode: pomPrompt ?? "",
216
+ nonSpecFileCode: nonSpecFilePrompt ?? "",
217
+ testCase: testCase,
218
+ options: testGenOptions,
219
+ });
220
+ }
221
+ trace?.event({ name: "format-file" });
222
+ await (0, web_1.formatCode)(fileChange.filePath, trace);
223
+ logger?.success(`${fileChange.filePath} file formatted successfully!`);
224
+ }
225
+ catch (e) {
226
+ console.error(`Error while formatting the file ${fileChange.filePath}`, e);
227
+ }
228
+ }
229
+ await (0, llm_1.flushAllTraces)();
230
+ }
231
+ exports.validateTypesAndFormatCode = validateTypesAndFormatCode;
package/dist/bin/utils/platform/web/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAOhD,OAAO,EAGL,IAAI,EAEJ,UAAU,EAEX,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,eAAO,MAAM,gCAAgC,eAC/B,UAAU,KACrB,MAgBF,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB,CA2CA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,WAAW,iBASrE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA4CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC;AAEF,eAAO,MAAM,iCAAiC,cACjC,MAAM,EAAE,gBACL,MAAM,sBAyBrB,CAAC;AAEF,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,QAAQ,EACR,MAAM,GACP,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,iBA8CA;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,iBAsBzB;AAED,wBAAgB,aAAa,CAAC,EAC5B,QAAQ,EACR,QAAQ,GACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;CACpB,WAYA;AAED,wBAAgB,mBAAmB,CAAC,EAClC,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,UAOA;AAED,wBAAgB,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA4B5E;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAQnD"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAOhD,OAAO,EAGL,IAAI,EAEJ,UAAU,EAEX,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,eAAO,MAAM,gCAAgC,eAC/B,UAAU,KACrB,MAgBF,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB,CA2CA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,WAAW,iBAgBrE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA4CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC;AAEF,eAAO,MAAM,iCAAiC,cACjC,MAAM,EAAE,gBACL,MAAM,sBAyBrB,CAAC;AAEF,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,QAAQ,EACR,MAAM,GACP,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,iBA8CA;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,iBAsBzB;AAED,wBAAgB,aAAa,CAAC,EAC5B,QAAQ,EACR,QAAQ,GACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;CACpB,WAYA;AAED,wBAAgB,mBAAmB,CAAC,EAClC,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,UAOA;AAED,wBAAgB,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA4B5E;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAQnD"}
package/dist/bin/utils/platform/web/index.js CHANGED
@@ -240,6 +240,13 @@ async function lintErrors(filePath) {
240
240
  exports.lintErrors = lintErrors;
241
241
  async function formatCode(filePath, trace) {
242
242
  const fileContent = fs_extra_1.default.readFileSync(filePath, "utf8");
243
+ if (!fileContent) {
244
+ trace?.span({
245
+ name: "prettier-format-output",
246
+ output: `${filePath} file is empty`,
247
+ });
248
+ return;
249
+ }
243
250
  const prettierConfig = {};
244
251
  const formattedContent = await prettier_1.default.format(fileContent, {
245
252
  ...prettierConfig,
package/dist/evals/add-scenario-agent.evals.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,QAAA,MAAM,4BAA4B,EAAE,UAkBnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
1
+ {"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AA+BpC,QAAA,MAAM,4BAA4B,EAAE,UA6BnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
package/dist/evals/add-scenario-agent.evals.js CHANGED
@@ -1,23 +1,42 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- const run_1 = require("../agent/codegen/run");
3
+ const repo_edit_1 = require("../agent/codegen/repo-edit");
4
+ function evaluateEqualityScore({ currentOutput, expectedOutput, }) {
5
+ for (const expectedFile of expectedOutput) {
6
+ if (!currentOutput.find((output) => {
7
+ return output.filePath === expectedFile.filePath;
8
+ })) {
9
+ return 0;
10
+ }
11
+ }
12
+ return 1;
13
+ }
4
14
  const addScenarioCodeAgentEvaluate = async ({ item, trace }) => {
5
- const { testCase, testFiles, pageFiles, testFilePath } = item.input;
6
- const response = await (0, run_1.getAddScenarioCompletion)({
7
- testCase,
8
- testFiles,
9
- pageFiles,
10
- testFilePath,
15
+ const { testCase, testFilePath, pageFiles, testFiles } = item.input;
16
+ const task = `
17
+ Create a new test at the given test file path and perform the relevant changes required:\n
18
+ Scenario name: ${testCase.name}
19
+ Test file path: ${testFilePath}
20
+ Test suite: ${testCase.suites?.join("->") || "-"}
21
+ Scenario: ${testCase.steps.join("\n")}
22
+ `;
23
+ const repoFiles = pageFiles + testFiles;
24
+ const repoAgentOutput = await (0, repo_edit_1.generateCodeUsingRepoAgent)({
25
+ task,
11
26
  trace,
27
+ repoFiles,
12
28
  });
13
29
  return {
14
30
  scores: [
15
31
  {
16
32
  name: "equality",
17
- value: item.expectedOutput === response ? 1 : 0,
33
+ value: evaluateEqualityScore({
34
+ currentOutput: repoAgentOutput.fileChanges,
35
+ expectedOutput: item.expectedOutput,
36
+ }),
18
37
  },
19
38
  ],
20
- output: response,
39
+ output: repoAgentOutput.fileChanges,
21
40
  };
22
41
  };
23
42
  exports.default = addScenarioCodeAgentEvaluate;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@empiricalrun/test-gen",
3
- "version": "0.39.0",
3
+ "version": "0.40.1",
4
4
  "publishConfig": {
5
5
  "registry": "https://registry.npmjs.org/",
6
6
  "access": "public"