@empiricalrun/test-gen 0.32.1 → 0.32.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/dist/agent/browsing/utils.d.ts.map +1 -1
  3. package/dist/agent/browsing/utils.js +21 -2
  4. package/dist/agent/codegen/run.d.ts.map +1 -1
  5. package/dist/agent/codegen/run.js +1 -1
  6. package/dist/agent/master/run.d.ts.map +1 -1
  7. package/dist/agent/master/run.js +3 -1
  8. package/dist/bin/index.js +2 -2
  9. package/package.json +2 -2
package/CHANGELOG.md CHANGED
@@ -1,5 +1,27 @@
1
1
  # @empiricalrun/test-gen
2
2
 
3
+ ## 0.32.4
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies [e20abfb]
8
+ - @empiricalrun/llm@0.9.6
9
+
10
+ ## 0.32.3
11
+
12
+ ### Patch Changes
13
+
14
+ - 0435eed: fix: geenration error incorrectly shown
15
+
16
+ ## 0.32.2
17
+
18
+ ### Patch Changes
19
+
20
+ - d2bc4c5: fix: add logs and traces
21
+ - Updated dependencies [0eb7700]
22
+ - Updated dependencies [55f21cf]
23
+ - @empiricalrun/llm@0.9.5
24
+
3
25
  ## 0.32.1
4
26
 
5
27
  ### Patch Changes
package/dist/agent/browsing/utils.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAcvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAK5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAgED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CA2CjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA6BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAcvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAK5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA6BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}
package/dist/agent/browsing/utils.js CHANGED
@@ -30,7 +30,7 @@ exports.prepareBrowsingAgentTask = prepareBrowsingAgentTask;
30
30
  * Function to prepare test file for update scenarios for master agent to run
31
31
  * @param {TestGenConfig} genConfig
32
32
  */
33
- async function prepareFileForUpdateScenario(genConfig) {
33
+ async function prepareFileForUpdateScenario(genConfig, trace) {
34
34
  const { specPath, testCase } = genConfig;
35
35
  const { name, suites } = testCase;
36
36
  await (0, web_1.addUserContextFixture)({
@@ -42,6 +42,7 @@ async function prepareFileForUpdateScenario(genConfig) {
42
42
  // TODO: reduce the payload for this LLM call. Only provide test file and page files which are used in the test block
43
43
  // this will help with faster response
44
44
  const [suggestion] = await (0, update_flow_1.appendCreateTestBlock)({
45
+ trace,
45
46
  testCase,
46
47
  file: specPath,
47
48
  validateTypes: false,
@@ -50,9 +51,17 @@ async function prepareFileForUpdateScenario(genConfig) {
50
51
  const createTestFilePath = suggestion?.updatedFiles[0] || "";
51
52
  console.log("appending to existing test block");
52
53
  console.log("updated test file path", createTestFilePath);
54
+ const updateFileSpan = trace?.span({
55
+ name: "update-file-span",
56
+ input: {
57
+ createTestFilePath,
58
+ },
59
+ });
53
60
  await fs_extra_1.default.writeFile(createTestFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(createTestFilePath, "utf-8"), ["createTest"], "@empiricalrun/test-gen"));
61
+ updateFileSpan?.end();
54
62
  const { codePrompt, pomPrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
55
63
  await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
64
+ trace,
56
65
  file: createTestFilePath,
57
66
  testCode: codePrompt,
58
67
  pomCode: pomPrompt,
@@ -79,14 +88,23 @@ async function prepareFileForUpdateScenario(genConfig) {
79
88
  * @return {*} {Promise<string>}
80
89
  */
81
90
  async function prepareFileForMasterAgent(genConfig) {
91
+ const trace = llm_1.langfuseInstance.trace({
92
+ name: "prepare-file-for-master-agent",
93
+ });
94
+ new logger_1.CustomLogger({ useReporter: true }).log(`Preparing file for master agent. [view trace](${trace.getTraceUrl()})`);
82
95
  const { specPath, testCase } = genConfig;
83
96
  const { name, suites } = testCase;
84
97
  // check if the spec file exists
85
98
  // if no then create a new file with test and expect imports
86
99
  if (!fs_extra_1.default.existsSync(specPath)) {
100
+ const fileCreateSpan = trace.span({
101
+ name: "create-file",
102
+ input: { specPath },
103
+ });
87
104
  await fs_extra_1.default.createFile(specPath);
88
105
  const fileContentWithImports = (0, web_1.addNewImport)("", ["test", "expect"], (0, web_1.getFixtureImportPath)(specPath));
89
106
  await fs_extra_1.default.writeFile(specPath, fileContentWithImports, "utf-8");
107
+ fileCreateSpan.end({ output: { specPath, fileContentWithImports } });
90
108
  }
91
109
  const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
92
110
  const { testBlock } = (0, web_1.getTypescriptTestBlock)({
@@ -96,6 +114,7 @@ async function prepareFileForMasterAgent(genConfig) {
96
114
  });
97
115
  if (!testBlock) {
98
116
  const newTestBlock = await (0, create_test_block_1.createEmptyTestCaseBlock)({
117
+ trace,
99
118
  testCase: genConfig.testCase,
100
119
  file: specPath,
101
120
  options: genConfig.options,
@@ -108,7 +127,7 @@ async function prepareFileForMasterAgent(genConfig) {
108
127
  });
109
128
  await fs_extra_1.default.writeFile(specPath, updatedContent, "utf-8");
110
129
  }
111
- await prepareFileForUpdateScenario(genConfig);
130
+ await prepareFileForUpdateScenario(genConfig, trace);
112
131
  return specPath;
113
132
  }
114
133
  exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
package/dist/agent/codegen/run.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAuGrB"}
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyGrB"}
package/dist/agent/codegen/run.js CHANGED
@@ -101,7 +101,7 @@ async function generateTest(testCase, file, options) {
101
101
  trace.event({ name: "format-file" });
102
102
  await (0, web_1.formatCode)(file);
103
103
  logger.success("File formatted successfully!");
104
- logger.log(`Trace: ${trace.getTraceUrl()}`);
104
+ logger.log(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
105
105
  generatedTestCases.push(testCase);
106
106
  trace.update({ input: { testCase }, output: { response } });
107
107
  await (0, llm_1.flushAllTraces)();
package/dist/agent/master/run.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAOrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,aAAa,GACd,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;CACxB,2FAwDA;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GAmLA"}
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGlC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAOrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,aAAa,GACd,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;CACxB,2FAwDA;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GAsLA"}
package/dist/agent/master/run.js CHANGED
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.createTestUsingMasterAgent = exports.getNextAction = void 0;
4
4
  const llm_1 = require("@empiricalrun/llm");
5
5
  const vision_1 = require("@empiricalrun/llm/vision");
6
+ const test_1 = require("playwright/test");
6
7
  const actions_1 = require("../../actions");
7
8
  const next_task_1 = require("../../actions/next-task");
8
9
  const skill_1 = require("../../actions/skill");
@@ -82,6 +83,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
82
83
  options.metadata?.environment,
83
84
  ].filter((s) => !!s),
84
85
  });
86
+ void testgenUpdatesReporter.sendMessage(`Starting master agent. [view trace](${trace.getTraceUrl()})`);
85
87
  const llm = new llm_1.LLM({
86
88
  trace,
87
89
  provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
@@ -218,7 +220,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
218
220
  output: { action: output.action, reason: output.reason },
219
221
  });
220
222
  }
221
- await page.close();
223
+ test_1.test.skip();
222
224
  const { code, importPaths } = actions.generateCode();
223
225
  trace.update({ input: { task }, output: { code } });
224
226
  logger.success("Successfully generated code for the given task");
package/dist/bin/index.js CHANGED
@@ -27,7 +27,7 @@ async function runAgent(testGenConfig) {
27
27
  const { specPath, testCase } = testGenConfig;
28
28
  if (testGenConfig.options?.agent !== "code") {
29
29
  // this assumes we have only one scenario in test config
30
- logger.success(`Generating test using ${testGenConfig.options?.agent} agent`);
30
+ logger.success(`Generating test using ${testGenConfig.options?.agent} agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
31
31
  const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)(testGenConfig);
32
32
  await (0, run_1.generateTestsUsingMasterAgent)({
33
33
  testFilePath: specPath,
@@ -36,7 +36,7 @@ async function runAgent(testGenConfig) {
36
36
  });
37
37
  }
38
38
  else {
39
- logger.success("Generating test using coding agent");
39
+ logger.success(`Generating test using coding agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
40
40
  await (0, run_2.generateTest)(testCase, specPath, testGenConfig.options);
41
41
  }
42
42
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@empiricalrun/test-gen",
3
- "version": "0.32.1",
3
+ "version": "0.32.4",
4
4
  "publishConfig": {
5
5
  "registry": "https://registry.npmjs.org/",
6
6
  "access": "public"
@@ -44,8 +44,8 @@
44
44
  "ts-morph": "^23.0.0",
45
45
  "tsx": "^4.16.2",
46
46
  "typescript": "^5.3.3",
47
+ "@empiricalrun/llm": "^0.9.6",
47
48
  "@empiricalrun/r2-uploader": "^0.3.4",
48
- "@empiricalrun/llm": "^0.9.4",
49
49
  "@empiricalrun/reporter": "^0.20.7"
50
50
  },
51
51
  "devDependencies": {