@empiricalrun/test-gen 0.32.1 → 0.32.2

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @empiricalrun/test-gen
 
+## 0.32.2
+
+### Patch Changes
+
+- d2bc4c5: fix: add logs and traces
+- Updated dependencies [0eb7700]
+- Updated dependencies [55f21cf]
+  - @empiricalrun/llm@0.9.5
+
 ## 0.32.1
 
 ### Patch Changes

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAcvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAK5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAgED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CA2CjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA6BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAcvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAK5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA6BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}

package/dist/agent/browsing/utils.js

@@ -30,7 +30,7 @@ exports.prepareBrowsingAgentTask = prepareBrowsingAgentTask;
  * Function to prepare test file for update scenarios for master agent to run
  * @param {TestGenConfig} genConfig
  */
-async function prepareFileForUpdateScenario(genConfig) {
+async function prepareFileForUpdateScenario(genConfig, trace) {
     const { specPath, testCase } = genConfig;
     const { name, suites } = testCase;
     await (0, web_1.addUserContextFixture)({
@@ -42,6 +42,7 @@ async function prepareFileForUpdateScenario(genConfig) {
     // TODO: reduce the payload for this LLM call. Only provide test file and page files which are used in the test block
     // this will help with faster response
     const [suggestion] = await (0, update_flow_1.appendCreateTestBlock)({
+        trace,
         testCase,
         file: specPath,
         validateTypes: false,
@@ -50,9 +51,17 @@ async function prepareFileForUpdateScenario(genConfig) {
     const createTestFilePath = suggestion?.updatedFiles[0] || "";
     console.log("appending to existing test block");
     console.log("updated test file path", createTestFilePath);
+    const updateFileSpan = trace?.span({
+        name: "update-file-span",
+        input: {
+            createTestFilePath,
+        },
+    });
     await fs_extra_1.default.writeFile(createTestFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(createTestFilePath, "utf-8"), ["createTest"], "@empiricalrun/test-gen"));
+    updateFileSpan?.end();
     const { codePrompt, pomPrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
     await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
+        trace,
         file: createTestFilePath,
         testCode: codePrompt,
         pomCode: pomPrompt,
@@ -79,14 +88,23 @@ async function prepareFileForUpdateScenario(genConfig) {
  * @return {*}  {Promise<string>}
  */
 async function prepareFileForMasterAgent(genConfig) {
+    const trace = llm_1.langfuseInstance.trace({
+        name: "prepare-file-for-master-agent",
+    });
+    new logger_1.CustomLogger({ useReporter: true }).log(`Preparing file for master agent. [view trace](${trace.getTraceUrl()})`);
     const { specPath, testCase } = genConfig;
     const { name, suites } = testCase;
     // check if the spec file exists
     // if no then create a new file with test and expect imports
     if (!fs_extra_1.default.existsSync(specPath)) {
+        const fileCreateSpan = trace.span({
+            name: "create-file",
+            input: { specPath },
+        });
         await fs_extra_1.default.createFile(specPath);
         const fileContentWithImports = (0, web_1.addNewImport)("", ["test", "expect"], (0, web_1.getFixtureImportPath)(specPath));
         await fs_extra_1.default.writeFile(specPath, fileContentWithImports, "utf-8");
+        fileCreateSpan.end({ output: { specPath, fileContentWithImports } });
     }
     const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
     const { testBlock } = (0, web_1.getTypescriptTestBlock)({
@@ -96,6 +114,7 @@ async function prepareFileForMasterAgent(genConfig) {
     });
     if (!testBlock) {
         const newTestBlock = await (0, create_test_block_1.createEmptyTestCaseBlock)({
+            trace,
             testCase: genConfig.testCase,
             file: specPath,
             options: genConfig.options,
@@ -108,7 +127,7 @@ async function prepareFileForMasterAgent(genConfig) {
         });
         await fs_extra_1.default.writeFile(specPath, updatedContent, "utf-8");
     }
-    await prepareFileForUpdateScenario(genConfig);
+    await prepareFileForUpdateScenario(genConfig, trace);
     return specPath;
 }
 exports.prepareFileForMasterAgent = prepareFileForMasterAgent;

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAuGrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyGrB"}

package/dist/agent/codegen/run.js

@@ -101,7 +101,7 @@ async function generateTest(testCase, file, options) {
     trace.event({ name: "format-file" });
     await (0, web_1.formatCode)(file);
     logger.success("File formatted successfully!");
-    logger.log(`Trace: ${trace.getTraceUrl()}`);
+    logger.log(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
     generatedTestCases.push(testCase);
     trace.update({ input: { testCase }, output: { response } });
     await (0, llm_1.flushAllTraces)();

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAOrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,aAAa,GACd,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;CACxB,2FAwDA;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GAmLA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAOrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,aAAa,GACd,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;CACxB,2FAwDA;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GAsLA"}

package/dist/agent/master/run.js

@@ -82,6 +82,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
             options.metadata?.environment,
         ].filter((s) => !!s),
     });
+    void testgenUpdatesReporter.sendMessage(`Starting master agent. [view trace](${trace.getTraceUrl()})`);
     const llm = new llm_1.LLM({
         trace,
         provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,

package/dist/bin/index.js

@@ -27,7 +27,7 @@ async function runAgent(testGenConfig) {
     const { specPath, testCase } = testGenConfig;
     if (testGenConfig.options?.agent !== "code") {
         // this assumes we have only one scenario in test config
-        logger.success(`Generating test using ${testGenConfig.options?.agent} agent`);
+        logger.success(`Generating test using ${testGenConfig.options?.agent} agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
         const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)(testGenConfig);
         await (0, run_1.generateTestsUsingMasterAgent)({
             testFilePath: specPath,
@@ -36,7 +36,7 @@ async function runAgent(testGenConfig) {
         });
     }
     else {
-        logger.success("Generating test using coding agent");
+        logger.success(`Generating test using coding agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
         await (0, run_2.generateTest)(testCase, specPath, testGenConfig.options);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.32.1",
+  "version": "0.32.2",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -44,8 +44,8 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
+    "@empiricalrun/llm": "^0.9.5",
     "@empiricalrun/r2-uploader": "^0.3.4",
-    "@empiricalrun/llm": "^0.9.4",
     "@empiricalrun/reporter": "^0.20.7"
   },
   "devDependencies": {