@empiricalrun/test-gen 0.17.6 → 0.19.1

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @empiricalrun/test-gen
 
+## 0.19.1
+
+### Patch Changes
+
+- Updated dependencies [b7797f9]
+  - @empiricalrun/reporter@0.12.3
+
+## 0.19.0
+
+### Minor Changes
+
+- 2087461: feat: remove LLMTracing class and use native methods
+
+### Patch Changes
+
+- Updated dependencies [2087461]
+  - @empiricalrun/llm@0.4.0
+
+## 0.18.0
+
+### Minor Changes
+
+- 4390c31: fix: filter video based on test name
+
 ## 0.17.6
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAInD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAgG9B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAInD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAkG9B"}

package/dist/agent/browsing/index.js

@@ -10,9 +10,11 @@ const html_1 = require("../../utils/html");
 const utils_1 = require("./utils");
 async function browsingAgent(task, page, options) {
     const logger = new logger_1.CustomLogger();
-    const trace = new llm_1.LLMTracing({
+    const session = (0, session_1.getSessionDetails)();
+    const trace = llm_1.langfuseInstance.trace({
         name: "browsing-agent",
-        sessionDetails: (0, session_1.getSessionDetails)(),
+        id: session.id,
+        release: session.version,
         tags: [
             options.metadata?.projectName,
             options.metadata?.environment,
@@ -26,13 +28,13 @@ async function browsingAgent(task, page, options) {
     trace.update({ input: { task } });
     let lastActionExecTrace = "";
     while (!isTaskDone) {
-        const pageContentSpan = trace.startSpan("page-content");
+        const pageContentSpan = trace.span({ name: "page-content" });
         const pageContent = await page.content();
         pageContentSpan.end({ output: { pageContent } });
-        const sanitizationSpan = trace.startSpan("page-sanitization");
+        const sanitizationSpan = trace.span({ name: "page-sanitization" });
         const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
         sanitizationSpan.end({ output: { pageSnapshot } });
-        const promptSpan = trace.startSpan("page-prompt");
+        const promptSpan = trace.span({ name: "page-prompt" });
         // extract all successful actions
         const successfulActions = executedActions
             .filter((a) => !a.isError)
@@ -93,7 +95,7 @@ async function browsingAgent(task, page, options) {
     const code = actions.generateCode();
     trace.update({ input: { task }, output: { code } });
     logger.success("Successfully generated code for the given task");
-    logger.log(`Trace: ${trace.url}`);
+    logger.log(`Trace: ${trace.getTraceUrl()}`);
     return code;
 }
 exports.browsingAgent = browsingAgent;

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CA8IrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAmJrB"}

package/dist/agent/codegen/run.js

@@ -21,9 +21,11 @@ async function generateTest(testCase, file, options) {
     const { codePrompt, pomPrompt, testFileContent } = context;
     const generatedTestCases = [];
     logger.logEmptyLine();
-    const trace = new llm_1.LLMTracing({
+    const session = (0, session_1.getSessionDetails)();
+    const trace = llm_1.langfuseInstance.trace({
         name: "generate-test",
-        sessionDetails: (0, session_1.getSessionDetails)(),
+        id: session.id,
+        release: session.version,
         tags: [options.metadata.projectName, options.metadata.environment].filter((s) => !!s),
     });
     trace.event({
@@ -36,7 +38,9 @@ async function generateTest(testCase, file, options) {
     });
     trace.update({ input: { testCase } });
     const isUpdate = testFileContent.includes(`test("${testCase?.name}"`);
-    const promptSpan = trace.startSpan(isUpdate ? "update-scenario-prompt" : "add-scenario-prompt");
+    const promptSpan = trace.span({
+        name: isUpdate ? "update-scenario-prompt" : "add-scenario-prompt",
+    });
     const promptName = isUpdate ? "update-scenario" : "add-scenario";
     const instruction = await (0, llm_1.getPrompt)(promptName, {
         testFiles: codePrompt,
@@ -59,7 +63,7 @@ async function generateTest(testCase, file, options) {
     });
     let response = firstShotMessage?.content || "";
     logger.success("Test generated successfully!");
-    const readWriteFileSpan = trace.startSpan("write-to-file");
+    const readWriteFileSpan = trace.span({ name: "write-to-file" });
     let contents = fs_extra_1.default.readFileSync(file, "utf-8");
     const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response);
     let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
@@ -73,7 +77,7 @@ async function generateTest(testCase, file, options) {
     logger.log("Linting generated code...");
     trace.event({ name: "lint-file" });
     await (0, web_1.lintErrors)(file);
-    const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
+    const validateTypesSpan = trace.span({ name: "detect-type-errors-in-file" });
     logger.log("Validating types...");
     let errors = (0, web_1.validateTypescript)(file);
     validateTypesSpan.end({ output: { errors } });
@@ -90,7 +94,7 @@ async function generateTest(testCase, file, options) {
             logger.error([
                 `Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors.`,
                 `Run the test-gen command again, once errors are fixed.`,
-                `Trace: ${trace.url}`,
+                `Trace: ${trace.getTraceUrl()}`,
             ].join("\n"));
             break;
         }
@@ -98,7 +102,7 @@ async function generateTest(testCase, file, options) {
         logger.warn("Found few errors while validating types:");
         errors.forEach((e) => logger.warn(e));
         logger.log("Trying to fix above errors...");
-        const promptSpan = trace.startSpan("fix-type-errors-prompt");
+        const promptSpan = trace.span({ name: "fix-type-errors-prompt" });
         const instruction = await (0, llm_1.getPrompt)("fix-file-errors-ts", {
             testFiles: codePrompt || "",
             pageFiles: pomPrompt || "",
@@ -120,12 +124,14 @@ async function generateTest(testCase, file, options) {
             },
         });
         response = message?.content || "";
-        const readWriteFileSpan = trace.startSpan("write-to-file");
+        const readWriteFileSpan = trace.span({ name: "write-to-file" });
         await fs_extra_1.default.writeFile(file, response, "utf-8");
         readWriteFileSpan.end({ output: { response } });
         trace.event({ name: "lint-file" });
         await (0, web_1.lintErrors)(file);
-        const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
+        const validateTypesSpan = trace.span({
+            name: "detect-type-errors-in-file",
+        });
         errors = (0, web_1.validateTypescript)(file);
         validateTypesSpan.end({ output: { errors } });
         if (!errors.length) {
@@ -135,7 +141,7 @@ async function generateTest(testCase, file, options) {
     trace.event({ name: "format-file" });
     await (0, web_1.formatCode)(file);
     logger.success("File formatted successfully!");
-    logger.log(`Trace: ${trace.url}`);
+    logger.log(`Trace: ${trace.getTraceUrl()}`);
     generatedTestCases.push(testCase);
     trace.update({ input: { testCase }, output: { response } });
     return generatedTestCases;

package/dist/bin/index.js

@@ -29,6 +29,7 @@ async function runAgent(sourceFile, testGenConfig) {
         await (0, run_1.generateTestsUsingBrowsingAgent)(specPath);
         await (0, reporter_1.reportTestGenVideos)({
             projectRepoName: testGenConfig.options.metadata.projectRepoName,
+            testName: testCase.name,
         });
     }
     else {

package/dist/reporter/index.d.ts

@@ -10,8 +10,9 @@ export declare function getReporter(): Reporter | undefined;
  * }
  * @returns Promise<void> returns void
  */
-export declare function reportTestGenVideos({ projectRepoName, }: {
+export declare function reportTestGenVideos({ projectRepoName, testName, }: {
     projectRepoName: string;
+    testName: string;
 }): Promise<void>;
 export declare function setReporterConfig(config: any): void;
 //# sourceMappingURL=index.d.ts.map

package/dist/reporter/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAW5E,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CAAC,EACxC,eAAe,GAChB,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;CACzB,iBA4BA;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAGnD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAW5E,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CAAC,EACxC,eAAe,EACf,QAAQ,GACT,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBA6BA;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAGnD"}

package/dist/reporter/index.js

@@ -28,7 +28,7 @@ exports.getReporter = getReporter;
  * }
  * @returns Promise<void> returns void
  */
-async function reportTestGenVideos({ projectRepoName, }) {
+async function reportTestGenVideos({ projectRepoName, testName, }) {
     const logger = new logger_1.CustomLogger();
     try {
         if (!(0, uploader_1.checkIfResultsUploadAllowed)()) {
@@ -36,6 +36,7 @@ async function reportTestGenVideos({ projectRepoName, }) {
         }
         const { videoUrls } = await (0, uploader_1.uploadTestResultsUsingPrjtRepo)({
             projectRepoName,
+            testName,
         });
         const reporter = getReporter();
         const reporterMessage = `
@@ -44,7 +45,7 @@ async function reportTestGenVideos({ projectRepoName, }) {
   
   ${videoUrls
             .map((url) => `
-  <video src="${url}" autoplay="true" muted="true" controls loop playsinline></video>`)
+  <video src="${url}" autoplay="true" muted="true" controls playsinline></video>`)
             .join("\n")}
     `;
         await reporter?.report(new reporter_1.ProcessLogMessageBuilder({ message: reporterMessage }));

package/dist/uploader/index.d.ts

@@ -1,11 +1,16 @@
 /**
- * function to upload test results to r2 using the project repo name
- * this only uploads json summary of test results
- * @param { projectName: string } projectRepoName - name of the project repo
- * @returns urls of videos and summary json
+ * Function to upload test results to R2 using the project repo name and test name.
+ * This function uploads both the JSON summary of test results and associated video files.
+ * @param {Object} params - The parameters for the function.
+ * @param {string} params.projectRepoName - Name of the project repository.
+ * @param {string} params.testName - Name of the testcase which called this test-gen.
+ * @returns {Promise<Object>} An object containing arrays of video URLs and the summary JSON URL.
+ * @returns {string[]} returns.videoUrls - URLs of the uploaded video files.
+ * @returns {string} returns.summaryUrl - URL of the uploaded summary JSON file.
  */
-export declare function uploadTestResultsUsingPrjtRepo({ projectRepoName, }: {
+export declare function uploadTestResultsUsingPrjtRepo({ projectRepoName, testName, }: {
     projectRepoName: string;
+    testName: string;
 }): Promise<{
     videoUrls: string[];
     summaryUrl: string;

package/dist/uploader/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/uploader/index.ts"],"names":[],"mappings":"AAiBA;;;;;GAKG;AACH,wBAAsB,8BAA8B,CAAC,EACnD,eAAe,GAChB,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC;IACV,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAyBD;AAED,wBAAgB,2BAA2B,uBAQ1C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/uploader/index.ts"],"names":[],"mappings":"AAmBA;;;;;;;;;GASG;AACH,wBAAsB,8BAA8B,CAAC,EACnD,eAAe,EACf,QAAQ,GACT,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC;IACV,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAgDD;AAED,wBAAgB,2BAA2B,uBAQ1C"}

package/dist/uploader/index.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.checkIfResultsUploadAllowed = exports.uploadTestResultsUsingPrjtRepo = void 0;
+const reporter_1 = require("@empiricalrun/reporter");
 const path_1 = __importDefault(require("path"));
 const r2_1 = require("./r2");
 // json summary of test results
@@ -15,12 +16,16 @@ function getFullUploadPath(filePath, uploadDir) {
     return `${UPLOAD_DOMAIN}/${uploadDir}${relativeFilePath}`;
 }
 /**
- * function to upload test results to r2 using the project repo name
- * this only uploads json summary of test results
- * @param { projectName: string } projectRepoName - name of the project repo
- * @returns urls of videos and summary json
+ * Function to upload test results to R2 using the project repo name and test name.
+ * This function uploads both the JSON summary of test results and associated video files.
+ * @param {Object} params - The parameters for the function.
+ * @param {string} params.projectRepoName - Name of the project repository.
+ * @param {string} params.testName - Name of the testcase which called this test-gen.
+ * @returns {Promise<Object>} An object containing arrays of video URLs and the summary JSON URL.
+ * @returns {string[]} returns.videoUrls - URLs of the uploaded video files.
+ * @returns {string} returns.summaryUrl - URL of the uploaded summary JSON file.
  */
-async function uploadTestResultsUsingPrjtRepo({ projectRepoName, }) {
+async function uploadTestResultsUsingPrjtRepo({ projectRepoName, testName, }) {
     const uploadUniqueId = crypto.randomUUID();
     // project repo name is the github repo name
     // the folder names in r2 are the github repo name without the `-tests` suffix
@@ -30,12 +35,29 @@ async function uploadTestResultsUsingPrjtRepo({ projectRepoName, }) {
         destinationDir: uploadDir,
         uploadBucket: UPLOAD_BUCKET,
     });
-    const fileNames = Object.keys(files);
-    // TODO: parse the json summary and then detect video attachments
+    const fileNames = Object.keys(files); // fileNames are absolute paths of the input files
+    const defaultLocation = path_1.default.join(process.cwd(), "test-results", "summary.json");
+    const results = (0, reporter_1.parseJsonReport)(defaultLocation);
+    const flatTestsList = (0, reporter_1.getFlattenedTestList)(results.suites);
+    const testVideos = [];
+    for (const test of flatTestsList) {
+        if (test.title === testName) {
+            if (test.tests[0]) {
+                if (test.tests[0].results[0]) {
+                    // results array is basically made by retries
+                    for (const attachments of test.tests[0].results[0].attachments) {
+                        if (attachments.path) {
+                            testVideos.push(attachments.path);
+                        }
+                    }
+                }
+            }
+        }
+    }
     // current assumption
     // - test gen will only run on a single spec file
-    // - the video files will be of format <spec-file-name>.webm
-    const videoFiles = fileNames.filter((fileName) => fileName.endsWith(".webm"));
+    // - the video files are of the format - <some-directory>/video.webm
+    const videoFiles = fileNames.filter((fileName) => fileName.endsWith(".webm") && testVideos.includes(fileName));
     return {
         videoUrls: videoFiles.map((fileName) => getFullUploadPath(fileName, uploadDir)),
         summaryUrl: getFullUploadPath("/test-results/summary.json", uploadDir),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.17.6",
+  "version": "0.19.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -18,6 +18,7 @@
     "@actions/core": "^1.10.1",
     "@aws-sdk/client-s3": "^3.614.0",
     "@aws-sdk/s3-request-presigner": "^3.614.0",
+    "@playwright/test": "^1.44.1",
     "@types/sanitize-html": "^2.11.0",
     "commander": "^12.1.0",
     "detect-port": "^1.6.1",
@@ -39,8 +40,8 @@
     "slugify": "^1.6.6",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.3.0",
-    "@empiricalrun/reporter": "^0.12.2"
+    "@empiricalrun/llm": "^0.4.0",
+    "@empiricalrun/reporter": "^0.12.3"
   },
   "devDependencies": {
     "@types/detect-port": "^1.3.5",