@empiricalrun/test-gen 0.16.3 → 0.16.5

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.16.5
+
+### Patch Changes
+
+- Updated dependencies [0b41c5e]
+  - @empiricalrun/reporter@0.11.1
+
+## 0.16.4
+
+### Patch Changes
+
+- 4c32cf9: feat: add support for uploading test video of browsing agent generated tests
+
 ## 0.16.3
 
 ### Patch Changes

package/dist/bin/index.js

@@ -9,9 +9,9 @@ const dotenv_1 = __importDefault(require("dotenv"));
 const run_1 = require("../agent/browsing/run");
 const utils_1 = require("../agent/browsing/utils");
 const run_2 = require("../agent/codegen/run");
+const reporter_1 = require("../reporter");
+const ci_1 = require("../reporter/ci");
 const logger_1 = require("./logger");
-const reporter_1 = require("./reporter");
-const ci_1 = require("./reporter/ci");
 const utils_2 = require("./utils");
 dotenv_1.default.config({
     path: [".env.local", ".env"],
@@ -30,6 +30,9 @@ async function runAgent(sourceFile, isUpdate, testGenConfigs) {
             logger.success("Generating test using browsing agent");
             await (0, utils_1.prepareFileForBrowsingAgent)(testGenConfig);
             await (0, run_1.generateTestsUsingBrowsingAgent)(specPath);
+            await (0, reporter_1.reportTestGenVideos)({
+                projectRepoName: testGenConfig.options.metadata.projectRepoName,
+            });
             generatedTestScenarios.push(...testGenConfig.scenarios);
         }
         else {
@@ -49,6 +52,7 @@ async function runAgent(sourceFile, isUpdate, testGenConfigs) {
     const { sourceFile, testGenConfigs, isUpdate } = await (0, utils_2.parseCliArgs)();
     (0, reporter_1.setReporterConfig)(testGenConfigs[0]?.options?.metadata);
     const generated = await runAgent(sourceFile, isUpdate, testGenConfigs);
+    // TODO: move these reporters to a better lifecycle
     await (0, ci_1.reportOnCI)(generated);
     process.exit(0);
 })();

package/dist/bin/logger/index.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CustomLogger = void 0;
 const reporter_1 = require("@empiricalrun/reporter");
 const picocolors_1 = require("picocolors");
-const reporter_2 = require("../reporter");
+const reporter_2 = require("../../reporter");
 class CustomLogger {
     useReporter = false;
     constructor({ useReporter = true } = {}) {

package/dist/index.js

@@ -6,9 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTest = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const browsing_1 = require("./agent/browsing");
-const reporter_1 = require("./bin/reporter");
 const utils_1 = require("./bin/utils");
 const client_1 = __importDefault(require("./file/client"));
+const reporter_1 = require("./reporter");
 process.on("exit", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());

package/dist/{bin/reporter → reporter}/ci.d.ts

@@ -1,3 +1,3 @@
-import { Scenario } from "../../types";
+import { Scenario } from "../types";
 export declare function reportOnCI(scenarios: Scenario[]): Promise<Scenario[]>;
 //# sourceMappingURL=ci.d.ts.map

package/dist/reporter/ci.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC,wBAAsB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,uBAerD"}

package/dist/reporter/index.d.ts

@@ -0,0 +1,17 @@
+import { Reporter } from "@empiricalrun/reporter";
+export declare function getReporter(): Reporter | undefined;
+/**
+ * function will upload videos and json summary of test results to r2 and report them to reporter.
+ * method won't throw error if it fails to report
+ * @param {{
+ *   projectRepoName: string;
+ * }} {
+ *   projectRepoName
+ * }
+ * @returns Promise<void> returns void
+ */
+export declare function reportTestGenVideos({ projectRepoName, }: {
+    projectRepoName: string;
+}): Promise<void>;
+export declare function setReporterConfig(config: any): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/reporter/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAW5E,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CAAC,EACxC,eAAe,GAChB,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;CACzB,iBA4BA;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAGnD"}

package/dist/reporter/index.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setReporterConfig = exports.reportTestGenVideos = exports.getReporter = void 0;
+const reporter_1 = require("@empiricalrun/reporter");
+const logger_1 = require("../bin/logger");
+const uploader_1 = require("../uploader");
+let reporterInstance = undefined;
+let reporterConfig = undefined;
+function getReporter() {
+    // if no config is set, return undefined
+    if (!reporterConfig) {
+        console.warn("initialising reporter without any config");
+    }
+    // initialise once config is set
+    if (!reporterInstance && reporterConfig) {
+        reporterInstance = new reporter_1.Reporter(reporterConfig);
+    }
+    return reporterInstance;
+}
+exports.getReporter = getReporter;
+/**
+ * function will upload videos and json summary of test results to r2 and report them to reporter.
+ * method won't throw error if it fails to report
+ * @param {{
+ *   projectRepoName: string;
+ * }} {
+ *   projectRepoName
+ * }
+ * @returns Promise<void> returns void
+ */
+async function reportTestGenVideos({ projectRepoName, }) {
+    const logger = new logger_1.CustomLogger();
+    try {
+        if (!(0, uploader_1.checkIfResultsUploadAllowed)()) {
+            logger.log("Skipped uploading generated test video");
+        }
+        const { videoUrls } = await (0, uploader_1.uploadTestResultsUsingPrjtRepo)({
+            projectRepoName,
+        });
+        const reporter = getReporter();
+        const reporterMessage = `
+  
+  Here are the videos of the generated test:
+  
+  ${videoUrls
+            .map((url) => `
+  <video src="${url}" autoplay="true" muted="true" controls loop playsinline></video>`)
+            .join("\n")}
+    `;
+        await reporter?.report(new reporter_1.ProcessLogMessageBuilder({ message: reporterMessage }));
+    }
+    catch (err) {
+        logger.error("Failed to report test results");
+        console.error(err);
+    }
+}
+exports.reportTestGenVideos = reportTestGenVideos;
+function setReporterConfig(config) {
+    console.info("initialised reporter config");
+    reporterConfig = config;
+}
+exports.setReporterConfig = setReporterConfig;

package/dist/uploader/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * function to upload test results to r2 using the project repo name
+ * this only uploads json summary of test results
+ * @param { projectName: string } projectRepoName - name of the project repo
+ * @returns urls of videos and summary json
+ */
+export declare function uploadTestResultsUsingPrjtRepo({ projectRepoName, }: {
+    projectRepoName: string;
+}): Promise<{
+    videoUrls: string[];
+    summaryUrl: string;
+}>;
+export declare function checkIfResultsUploadAllowed(): string | undefined;
+//# sourceMappingURL=index.d.ts.map

package/dist/uploader/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/uploader/index.ts"],"names":[],"mappings":"AAiBA;;;;;GAKG;AACH,wBAAsB,8BAA8B,CAAC,EACnD,eAAe,GAChB,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC;IACV,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAyBD;AAED,wBAAgB,2BAA2B,uBAQ1C"}

package/dist/uploader/index.js

@@ -0,0 +1,52 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.checkIfResultsUploadAllowed = exports.uploadTestResultsUsingPrjtRepo = void 0;
+const path_1 = __importDefault(require("path"));
+const r2_1 = require("./r2");
+// json summary of test results
+const TEST_RESULTS_DIR = "test-results";
+const UPLOAD_BUCKET = "test-report";
+const UPLOAD_DOMAIN = "https://reports.empirical.run"; // domain based on bucket mentioned above
+function getFullUploadPath(filePath, uploadDir) {
+    const relativeFilePath = filePath.replace(path_1.default.join(process.cwd(), TEST_RESULTS_DIR), "");
+    return `${UPLOAD_DOMAIN}/${uploadDir}${relativeFilePath}`;
+}
+/**
+ * function to upload test results to r2 using the project repo name
+ * this only uploads json summary of test results
+ * @param { projectName: string } projectRepoName - name of the project repo
+ * @returns urls of videos and summary json
+ */
+async function uploadTestResultsUsingPrjtRepo({ projectRepoName, }) {
+    const uploadUniqueId = crypto.randomUUID();
+    // project repo name is the github repo name
+    // the folder names in r2 are the github repo name without the `-tests` suffix
+    const uploadDir = `test-generation/${projectRepoName.replace("-tests", "")}/${uploadUniqueId}`;
+    const files = await (0, r2_1.uploadDirectory)({
+        sourceDir: path_1.default.join(process.cwd(), TEST_RESULTS_DIR),
+        destinationDir: uploadDir,
+        uploadBucket: UPLOAD_BUCKET,
+    });
+    const fileNames = Object.keys(files);
+    // TODO: parse the json summary and then detect video attachments
+    // current assumption
+    // - test gen will only run on a single spec file
+    // - the video files will be of format <spec-file-name>.webm
+    const videoFiles = fileNames.filter((fileName) => fileName.endsWith(".webm"));
+    return {
+        videoUrls: videoFiles.map((fileName) => getFullUploadPath(fileName, uploadDir)),
+        summaryUrl: getFullUploadPath("/test-results/summary.json", uploadDir),
+    };
+}
+exports.uploadTestResultsUsingPrjtRepo = uploadTestResultsUsingPrjtRepo;
+function checkIfResultsUploadAllowed() {
+    // TODO: check for valid R2 credentials
+    // check for project repo name, and r2 creds
+    return (process.env.R2_ACCOUNT_ID &&
+        process.env.R2_ACCESS_KEY_ID &&
+        process.env.R2_SECRET_ACCESS_KEY);
+}
+exports.checkIfResultsUploadAllowed = checkIfResultsUploadAllowed;

package/dist/uploader/r2.d.ts

@@ -0,0 +1,10 @@
+interface FileMap {
+    [file: string]: string;
+}
+export declare function uploadDirectory({ sourceDir, destinationDir, uploadBucket, }: {
+    sourceDir: string;
+    destinationDir: string;
+    uploadBucket: string;
+}): Promise<FileMap>;
+export {};
+//# sourceMappingURL=r2.d.ts.map

package/dist/uploader/r2.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"r2.d.ts","sourceRoot":"","sources":["../../src/uploader/r2.ts"],"names":[],"mappings":"AAqBA,UAAU,OAAO;IACf,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAgGD,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,cAAc,EACd,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,OAAO,CAAC,CAWnB"}

package/dist/uploader/r2.js

@@ -0,0 +1,124 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadDirectory = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const fs = __importStar(require("fs"));
+const md5_1 = __importDefault(require("md5"));
+const path_1 = __importDefault(require("path"));
+const getFileList = (dir) => {
+    let files = [];
+    const items = fs.readdirSync(dir, {
+        withFileTypes: true,
+    });
+    for (const item of items) {
+        const isDir = item.isDirectory();
+        const absolutePath = `${dir}/${item.name}`;
+        if (isDir) {
+            files = [...files, ...getFileList(absolutePath)];
+        }
+        else {
+            files.push(absolutePath);
+        }
+    }
+    return files;
+};
+const run = async (config) => {
+    const map = new Map();
+    const urls = {};
+    const S3 = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    const files = getFileList(config.sourceDir);
+    const mime = (await import("mime")).default;
+    await Promise.all(files.map(async (file) => {
+        console.log(file);
+        const fileStream = fs.readFileSync(file);
+        console.log(config.sourceDir);
+        console.log(config.destinationDir);
+        //const fileName = file.replace(/^.*[\\\/]/, "");
+        const fileName = file.replace(config.sourceDir, "");
+        const fileKey = path_1.default.join(config.destinationDir !== "" ? config.destinationDir : config.sourceDir, fileName);
+        if (fileKey.includes(".gitkeep"))
+            return;
+        console.log(fileKey);
+        const mimeType = mime.getType(file);
+        const uploadParams = {
+            Bucket: config.bucket,
+            Key: fileKey,
+            Body: fileStream,
+            ContentLength: fs.statSync(file).size,
+            ContentType: mimeType ?? "application/octet-stream",
+        };
+        const cmd = new client_s3_1.PutObjectCommand(uploadParams);
+        const digest = (0, md5_1.default)(fileStream);
+        cmd.middlewareStack.add((next) => async (args) => {
+            args.request.headers["if-none-match"] = `"${digest}"`;
+            return await next(args);
+        }, {
+            step: "build",
+            name: "addETag",
+        });
+        try {
+            const data = await S3.send(cmd);
+            console.log(`R2 Success - ${file}`);
+            map.set(file, data);
+            const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
+            urls[file] = fileUrl;
+        }
+        catch (err) {
+            const error = err;
+            if (error["$metadata"]) {
+                if (error.$metadata.httpStatusCode !== 412)
+                    // If-None-Match
+                    throw error;
+            }
+        }
+        return;
+    }));
+    return urls;
+};
+async function uploadDirectory({ sourceDir, destinationDir, uploadBucket, }) {
+    let config = {
+        accountId: process.env.R2_ACCOUNT_ID,
+        accessKeyId: process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        sourceDir,
+        destinationDir,
+    };
+    const uploadedFiles = await run(config);
+    return uploadedFiles;
+}
+exports.uploadDirectory = uploadDirectory;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.16.3",
+  "version": "0.16.5",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -15,6 +15,8 @@
   },
   "author": "Empirical Team <hey@empirical.run>",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.614.0",
+    "@aws-sdk/s3-request-presigner": "^3.614.0",
     "@types/sanitize-html": "^2.11.0",
     "commander": "^12.1.0",
     "detect-port": "^1.6.1",
@@ -25,6 +27,8 @@
     "google-auth-library": "^9.10.0",
     "google-spreadsheet": "^4.1.2",
     "ignore": "^5.3.1",
+    "md5": "^2.3.0",
+    "mime": "^4.0.4",
     "minimatch": "^10.0.1",
     "openai": "^4.47.2",
     "picocolors": "^1.0.1",
@@ -35,12 +39,13 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "@empiricalrun/llm": "^0.2.0",
-    "@empiricalrun/reporter": "^0.11.0"
+    "@empiricalrun/reporter": "^0.11.1"
   },
   "devDependencies": {
     "@types/detect-port": "^1.3.5",
     "@types/express": "^4.17.21",
-    "@types/fs-extra": "^11.0.4"
+    "@types/fs-extra": "^11.0.4",
+    "@types/md5": "^2.3.5"
   },
   "scripts": {
     "dev": "tsc --build --watch",

package/dist/bin/reporter/ci.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/bin/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,wBAAsB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,uBAerD"}

package/dist/bin/reporter/index.d.ts

@@ -1,4 +0,0 @@
-import { Reporter } from "@empiricalrun/reporter";
-export declare function getReporter(): Reporter | undefined;
-export declare function setReporterConfig(config: any): void;
-//# sourceMappingURL=index.d.ts.map

package/dist/bin/reporter/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAIlD,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAGnD"}

package/dist/bin/reporter/index.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.setReporterConfig = exports.getReporter = void 0;
-const reporter_1 = require("@empiricalrun/reporter");
-let reporterInstance = undefined;
-let reporterConfig = undefined;
-function getReporter() {
-    // if no config is set, return undefined
-    if (!reporterConfig) {
-        console.warn("initialising reporter without any config");
-    }
-    // initialise once config is set
-    if (!reporterInstance && reporterConfig) {
-        reporterInstance = new reporter_1.Reporter(reporterConfig);
-    }
-    return reporterInstance;
-}
-exports.getReporter = getReporter;
-function setReporterConfig(config) {
-    console.info("initialised reporter config");
-    reporterConfig = config;
-}
-exports.setReporterConfig = setReporterConfig;