@empiricalrun/test-gen 0.16.2 → 0.16.4

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @empiricalrun/test-gen
 
+## 0.16.4
+
+### Patch Changes
+
+- 4c32cf9: feat: add support for uploading test video of browsing agent generated tests
+
+## 0.16.3
+
+### Patch Changes
+
+- 8243b11: fix: add logs for errors while running playwright
+
 ## 0.16.2
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts

@@ -1,7 +1,10 @@
 import { Page } from "playwright";
-export declare function browsingAgent(task: string, page: Page, options?: {
+import { TestGenConfigOptions } from "../../types";
+type BrowsingAgentOptions = Partial<TestGenConfigOptions> & {
     htmlSanitize?: {
         disallowedStrings?: string[];
     };
-}): Promise<string>;
+};
+export declare function browsingAgent(task: string, page: Page, options: BrowsingAgentOptions): Promise<string>;
+export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,GAAE;IACP,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACE,mBAyFP"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAInD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBA4F9B"}

package/dist/agent/browsing/index.js

@@ -4,10 +4,11 @@ exports.browsingAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const actions_1 = require("../../actions");
 const logger_1 = require("../../bin/logger");
+const constants_1 = require("../../constants");
 const session_1 = require("../../session");
 const html_1 = require("../../utils/html");
 const utils_1 = require("./utils");
-async function browsingAgent(task, page, options = {}) {
+async function browsingAgent(task, page, options) {
     const logger = new logger_1.CustomLogger();
     const trace = new llm_1.LLMTracing({
         name: "browsing-agent",
@@ -43,10 +44,12 @@ async function browsingAgent(task, page, options = {}) {
             messages,
             tools,
             trace,
-            model: "gpt-4o",
-            provider: "openai",
+            model: options.model || constants_1.DEFAULT_MODEL,
+            provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+            providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
             modelParameters: {
-                temperature: 0.5,
+                ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                ...options.modelParameters,
                 tool_choice: "required",
             },
         });

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAaA,wBAAsB,+BAA+B,CAAC,YAAY,EAAE,MAAM,iBA0BzE"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAaA,wBAAsB,+BAA+B,CAAC,YAAY,EAAE,MAAM,iBA4BzE"}

package/dist/agent/browsing/run.js

@@ -26,15 +26,18 @@ async function generateTestsUsingBrowsingAgent(testFilePath) {
     if (!process.env.CI) {
         command = command.concat(` --headed`);
     }
-    const exitCode = await (0, exec_1.cmd)(command.split(" "), {
-        env: {
-            APP_PORT: port.toString(),
-            PW_TEST_HTML_REPORT_OPEN: "never",
-            TEST_GEN_TOKEN: (0, utils_1.getTestConfigCliArg)(),
-        },
-    });
-    if (exitCode != 0) {
-        process.exit(exitCode);
+    try {
+        await (0, exec_1.cmd)(command.split(" "), {
+            env: {
+                APP_PORT: port.toString(),
+                PW_TEST_HTML_REPORT_OPEN: "never",
+                TEST_GEN_TOKEN: (0, utils_1.getTestConfigCliArg)(),
+            },
+        });
+    }
+    catch (e) {
+        logger.error(e);
+        process.exit(1);
     }
     await (0, web_1.removeTestOnly)(testFilePath);
 }

package/dist/agent/codegen/run.d.ts

@@ -1,3 +1,3 @@
-import { Scenario } from "../../types";
-export declare function generateTest(scenarios: Scenario[], file: string, isUpdate: boolean): Promise<Scenario[]>;
+import { Scenario, TestGenConfigOptions } from "../../types";
+export declare function generateTest(scenarios: Scenario[], file: string, isUpdate: boolean, options: TestGenConfigOptions): Promise<Scenario[]>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,wBAAsB,YAAY,CAChC,SAAS,EAAE,QAAQ,EAAE,EACrB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6IrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,SAAS,EAAE,QAAQ,EAAE,EACrB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,OAAO,EACjB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAqJrB"}

package/dist/agent/codegen/run.js

@@ -9,8 +9,9 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../../bin/logger");
 const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
+const constants_1 = require("../../constants");
 const session_1 = require("../../session");
-async function generateTest(scenarios, file, isUpdate) {
+async function generateTest(scenarios, file, isUpdate, options) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(file)) {
         logger.log(`Creating a new spec file: ${file}`);
@@ -55,10 +56,12 @@ async function generateTest(scenarios, file, isUpdate) {
         const firstShotMessage = await (0, llm_1.getLLMResult)({
             messages: instruction,
             trace,
-            model: "gpt-4o",
-            provider: "openai",
+            model: options.model || constants_1.DEFAULT_MODEL,
+            provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+            providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
             modelParameters: {
-                temperature: 0.5,
+                ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                ...options.modelParameters,
             },
         });
         let response = firstShotMessage?.content || "";
@@ -115,9 +118,13 @@ async function generateTest(scenarios, file, isUpdate) {
             const message = await (0, llm_1.getLLMResult)({
                 messages: instruction,
                 trace,
-                model: "gpt-4o",
-                provider: "openai",
-                modelParameters: { temperature: 0.5 },
+                model: options.model || constants_1.DEFAULT_MODEL,
+                provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+                providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+                modelParameters: {
+                    ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                    ...options.modelParameters,
+                },
             });
             response = message?.content || "";
             const readWriteFileSpan = trace.startSpan("write-to-file");

package/dist/bin/index.js

@@ -9,9 +9,9 @@ const dotenv_1 = __importDefault(require("dotenv"));
 const run_1 = require("../agent/browsing/run");
 const utils_1 = require("../agent/browsing/utils");
 const run_2 = require("../agent/codegen/run");
+const reporter_1 = require("../reporter");
+const ci_1 = require("../reporter/ci");
 const logger_1 = require("./logger");
-const reporter_1 = require("./reporter");
-const ci_1 = require("./reporter/ci");
 const utils_2 = require("./utils");
 dotenv_1.default.config({
     path: [".env.local", ".env"],
@@ -30,11 +30,14 @@ async function runAgent(sourceFile, isUpdate, testGenConfigs) {
             logger.success("Generating test using browsing agent");
             await (0, utils_1.prepareFileForBrowsingAgent)(testGenConfig);
             await (0, run_1.generateTestsUsingBrowsingAgent)(specPath);
+            await (0, reporter_1.reportTestGenVideos)({
+                projectRepoName: testGenConfig.options.metadata.projectRepoName,
+            });
             generatedTestScenarios.push(...testGenConfig.scenarios);
         }
         else {
             logger.success("Generating test using coding agent");
-            const gen = await (0, run_2.generateTest)(scenarios, specPath, isUpdate);
+            const gen = await (0, run_2.generateTest)(scenarios, specPath, isUpdate, testGenConfig.options);
             generatedTestScenarios.push(...gen);
         }
     }
@@ -49,6 +52,7 @@ async function runAgent(sourceFile, isUpdate, testGenConfigs) {
     const { sourceFile, testGenConfigs, isUpdate } = await (0, utils_2.parseCliArgs)();
     (0, reporter_1.setReporterConfig)(testGenConfigs[0]?.options?.metadata);
     const generated = await runAgent(sourceFile, isUpdate, testGenConfigs);
+    // TODO: move these reporters to a better lifecycle
     await (0, ci_1.reportOnCI)(generated);
     process.exit(0);
 })();

package/dist/bin/logger/index.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CustomLogger = void 0;
 const reporter_1 = require("@empiricalrun/reporter");
 const picocolors_1 = require("picocolors");
-const reporter_2 = require("../reporter");
+const reporter_2 = require("../../reporter");
 class CustomLogger {
     useReporter = false;
     constructor({ useReporter = true } = {}) {

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAQpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwC7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAM3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAOhD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAQpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAiD7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAM3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAOhD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD"}

package/dist/bin/utils/platform/web/index.js

@@ -56,7 +56,17 @@ function validateTypescript(filePath) {
     const semanticDiagnostics = program.getSemanticDiagnostics(sourceFile);
     if (semanticDiagnostics.length > 0) {
         semanticDiagnostics.forEach((diagnostic) => {
-            errors.push(diagnostic.messageText.toString());
+            if (typeof diagnostic.messageText === "string") {
+                errors.push(diagnostic.messageText);
+            }
+            else {
+                let messageChain = diagnostic.messageText;
+                errors.push(messageChain.messageText);
+                while (messageChain?.next) {
+                    messageChain = messageChain.next[0];
+                    errors.push(messageChain?.messageText);
+                }
+            }
         });
     }
     return errors;

package/dist/constants/index.d.ts

@@ -0,0 +1,6 @@
+import { LLMModel, LLMProvider, ModelParameters } from "@empiricalrun/llm";
+export declare const MODEL_API_KEYS: Record<LLMProvider, string>;
+export declare const DEFAULT_MODEL_PROVIDER: LLMProvider;
+export declare const DEFAULT_MODEL: LLMModel;
+export declare const DEFAULT_MODEL_PARAMETERS: ModelParameters;
+//# sourceMappingURL=index.d.ts.map

package/dist/constants/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/constants/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAE3E,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAItD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,WAAsB,CAAC;AAE5D,eAAO,MAAM,aAAa,EAAE,QAAmB,CAAC;AAEhD,eAAO,MAAM,wBAAwB,EAAE,eAEtC,CAAC"}

package/dist/constants/index.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_MODEL_PARAMETERS = exports.DEFAULT_MODEL = exports.DEFAULT_MODEL_PROVIDER = exports.MODEL_API_KEYS = void 0;
+exports.MODEL_API_KEYS = {
+    google: process.env.GOOGLE_API_KEY || "",
+    anthropic: process.env.ANTHROPIC_API_KEY || "",
+    openai: process.env.OPENAI_API_KEY || "",
+};
+exports.DEFAULT_MODEL_PROVIDER = "openai";
+exports.DEFAULT_MODEL = "gpt-4o";
+exports.DEFAULT_MODEL_PARAMETERS = {
+    temperature: 0.5,
+};

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBAiBnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBAkBnE"}

package/dist/index.js

@@ -6,9 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTest = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const browsing_1 = require("./agent/browsing");
-const reporter_1 = require("./bin/reporter");
 const utils_1 = require("./bin/utils");
 const client_1 = __importDefault(require("./file/client"));
+const reporter_1 = require("./reporter");
 process.on("exit", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());
@@ -17,13 +17,14 @@ async function createTest(task, page, test) {
     const testConfigArg = process.env.TEST_GEN_TOKEN;
     const { testGenConfigs } = await (0, utils_1.parseCliArgs)(testConfigArg);
     const [testGenConfig] = testGenConfigs;
-    (0, reporter_1.setReporterConfig)(testGenConfig?.options?.metadata);
+    (0, reporter_1.setReporterConfig)(testGenConfig.options?.metadata);
     const fileService = new client_1.default(Number(port));
     test.setTimeout(900000);
     const code = await (0, browsing_1.browsingAgent)(task, page, {
         htmlSanitize: {
             disallowedStrings: ["v-data-table__td v-data-table-column--align-start"],
         },
+        ...testGenConfig.options,
     });
     await fileService.updateTest({
         task,

package/dist/{bin/reporter → reporter}/ci.d.ts

@@ -1,3 +1,3 @@
-import { Scenario } from "../../types";
+import { Scenario } from "../types";
 export declare function reportOnCI(scenarios: Scenario[]): Promise<Scenario[]>;
 //# sourceMappingURL=ci.d.ts.map

package/dist/reporter/ci.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC,wBAAsB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,uBAerD"}

package/dist/reporter/index.d.ts

@@ -0,0 +1,17 @@
+import { Reporter } from "@empiricalrun/reporter";
+export declare function getReporter(): Reporter | undefined;
+/**
+ * function will upload videos and json summary of test results to r2 and report them to reporter.
+ * method won't throw error if it fails to report
+ * @param {{
+ *   projectRepoName: string;
+ * }} {
+ *   projectRepoName
+ * }
+ * @returns Promise<void> returns void
+ */
+export declare function reportTestGenVideos({ projectRepoName, }: {
+    projectRepoName: string;
+}): Promise<void>;
+export declare function setReporterConfig(config: any): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/reporter/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAW5E,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CAAC,EACxC,eAAe,GAChB,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;CACzB,iBA4BA;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAGnD"}

package/dist/reporter/index.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setReporterConfig = exports.reportTestGenVideos = exports.getReporter = void 0;
+const reporter_1 = require("@empiricalrun/reporter");
+const logger_1 = require("../bin/logger");
+const uploader_1 = require("../uploader");
+let reporterInstance = undefined;
+let reporterConfig = undefined;
+function getReporter() {
+    // if no config is set, return undefined
+    if (!reporterConfig) {
+        console.warn("initialising reporter without any config");
+    }
+    // initialise once config is set
+    if (!reporterInstance && reporterConfig) {
+        reporterInstance = new reporter_1.Reporter(reporterConfig);
+    }
+    return reporterInstance;
+}
+exports.getReporter = getReporter;
+/**
+ * function will upload videos and json summary of test results to r2 and report them to reporter.
+ * method won't throw error if it fails to report
+ * @param {{
+ *   projectRepoName: string;
+ * }} {
+ *   projectRepoName
+ * }
+ * @returns Promise<void> returns void
+ */
+async function reportTestGenVideos({ projectRepoName, }) {
+    const logger = new logger_1.CustomLogger();
+    try {
+        if (!(0, uploader_1.checkIfResultsUploadAllowed)()) {
+            logger.log("Skipped uploading generated test video");
+        }
+        const { videoUrls } = await (0, uploader_1.uploadTestResultsUsingPrjtRepo)({
+            projectRepoName,
+        });
+        const reporter = getReporter();
+        const reporterMessage = `
+  
+  Here are the videos of the generated test:
+  
+  ${videoUrls
+            .map((url) => `
+  <video src="${url}" autoplay="true" muted="true" controls loop playsinline></video>`)
+            .join("\n")}
+    `;
+        await reporter?.report(new reporter_1.ProcessLogMessageBuilder({ message: reporterMessage }));
+    }
+    catch (err) {
+        logger.error("Failed to report test results");
+        console.error(err);
+    }
+}
+exports.reportTestGenVideos = reportTestGenVideos;
+function setReporterConfig(config) {
+    console.info("initialised reporter config");
+    reporterConfig = config;
+}
+exports.setReporterConfig = setReporterConfig;

package/dist/types/index.d.ts

@@ -1,3 +1,4 @@
+import { LLMModel, LLMProvider, ModelParameters } from "@empiricalrun/llm";
 import OpenAI from "openai";
 import { Page } from "playwright";
 export type FileContent = {
@@ -6,9 +7,10 @@ export type FileContent = {
 };
 export type TestGenConfigOptions = {
     agent: "code" | "browser";
-    model: string;
+    model: LLMModel;
+    modelProvider: LLMProvider;
+    modelParameters?: ModelParameters;
     metadata: {
-        apiToken: string;
         testSessionId: number;
         testCaseName: string;
         projectRepoName: string;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}

package/dist/uploader/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * function to upload test results to r2 using the project repo name
+ * this only uploads json summary of test results
+ * @param { projectName: string } projectRepoName - name of the project repo
+ * @returns urls of videos and summary json
+ */
+export declare function uploadTestResultsUsingPrjtRepo({ projectRepoName, }: {
+    projectRepoName: string;
+}): Promise<{
+    videoUrls: string[];
+    summaryUrl: string;
+}>;
+export declare function checkIfResultsUploadAllowed(): string | undefined;
+//# sourceMappingURL=index.d.ts.map

package/dist/uploader/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/uploader/index.ts"],"names":[],"mappings":"AAiBA;;;;;GAKG;AACH,wBAAsB,8BAA8B,CAAC,EACnD,eAAe,GAChB,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC;IACV,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAyBD;AAED,wBAAgB,2BAA2B,uBAQ1C"}

package/dist/uploader/index.js

@@ -0,0 +1,52 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.checkIfResultsUploadAllowed = exports.uploadTestResultsUsingPrjtRepo = void 0;
+const path_1 = __importDefault(require("path"));
+const r2_1 = require("./r2");
+// json summary of test results
+const TEST_RESULTS_DIR = "test-results";
+const UPLOAD_BUCKET = "test-report";
+const UPLOAD_DOMAIN = "https://reports.empirical.run"; // domain based on bucket mentioned above
+function getFullUploadPath(filePath, uploadDir) {
+    const relativeFilePath = filePath.replace(path_1.default.join(process.cwd(), TEST_RESULTS_DIR), "");
+    return `${UPLOAD_DOMAIN}/${uploadDir}${relativeFilePath}`;
+}
+/**
+ * function to upload test results to r2 using the project repo name
+ * this only uploads json summary of test results
+ * @param { projectName: string } projectRepoName - name of the project repo
+ * @returns urls of videos and summary json
+ */
+async function uploadTestResultsUsingPrjtRepo({ projectRepoName, }) {
+    const uploadUniqueId = crypto.randomUUID();
+    // project repo name is the github repo name
+    // the folder names in r2 are the github repo name without the `-tests` suffix
+    const uploadDir = `test-generation/${projectRepoName.replace("-tests", "")}/${uploadUniqueId}`;
+    const files = await (0, r2_1.uploadDirectory)({
+        sourceDir: path_1.default.join(process.cwd(), TEST_RESULTS_DIR),
+        destinationDir: uploadDir,
+        uploadBucket: UPLOAD_BUCKET,
+    });
+    const fileNames = Object.keys(files);
+    // TODO: parse the json summary and then detect video attachments
+    // current assumption
+    // - test gen will only run on a single spec file
+    // - the video files will be of format <spec-file-name>.webm
+    const videoFiles = fileNames.filter((fileName) => fileName.endsWith(".webm"));
+    return {
+        videoUrls: videoFiles.map((fileName) => getFullUploadPath(fileName, uploadDir)),
+        summaryUrl: getFullUploadPath("/test-results/summary.json", uploadDir),
+    };
+}
+exports.uploadTestResultsUsingPrjtRepo = uploadTestResultsUsingPrjtRepo;
+function checkIfResultsUploadAllowed() {
+    // TODO: check for valid R2 credentials
+    // check for project repo name, and r2 creds
+    return (process.env.R2_ACCOUNT_ID &&
+        process.env.R2_ACCESS_KEY_ID &&
+        process.env.R2_SECRET_ACCESS_KEY);
+}
+exports.checkIfResultsUploadAllowed = checkIfResultsUploadAllowed;

package/dist/uploader/r2.d.ts

@@ -0,0 +1,10 @@
+interface FileMap {
+    [file: string]: string;
+}
+export declare function uploadDirectory({ sourceDir, destinationDir, uploadBucket, }: {
+    sourceDir: string;
+    destinationDir: string;
+    uploadBucket: string;
+}): Promise<FileMap>;
+export {};
+//# sourceMappingURL=r2.d.ts.map

package/dist/uploader/r2.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"r2.d.ts","sourceRoot":"","sources":["../../src/uploader/r2.ts"],"names":[],"mappings":"AAqBA,UAAU,OAAO;IACf,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAgGD,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,cAAc,EACd,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,OAAO,CAAC,CAWnB"}

package/dist/uploader/r2.js

@@ -0,0 +1,124 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadDirectory = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const fs = __importStar(require("fs"));
+const md5_1 = __importDefault(require("md5"));
+const path_1 = __importDefault(require("path"));
+const getFileList = (dir) => {
+    let files = [];
+    const items = fs.readdirSync(dir, {
+        withFileTypes: true,
+    });
+    for (const item of items) {
+        const isDir = item.isDirectory();
+        const absolutePath = `${dir}/${item.name}`;
+        if (isDir) {
+            files = [...files, ...getFileList(absolutePath)];
+        }
+        else {
+            files.push(absolutePath);
+        }
+    }
+    return files;
+};
+const run = async (config) => {
+    const map = new Map();
+    const urls = {};
+    const S3 = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    const files = getFileList(config.sourceDir);
+    const mime = (await import("mime")).default;
+    await Promise.all(files.map(async (file) => {
+        console.log(file);
+        const fileStream = fs.readFileSync(file);
+        console.log(config.sourceDir);
+        console.log(config.destinationDir);
+        //const fileName = file.replace(/^.*[\\\/]/, "");
+        const fileName = file.replace(config.sourceDir, "");
+        const fileKey = path_1.default.join(config.destinationDir !== "" ? config.destinationDir : config.sourceDir, fileName);
+        if (fileKey.includes(".gitkeep"))
+            return;
+        console.log(fileKey);
+        const mimeType = mime.getType(file);
+        const uploadParams = {
+            Bucket: config.bucket,
+            Key: fileKey,
+            Body: fileStream,
+            ContentLength: fs.statSync(file).size,
+            ContentType: mimeType ?? "application/octet-stream",
+        };
+        const cmd = new client_s3_1.PutObjectCommand(uploadParams);
+        const digest = (0, md5_1.default)(fileStream);
+        cmd.middlewareStack.add((next) => async (args) => {
+            args.request.headers["if-none-match"] = `"${digest}"`;
+            return await next(args);
+        }, {
+            step: "build",
+            name: "addETag",
+        });
+        try {
+            const data = await S3.send(cmd);
+            console.log(`R2 Success - ${file}`);
+            map.set(file, data);
+            const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
+            urls[file] = fileUrl;
+        }
+        catch (err) {
+            const error = err;
+            if (error["$metadata"]) {
+                if (error.$metadata.httpStatusCode !== 412)
+                    // If-None-Match
+                    throw error;
+            }
+        }
+        return;
+    }));
+    return urls;
+};
+async function uploadDirectory({ sourceDir, destinationDir, uploadBucket, }) {
+    let config = {
+        accountId: process.env.R2_ACCOUNT_ID,
+        accessKeyId: process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        sourceDir,
+        destinationDir,
+    };
+    const uploadedFiles = await run(config);
+    return uploadedFiles;
+}
+exports.uploadDirectory = uploadDirectory;

package/dist/utils/exec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAGA,wBAAgB,GAAG,CACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GACxC,OAAO,CAAC,MAAM,CAAC,CAejB"}
+{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAGA,wBAAgB,GAAG,CACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GACxC,OAAO,CAAC,MAAM,CAAC,CAyBjB"}

package/dist/utils/exec.js

@@ -7,18 +7,29 @@ exports.cmd = void 0;
 const child_process_1 = require("child_process");
 const process_1 = __importDefault(require("process"));
 function cmd(command, options) {
-    return new Promise((resolveFunc) => {
+    let lastLog = "";
+    return new Promise((resolveFunc, rejectFunc) => {
         let p = (0, child_process_1.spawn)(command[0], command.slice(1), {
             env: { ...process_1.default.env, ...options.env },
         });
         p.stdout.on("data", (x) => {
-            process_1.default.stdout.write(x.toString());
+            const log = x.toString();
+            process_1.default.stdout.write(log);
+            lastLog = log;
         });
         p.stderr.on("data", (x) => {
+            const log = x.toString();
             process_1.default.stderr.write(x.toString());
+            lastLog = log;
         });
         p.on("exit", (code) => {
-            resolveFunc(code);
+            if (code != 0) {
+                // assuming last log is the error message before exiting
+                rejectFunc(lastLog);
+            }
+            else {
+                resolveFunc(code);
+            }
         });
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.16.2",
+  "version": "0.16.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -15,6 +15,8 @@
   },
   "author": "Empirical Team <hey@empirical.run>",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.614.0",
+    "@aws-sdk/s3-request-presigner": "^3.614.0",
     "@types/sanitize-html": "^2.11.0",
     "commander": "^12.1.0",
     "detect-port": "^1.6.1",
@@ -25,6 +27,8 @@
     "google-auth-library": "^9.10.0",
     "google-spreadsheet": "^4.1.2",
     "ignore": "^5.3.1",
+    "md5": "^2.3.0",
+    "mime": "^4.0.4",
     "minimatch": "^10.0.1",
     "openai": "^4.47.2",
     "picocolors": "^1.0.1",
@@ -40,7 +44,8 @@
   "devDependencies": {
     "@types/detect-port": "^1.3.5",
     "@types/express": "^4.17.21",
-    "@types/fs-extra": "^11.0.4"
+    "@types/fs-extra": "^11.0.4",
+    "@types/md5": "^2.3.5"
   },
   "scripts": {
     "dev": "tsc --build --watch",

package/dist/bin/reporter/ci.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/bin/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,wBAAsB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,uBAerD"}

package/dist/bin/reporter/index.d.ts

@@ -1,4 +0,0 @@
-import { Reporter } from "@empiricalrun/reporter";
-export declare function getReporter(): Reporter | undefined;
-export declare function setReporterConfig(config: any): void;
-//# sourceMappingURL=index.d.ts.map

package/dist/bin/reporter/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAIlD,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAGnD"}

package/dist/bin/reporter/index.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.setReporterConfig = exports.getReporter = void 0;
-const reporter_1 = require("@empiricalrun/reporter");
-let reporterInstance = undefined;
-let reporterConfig = undefined;
-function getReporter() {
-    // if no config is set, return undefined
-    if (!reporterConfig) {
-        console.warn("initialising reporter without any config");
-    }
-    // initialise once config is set
-    if (!reporterInstance && reporterConfig) {
-        reporterInstance = new reporter_1.Reporter(reporterConfig);
-    }
-    return reporterInstance;
-}
-exports.getReporter = getReporter;
-function setReporterConfig(config) {
-    console.info("initialised reporter config");
-    reporterConfig = config;
-}
-exports.setReporterConfig = setReporterConfig;