@empiricalrun/test-gen 0.16.13 → 0.17.1

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @empiricalrun/test-gen
 
+## 0.17.1
+
+### Patch Changes
+
+- ac0256e: feat: remove support for update flag in test gen
+
+## 0.17.0
+
+### Minor Changes
+
+- 914b207: fix: remove google sheet support in test gen
+
+### Patch Changes
+
+- 74eefc4: fix: remove comments of browser agent
+
 ## 0.16.13
 
 ### Patch Changes

package/README.md

@@ -8,61 +8,29 @@ Our agents that generate Playwright tests. There are 2 agents
 ## Usage
 
 ```sh
-npx @empiricalrun/test-gen $args
+npx @empiricalrun/test-gen TEST_GEN_TOKEN
 ```
 
-where `$args` can be one of the following
-
 ### Add new test
 
 ```sh
-npx @empiricalrun/test-gen base64string
+npx @empiricalrun/test-gen TEST_GEN_TOKEN
 ```
 
 - This will trigger browsing agent to write a new test for this scenario
-- See [Base64 payload](#base64-payload)
-
-```sh
-npx @empiricalrun/test-gen delete.spec.ts
-```
-
-- This will trigger browsing agent to write a new test in this file
-- **Note** that the file must have `createTest(...)` where the scenario is specified
-
-#### Bulk add
-
-```sh
-npx @empiricalrun/test-gen scenarios.yml
-
-# optional: filter for one scenario
-npx @empiricalrun/test-gen scenarios.yml $scenario_name
-```
-
-Using yml or google spreadsheet. **Note** that this only runs the code editing agent
-
-```yaml ./tests/scenarios.yaml
-dir: ./tests/ # [optional] directory where you want to create tests. Default - ./tests
-scenarios: # list of scenarios
-    - name: user should be able to login successfully # name of the test case to be generated
-      steps: # steps for the test case
-        - cLick on "Login" button
-        - click on input "username"
-        - fill username as "foo@bar.com"
-        - fill password as "bar@123"
-        - click on "Login Button"
-      assert: "Login successful" text should be visible # assertion
-```
+- See [Test gen token: Base64 payload](#test-gen-token-base64-payload)
 
 ### Update existing test
 
 ```sh
-npx @empiricalrun/test-gen base64string -u
+npx @empiricalrun/test-gen TEST_GEN_TOKEN
 ```
 
+- If the test case is already present in the file, the test gen agent will update the existing test as per the steps provided in the payload
 - This will trigger code editing agent to write a new test for this scenario
-- See [Base64 payload](#base64-payload)
+- See [Test gen token: Base64 payload](#test-gen-token-base64-payload)
 
-### Base64 payload
+### Test gen token: Base64 payload
 
 Convert this object into a base64 payload - for new and update test.
 
@@ -72,10 +40,10 @@ const scenario = {
   steps: [
     "fill in the username as foobar@gmail.com",
     "fill in the password as bazbaz",
-    "click on the login button"
+    "click on the login button",
   ],
   assert: "",
-  group: "home"
+  group: "home",
 };
-btoa(JSON.stringify(scenario))
+btoa(JSON.stringify(scenario));
 ```

package/dist/agent/browsing/utils.d.ts

@@ -2,7 +2,7 @@ import { Page } from "playwright";
 import { PlaywrightTestConfig } from "playwright/test";
 import { TestGenConfig } from "../../types";
 export declare function isRegExp(obj: any): obj is RegExp;
-export declare function prepareBrowsingAgentTask(steps: string[], assert?: string): string;
+export declare function prepareBrowsingAgentTask(steps: string[]): string;
 export declare function prepareFileForBrowsingAgent(genConfig: TestGenConfig): Promise<void>;
 export declare function injectPwLocatorGenerator(page: Page): Promise<void>;
 export declare function canRunBrowsingAgent(filePath: string): void;

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AASvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,UAIxE;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAqCzE;AAiBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,GACtB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,8EASA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AASvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAqCzE;AAiBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,GACtB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,8EASA"}

package/dist/agent/browsing/utils.js

@@ -16,15 +16,15 @@ function isRegExp(obj) {
         Object.prototype.toString.call(obj) === "[object RegExp]");
 }
 exports.isRegExp = isRegExp;
-function prepareBrowsingAgentTask(steps, assert) {
+function prepareBrowsingAgentTask(steps) {
     const sanitizedSteps = steps.map((step) => step.replace(/`/g, "\\`"));
-    const task = `${sanitizedSteps.join("\n")}\n${assert || ""}`;
+    const task = `${sanitizedSteps.join("\n")}\n`;
     return task;
 }
 exports.prepareBrowsingAgentTask = prepareBrowsingAgentTask;
 async function prepareFileForBrowsingAgent(genConfig) {
-    const { specPath, scenarios } = genConfig;
-    const { name, steps, assert } = scenarios[0];
+    const { specPath, testCase } = genConfig;
+    const { name, steps } = testCase;
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(specPath)) {
         await fs_extra_1.default.createFile(specPath);
@@ -33,7 +33,7 @@ async function prepareFileForBrowsingAgent(genConfig) {
     if (name && steps && steps.length) {
         const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
         const testBlock = (0, web_1.getTypescriptTestBlock)(name, existingContents);
-        const mergedSteps = prepareBrowsingAgentTask(steps, assert);
+        const mergedSteps = prepareBrowsingAgentTask(steps);
         let newContents = existingContents;
         if (testBlock) {
             logger.log("appending to existing test block");

package/dist/agent/codegen/run.d.ts

@@ -1,3 +1,3 @@
-import { Scenario, TestGenConfigOptions } from "../../types";
-export declare function generateTest(scenarios: Scenario[], file: string, isUpdate: boolean, options: TestGenConfigOptions): Promise<Scenario[]>;
+import { TestCase, TestGenConfigOptions } from "../../types";
+export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions): Promise<TestCase[]>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,SAAS,EAAE,QAAQ,EAAE,EACrB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,OAAO,EACjB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAwJrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CA8IrB"}

package/dist/agent/codegen/run.js

@@ -11,7 +11,7 @@ const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const session_1 = require("../../session");
-async function generateTest(scenarios, file, isUpdate, options) {
+async function generateTest(testCase, file, options) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(file)) {
         logger.log(`Creating a new spec file: ${file}`);
@@ -19,42 +19,96 @@ async function generateTest(scenarios, file, isUpdate, options) {
     }
     const context = await (0, context_1.contextForGeneration)(file);
     const { codePrompt, pomPrompt, testFileContent } = context;
-    const generatedScenarios = [];
-    for (const i in scenarios) {
-        logger.logEmptyLine();
-        const trace = new llm_1.LLMTracing({
-            name: "generate-test",
-            sessionDetails: (0, session_1.getSessionDetails)(),
-            tags: [options.metadata.projectName, options.metadata.environment].filter((s) => !!s),
-        });
-        trace.event({
-            name: "collate-files-as-text",
-            output: {
-                codePrompt,
-                pomPrompt,
-                testFileContent,
-            },
-        });
-        const scenario = scenarios[i];
-        trace.update({ input: { scenario } });
-        logger.log("Generating test for scenario:", scenario?.name);
-        if (!isUpdate && testFileContent.includes(`test("${scenario?.name}"`)) {
-            logger.success("Test already exists for this scenario");
-            trace.event({ name: "test-already-exists" });
-            continue;
+    const generatedTestCases = [];
+    logger.logEmptyLine();
+    const trace = new llm_1.LLMTracing({
+        name: "generate-test",
+        sessionDetails: (0, session_1.getSessionDetails)(),
+        tags: [options.metadata.projectName, options.metadata.environment].filter((s) => !!s),
+    });
+    trace.event({
+        name: "collate-files-as-text",
+        output: {
+            codePrompt,
+            pomPrompt,
+            testFileContent,
+        },
+    });
+    trace.update({ input: { testCase } });
+    const isUpdate = testFileContent.includes(`test("${testCase?.name}"`);
+    const promptSpan = trace.startSpan(isUpdate ? "update-scenario-prompt" : "add-scenario-prompt");
+    const promptName = isUpdate ? "update-scenario" : "add-scenario";
+    const instruction = await (0, llm_1.getPrompt)(promptName, {
+        testFiles: codePrompt,
+        pageFiles: pomPrompt,
+        scenarioName: testCase.name,
+        scenarioSteps: testCase.steps.join("\n"),
+        scenarioFile: file,
+    });
+    promptSpan.end({ output: { instruction } });
+    const firstShotMessage = await (0, llm_1.getLLMResult)({
+        messages: instruction,
+        trace,
+        model: options.model || constants_1.DEFAULT_MODEL,
+        provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+        providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+        modelParameters: {
+            ...constants_1.DEFAULT_MODEL_PARAMETERS,
+            ...options.modelParameters,
+        },
+    });
+    let response = firstShotMessage?.content || "";
+    logger.success("Test generated successfully!");
+    const readWriteFileSpan = trace.startSpan("write-to-file");
+    let contents = fs_extra_1.default.readFileSync(file, "utf-8");
+    const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response);
+    let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
+    if (isUpdate) {
+        const testBlock = (0, web_1.getTypescriptTestBlock)(testCase?.name, contents);
+        contents = contents.replace(testBlock, `\n\n${strippedContent}`);
+        updatedContent = prependContent + contents;
+    }
+    await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
+    readWriteFileSpan.end({ output: { updatedContent } });
+    logger.log("Linting generated code...");
+    trace.event({ name: "lint-file" });
+    await (0, web_1.lintErrors)(file);
+    const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
+    logger.log("Validating types...");
+    let errors = (0, web_1.validateTypescript)(file);
+    validateTypesSpan.end({ output: { errors } });
+    if (!errors.length) {
+        logger.success("Found no type issues!");
+    }
+    const maxIteration = 2;
+    let counter = 0;
+    while (errors.length > 0) {
+        const fileContent = fs_extra_1.default.readFileSync(file, "utf-8");
+        counter += 1;
+        if (counter > maxIteration) {
+            trace.event({ name: "code-fix-iteration-max-out" });
+            logger.error([
+                `Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors.`,
+                `Run the test-gen command again, once errors are fixed.`,
+                `Trace: ${trace.url}`,
+            ].join("\n"));
+            break;
         }
-        const promptSpan = trace.startSpan(isUpdate ? "update-scenario-prompt" : "add-scenario-prompt");
-        const promptName = isUpdate ? "update-scenario" : "add-scenario";
-        const instruction = await (0, llm_1.getPrompt)(promptName, {
-            testFiles: codePrompt,
-            pageFiles: pomPrompt,
-            scenarioName: scenario.name,
-            scenarioSteps: scenario.steps.join("\n"),
-            assert: scenario.assert,
+        trace.event({ name: "Found errors fixing" });
+        logger.warn("Found few errors while validating types:");
+        errors.forEach((e) => logger.warn(e));
+        logger.log("Trying to fix above errors...");
+        const promptSpan = trace.startSpan("fix-type-errors-prompt");
+        const instruction = await (0, llm_1.getPrompt)("fix-file-errors-ts", {
+            testFiles: codePrompt || "",
+            pageFiles: pomPrompt || "",
             scenarioFile: file,
+            errors: errors,
+            fileContent: fileContent,
+            scenaioName: testCase.name,
         });
         promptSpan.end({ output: { instruction } });
-        const firstShotMessage = await (0, llm_1.getLLMResult)({
+        const message = await (0, llm_1.getLLMResult)({
             messages: instruction,
             trace,
             model: options.model || constants_1.DEFAULT_MODEL,
@@ -65,88 +119,25 @@ async function generateTest(scenarios, file, isUpdate, options) {
                 ...options.modelParameters,
             },
         });
-        let response = firstShotMessage?.content || "";
-        logger.success("Test generated successfully!");
+        response = message?.content || "";
         const readWriteFileSpan = trace.startSpan("write-to-file");
-        let contents = fs_extra_1.default.readFileSync(file, "utf-8");
-        const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response);
-        let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
-        if (isUpdate) {
-            const testBlock = (0, web_1.getTypescriptTestBlock)(scenario?.name, contents);
-            contents = contents.replace(testBlock, `\n\n${strippedContent}`);
-            updatedContent = prependContent + contents;
-        }
-        await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
-        readWriteFileSpan.end({ output: { updatedContent } });
-        logger.log("Linting generated code...");
+        await fs_extra_1.default.writeFile(file, response, "utf-8");
+        readWriteFileSpan.end({ output: { response } });
         trace.event({ name: "lint-file" });
         await (0, web_1.lintErrors)(file);
         const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
-        logger.log("Validating types...");
-        let errors = (0, web_1.validateTypescript)(file);
+        errors = (0, web_1.validateTypescript)(file);
         validateTypesSpan.end({ output: { errors } });
         if (!errors.length) {
             logger.success("Found no type issues!");
         }
-        const maxIteration = 2;
-        let counter = 0;
-        while (errors.length > 0) {
-            const fileContent = fs_extra_1.default.readFileSync(file, "utf-8");
-            counter += 1;
-            if (counter > maxIteration) {
-                trace.event({ name: "code-fix-iteration-max-out" });
-                logger.error([
-                    `Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors.`,
-                    `Run the test-gen command again, once errors are fixed.`,
-                    `Trace: ${trace.url}`,
-                ].join("\n"));
-                break;
-            }
-            trace.event({ name: "Found errors fixing" });
-            logger.warn("Found few errors while validating types:");
-            errors.forEach((e) => logger.warn(e));
-            logger.log("Trying to fix above errors...");
-            const promptSpan = trace.startSpan("fix-type-errors-prompt");
-            const instruction = await (0, llm_1.getPrompt)("fix-file-errors-ts", {
-                testFiles: codePrompt || "",
-                pageFiles: pomPrompt || "",
-                scenarioFile: file,
-                errors: errors,
-                fileContent: fileContent,
-                scenaioName: scenario.name,
-            });
-            promptSpan.end({ output: { instruction } });
-            const message = await (0, llm_1.getLLMResult)({
-                messages: instruction,
-                trace,
-                model: options.model || constants_1.DEFAULT_MODEL,
-                provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-                providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
-                modelParameters: {
-                    ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                    ...options.modelParameters,
-                },
-            });
-            response = message?.content || "";
-            const readWriteFileSpan = trace.startSpan("write-to-file");
-            await fs_extra_1.default.writeFile(file, response, "utf-8");
-            readWriteFileSpan.end({ output: { response } });
-            trace.event({ name: "lint-file" });
-            await (0, web_1.lintErrors)(file);
-            const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
-            errors = (0, web_1.validateTypescript)(file);
-            validateTypesSpan.end({ output: { errors } });
-            if (!errors.length) {
-                logger.success("Found no type issues!");
-            }
-        }
-        trace.event({ name: "format-file" });
-        await (0, web_1.formatCode)(file);
-        logger.success("File formatted successfully!");
-        logger.log(`Trace: ${trace.url}`);
-        generatedScenarios.push(scenario);
-        trace.update({ input: { scenario }, output: { response } });
     }
-    return generatedScenarios;
+    trace.event({ name: "format-file" });
+    await (0, web_1.formatCode)(file);
+    logger.success("File formatted successfully!");
+    logger.log(`Trace: ${trace.url}`);
+    generatedTestCases.push(testCase);
+    trace.update({ input: { testCase }, output: { response } });
+    return generatedTestCases;
 }
 exports.generateTest = generateTest;

package/dist/bin/index.js

@@ -19,40 +19,33 @@ dotenv_1.default.config({
 process.on("exit", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());
-async function runAgent(sourceFile, isUpdate, testGenConfigs) {
+async function runAgent(sourceFile, testGenConfig) {
     const logger = new logger_1.CustomLogger();
-    const generatedTestScenarios = [];
-    const compatibleWithBrowsing = !sourceFile.startsWith("https://docs.google.com/spreadsheets");
-    for (const testGenConfig of testGenConfigs) {
-        const { specPath, scenarios } = testGenConfig;
-        if (compatibleWithBrowsing && testGenConfig.options?.agent !== "code") {
-            // this assumes we have only one scenario in test config
-            logger.success("Generating test using browsing agent");
-            await (0, utils_1.prepareFileForBrowsingAgent)(testGenConfig);
-            await (0, run_1.generateTestsUsingBrowsingAgent)(specPath);
-            await (0, reporter_1.reportTestGenVideos)({
-                projectRepoName: testGenConfig.options.metadata.projectRepoName,
-            });
-            generatedTestScenarios.push(...testGenConfig.scenarios);
-        }
-        else {
-            logger.success("Generating test using coding agent");
-            const gen = await (0, run_2.generateTest)(scenarios, specPath, isUpdate, testGenConfig.options);
-            generatedTestScenarios.push(...gen);
-        }
+    const { specPath, testCase } = testGenConfig;
+    if (testGenConfig.options?.agent !== "code") {
+        // this assumes we have only one scenario in test config
+        logger.success("Generating test using browsing agent");
+        await (0, utils_1.prepareFileForBrowsingAgent)(testGenConfig);
+        await (0, run_1.generateTestsUsingBrowsingAgent)(specPath);
+        await (0, reporter_1.reportTestGenVideos)({
+            projectRepoName: testGenConfig.options.metadata.projectRepoName,
+        });
+    }
+    else {
+        logger.success("Generating test using coding agent");
+        await (0, run_2.generateTest)(testCase, specPath, testGenConfig.options);
     }
-    return generatedTestScenarios;
 }
 (async function main() {
     const logger = new logger_1.CustomLogger({ useReporter: false });
     if (process.argv.length < 3) {
-        logger.error("Please provide path to scenarios using command:", "npx @empiricalrun/test-gen <SCENARIOS_FILE_PATH> -u");
+        logger.error("Please provide path to scenarios using command:", "npx @empiricalrun/test-gen <TEST_GEN_TOKEN>");
         process.exit(1);
     }
-    const { sourceFile, testGenConfigs, isUpdate } = await (0, utils_2.parseCliArgs)();
-    (0, reporter_1.setReporterConfig)(testGenConfigs[0]?.options?.metadata);
-    const generated = await runAgent(sourceFile, isUpdate, testGenConfigs);
+    const { sourceFile, testGenConfig } = await (0, utils_2.parseCliArgs)();
+    (0, reporter_1.setReporterConfig)(testGenConfig?.options?.metadata);
+    await runAgent(sourceFile, testGenConfig);
     // TODO: move these reporters to a better lifecycle
-    await (0, ci_1.reportOnCI)(generated);
+    await (0, ci_1.reportOnCI)(testGenConfig.testCase);
     process.exit(0);
 })();

package/dist/bin/utils/index.d.ts

@@ -1,8 +1,7 @@
 import { TestGenConfig } from "../../types";
 export declare function parseCliArgs(scenarioOrScenariosPath?: string): Promise<{
     sourceFile: string;
-    testGenConfigs: TestGenConfig[];
-    isUpdate: boolean;
+    testGenConfig: TestGenConfig;
 }>;
 export declare function getTestConfigCliArg(): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,wBAAsB,YAAY,CAChC,uBAAuB,GAAE,MAA8B;;;;GAWxD;AAED,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,wBAAsB,YAAY,CAChC,uBAAuB,GAAE,MAA8B;;;GASxD;AAED,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C"}

package/dist/bin/utils/index.js

@@ -3,12 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getTestConfigCliArg = exports.parseCliArgs = void 0;
 const scenarios_1 = require("./scenarios");
 async function parseCliArgs(scenarioOrScenariosPath = getTestConfigCliArg()) {
-    const isUpdate = process.argv.includes("-u");
-    const testGenConfigs = await (0, scenarios_1.loadTestConfigs)(scenarioOrScenariosPath);
+    const testGenConfig = await (0, scenarios_1.loadTestConfigs)(scenarioOrScenariosPath);
     return {
         sourceFile: scenarioOrScenariosPath,
-        testGenConfigs,
-        isUpdate,
+        testGenConfig,
     };
 }
 exports.parseCliArgs = parseCliArgs;

package/dist/bin/utils/scenarios/index.d.ts

@@ -1,4 +1,4 @@
 import { TestGenConfig } from "../../../types";
-declare function loadTestConfigs(scenariosPath: string): Promise<TestGenConfig[]>;
+declare function loadTestConfigs(scenariosPath: string): Promise<TestGenConfig>;
 export { loadTestConfigs };
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/scenarios/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/scenarios/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAY,aAAa,EAAwB,MAAM,gBAAgB,CAAC;AAkF/E,iBAAe,eAAe,CAC5B,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,aAAa,EAAE,CAAC,CAiC1B;AAED,OAAO,EAAE,eAAe,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/scenarios/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAwB,MAAM,gBAAgB,CAAC;AAUrE,iBAAe,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAY5E;AAED,OAAO,EAAE,eAAe,EAAE,CAAC"}

package/dist/bin/utils/scenarios/index.js

@@ -1,112 +1,17 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadTestConfigs = void 0;
-const google_auth_library_1 = require("google-auth-library");
-const slugify_1 = __importDefault(require("slugify"));
-function isValidJSON(str) {
-    try {
-        JSON.parse(str);
-        return true;
-    }
-    catch (e) {
-        return false;
-    }
-}
-/**
- * Method to update / add scenarios to the repo.
- * @param path
- * @returns updated paths of scenarios
- */
-async function loadScenariosFromGsheet(path) {
-    const { GoogleSpreadsheet } = await import("google-spreadsheet");
-    const url = new URL(path);
-    const docId = url.pathname.split("/")[3];
-    const searchParams = new URLSearchParams(url.hash.split("#")[1]);
-    const sheetId = Number(searchParams.get("gid")) || 0;
-    // TODO: use oauth 2
-    const serviceAccountAuth = new google_auth_library_1.JWT({
-        email: process.env.GOOGLE_SERVICE_EMAIL,
-        key: Buffer.from(process.env.GOOGLE_SERVICE_EMAIL_PRIVATE_KEY, "base64").toString(),
-        scopes: ["https://www.googleapis.com/auth/spreadsheets"],
-    });
-    const doc = new GoogleSpreadsheet(docId, serviceAccountAuth);
-    await doc.loadInfo();
-    const sheet = doc.sheetsById[sheetId];
-    const rows = await sheet.getRows();
-    const map = new Map();
-    rows.forEach((r) => {
-        // TODO: fix for case insensitive
-        const category = r.get("Category");
-        const name = r.get("Scenario");
-        const steps = r
-            .get("Steps")
-            .split("\n")
-            .map((s) => s.trim())
-            .filter((s) => !!s.length);
-        const assert = r.get("Assert");
-        const specPath = category
-            ? `./tests/${category}.spec.ts`
-            : `./tests/${(0, slugify_1.default)(name)}.spec.ts`;
-        const scenario = {
-            steps,
-            name,
-            assert,
-        };
-        if (!map.get(specPath)) {
-            map.set(specPath, [scenario]);
-        }
-        else {
-            const scenarios = map.get(specPath);
-            scenarios.push(scenario);
-            map.set(specPath, scenarios);
-        }
-    });
-    const results = [];
-    for (const [specPath, scenarios] of map.entries()) {
-        results.push({
-            specPath,
-            scenarios,
-        });
-    }
-    return results;
-}
 async function loadTestConfigs(scenariosPath) {
-    // google sheets generation flow
-    // TODO: remove this whenever we are comfortable demoing using dashboard
-    if (scenariosPath.startsWith("https://docs.google.com/spreadsheets")) {
-        return await loadScenariosFromGsheet(scenariosPath);
-        // dev testing flow
-    }
-    else if (scenariosPath.endsWith(".ts")) {
-        return [
-            {
-                specPath: scenariosPath,
-                scenarios: [],
-            },
-        ];
-        // api flow
-    }
-    else if (isValidJSON(atob(scenariosPath))) {
-        const str = atob(scenariosPath);
-        const config = JSON.parse(str);
-        const specPath = `./tests/${config.group || "index"}.spec.ts`;
-        return [
-            {
-                specPath,
-                scenarios: [
-                    {
-                        name: config.name,
-                        steps: config.steps.filter((s) => !!s),
-                        assert: config.assert,
-                    },
-                ],
-                options: config.options,
-            },
-        ];
-    }
-    throw Error("Invalid path for test scenarios");
+    const str = atob(scenariosPath);
+    const config = JSON.parse(str);
+    const specPath = `./tests/${config.group || "index"}.spec.ts`;
+    return {
+        specPath,
+        testCase: {
+            name: config.name,
+            steps: config.steps.filter((s) => !!s),
+        },
+        options: config.options,
+    };
 }
 exports.loadTestConfigs = loadTestConfigs;

package/dist/file/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAOA,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,IAAI,CAAa;gBACb,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE;IAGtC,WAAW,CAAC,QAAQ,EAAE,MAAM;IAGtB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAmC1C;AAED,wBAAsB,gBAAgB,kBAAK"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAMA,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,IAAI,CAAa;gBACb,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE;IAGtC,WAAW,CAAC,QAAQ,EAAE,MAAM;IAGtB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAkC1C;AAED,wBAAsB,gBAAgB,kBAAK"}

package/dist/file/server.js

@@ -8,7 +8,6 @@ const express_1 = __importDefault(require("express"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const web_1 = require("../bin/utils/platform/web");
-const string_1 = require("../utils/string");
 class FileService {
     filePath = "";
     port = 0;
@@ -21,17 +20,16 @@ class FileService {
     async startFileService() {
         const app = (0, express_1.default)();
         app.use(express_1.default.json());
-        app.post("/test", (req, res) => {
-            const { generatedCode, task } = req.body;
+        app.post("/test", async (req, res) => {
+            const { generatedCode } = req.body;
             try {
                 const testFilePath = path_1.default.resolve(process.cwd(), this.filePath);
                 if (testFilePath) {
                     const testFile = fs_1.default.readFileSync(testFilePath, "utf-8");
-                    const jsComments = (0, string_1.convertTextToJsComments)(task);
-                    const updatedTestFile = testFile.replace(/await createTest\([\s\S]*?\);\n/, jsComments + "\n" + generatedCode);
+                    const updatedTestFile = testFile.replace(/await createTest\([\s\S]*?\);\n/, "\n" + generatedCode);
                     const importStatement = `import { test, expect } from "@playwright/test";`;
                     fs_1.default.writeFileSync(testFilePath, importStatement + "\n" + updatedTestFile, "utf-8");
-                    (0, web_1.lintErrors)(testFilePath);
+                    await (0, web_1.lintErrors)(testFilePath);
                     return res.send({ success: true });
                 }
             }

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBAkBnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBAiBnE"}

package/dist/index.js

@@ -15,8 +15,7 @@ process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());
 async function createTest(task, page, test) {
     const port = process.env.APP_PORT || 3030;
     const testConfigArg = process.env.TEST_GEN_TOKEN;
-    const { testGenConfigs } = await (0, utils_1.parseCliArgs)(testConfigArg);
-    const [testGenConfig] = testGenConfigs;
+    const { testGenConfig } = await (0, utils_1.parseCliArgs)(testConfigArg);
     (0, reporter_1.setReporterConfig)(testGenConfig.options?.metadata);
     const fileService = new client_1.default(Number(port));
     test.setTimeout(900000);

package/dist/reporter/ci.d.ts

@@ -1,3 +1,3 @@
-import { Scenario } from "../types";
-export declare function reportOnCI(scenarios: Scenario[]): Promise<Scenario[]>;
+import { TestCase } from "../types";
+export declare function reportOnCI(testCase: TestCase): Promise<TestCase>;
 //# sourceMappingURL=ci.d.ts.map

package/dist/reporter/ci.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC,wBAAsB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,uBAYrD"}
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC,wBAAsB,UAAU,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAOtE"}

package/dist/reporter/ci.js

@@ -25,17 +25,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.reportOnCI = void 0;
 const core = __importStar(require("@actions/core"));
-async function reportOnCI(scenarios) {
+async function reportOnCI(testCase) {
     if ("true") {
-        const testNames = scenarios.map((s) => s.name).join(" ");
-        const scenariosOutput = scenarios
-            .map((s) => {
-            return `**Scenario:** ${s.name} \n\n**Steps:**\n  - ${s.steps.join("\n  - ")}`;
-        })
-            .join("\n ----- \n");
+        const scenariosOutput = `**Scenario:** ${testCase.name} \n\n**Steps:**\n  - ${testCase.steps.join("\n  - ")}`;
         core.setOutput("summary", scenariosOutput);
-        core.setOutput("test_names", testNames);
+        core.setOutput("test_names", testCase.name);
     }
-    return scenarios;
+    return testCase;
 }
 exports.reportOnCI = reportOnCI;

package/dist/types/index.d.ts

@@ -20,13 +20,12 @@ export type TestGenConfigOptions = {
 };
 export type TestGenConfig = {
     specPath: string;
-    scenarios: Scenario[];
+    testCase: TestCase;
     options?: TestGenConfigOptions;
 };
-export type Scenario = {
+export type TestCase = {
     name: string;
     steps: string[];
-    assert: string;
 };
 export type PlaywrightActionGenerator = (page: Page) => Action;
 export type ActionSchema = OpenAI.Chat.Completions.ChatCompletionTool;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.16.13",
+  "version": "0.17.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"