@empiricalrun/test-gen 0.9.0 → 0.10.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @empiricalrun/test-gen
 
+## 0.10.0
+
+### Minor Changes
+
+- e29b407: feat: start with browsing agent
+
+### Patch Changes
+
+- 057a0cb: feat: action for page navigation
+
 ## 0.9.0
 
 ### Minor Changes

package/README.md

@@ -1,14 +1,44 @@
 # test-gen
 
-Package to generate e2e Playwright tests.
+Our agents that generate Playwright tests. There are 2 agents
 
-## Usages
+- browsing agent
+- code editing agent
 
+## Usage
+
+```sh
+npx @empiricalrun/test-gen $args
+```
+
+where `$args` can be one of the following
+
+### Add new test
+
+```sh
+npx @empiricalrun/test-gen base64string
 ```
-npx @empiricalrun/test-gen ./scenarios/create-flow.yaml
+
+- This will trigger browsing agent to write a new test for this scenario
+- See [Base64 payload](#base64-payload)
+
+```sh
+npx @empiricalrun/test-gen delete.spec.ts
+```
+
+- This will trigger browsing agent to write a new test in this file
+- **Note** that the file must have `createTest(...)` where the scenario is specified
+
+#### Bulk add
+
+```sh
+npx @empiricalrun/test-gen scenarios.yml
+
+# optional: filter for one scenario
+npx @empiricalrun/test-gen scenarios.yml $scenario_name
 ```
 
-## Sample test scenarios
+Using yml or google spreadsheet. **Note** that this only runs the code editing agent
 
 ```yaml ./tests/scenarios.yaml
 dir: ./tests/ # [optional] directory where you want to create tests. Default - ./tests
@@ -22,3 +52,30 @@ scenarios: # list of scenarios
         - click on "Login Button"
       assert: "Login successful" text should be visible # assertion
 ```
+
+### Update existing test
+
+```sh
+npx @empiricalrun/test-gen base64string -u
+```
+
+- This will trigger code editing agent to write a new test for this scenario
+- See [Base64 payload](#base64-payload)
+
+### Base64 payload
+
+Convert this object into a base64 payload - for new and update test.
+
+```js
+const scenario = {
+  name: "incorrect password flow",
+  steps: [
+    "fill in the username as foobar@gmail.com",
+    "fill in the password as bazbaz",
+    "click on the login button"
+  ],
+  assert: "",
+  group: "home"
+};
+btoa(JSON.stringify(scenario))
+```

package/dist/actions/goto.d.ts

@@ -0,0 +1,4 @@
+import { PlaywrightActionGenerator } from "../types";
+export declare const PLAYWRIGHT_GOTO_ACTION_NAME = "page_goto";
+export declare const gotoActionGenerator: PlaywrightActionGenerator;
+//# sourceMappingURL=goto.d.ts.map

package/dist/actions/goto.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAoCjC,CAAC"}

package/dist/actions/goto.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.gotoActionGenerator = exports.PLAYWRIGHT_GOTO_ACTION_NAME = void 0;
+exports.PLAYWRIGHT_GOTO_ACTION_NAME = "page_goto";
+const gotoActionGenerator = (page) => {
+    return {
+        execute: async (args) => {
+            const url = args.url;
+            await page.goto(url);
+            await page.waitForTimeout(3000);
+        },
+        // TODO: args transformer to be kept at a single place
+        template: (args) => {
+            const url = args.url;
+            const templ = `await page.goto("${url}");`;
+            return templ;
+        },
+        name: exports.PLAYWRIGHT_GOTO_ACTION_NAME,
+        schema: {
+            type: "function",
+            function: {
+                name: exports.PLAYWRIGHT_GOTO_ACTION_NAME,
+                description: "navigate the page to a new URL",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        url: {
+                            type: "string",
+                            description: "URL to navigate page to",
+                        },
+                        reason: {
+                            type: "string",
+                            description: "reason for calling this function",
+                        },
+                    },
+                    required: ["url", "reason"],
+                },
+            },
+        },
+    };
+};
+exports.gotoActionGenerator = gotoActionGenerator;

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAU,YAAY,EAAE,MAAM,UAAU,CAAC;AAMhD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAmC;gBAC9C,IAAI,EAAE,IAAI;IAUhB,aAAa,CAAC,IAAI,oBAAa,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAkBhE,gBAAgB,IAAI,YAAY,EAAE;IAIlC,YAAY;IAIZ,UAAU;CAMX"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAU,YAAY,EAAE,MAAM,UAAU,CAAC;AAOhD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAmC;gBAC9C,IAAI,EAAE,IAAI;IAWhB,aAAa,CAAC,IAAI,oBAAa,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAkBhE,gBAAgB,IAAI,YAAY,EAAE;IAIlC,YAAY;IAIZ,UAAU;CAMX"}

package/dist/actions/index.js

@@ -5,12 +5,14 @@ const assertTextVisibility_1 = require("./assertTextVisibility");
 const click_1 = require("./click");
 const done_1 = require("./done");
 const fill_1 = require("./fill");
+const goto_1 = require("./goto");
 class PlaywrightActions {
     actions;
     recordedActions;
     constructor(page) {
         this.actions = [
             (0, fill_1.fillActionGenerator)(page),
+            (0, goto_1.gotoActionGenerator)(page),
             (0, click_1.clickActionGenerator)(page),
             (0, done_1.doneActionGenerator)(page),
             (0, assertTextVisibility_1.assertTextVisibilityActionGenerator)(page),

package/dist/agent/browsing/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,GAAE;IACP,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACE,mBA4DP"}

package/dist/agent/{index.js → browsing/index.js}

@@ -4,11 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.browsingAgent = void 0;
-const actions_1 = require("../actions");
-const ai_1 = require("../bin/ai");
-const provider_1 = require("../bin/ai/prompts/provider");
-const trace_1 = __importDefault(require("../bin/ai/trace"));
-const html_1 = require("../utils/html");
+const actions_1 = require("../../actions");
+const ai_1 = require("../../bin/ai");
+const provider_1 = require("../../bin/ai/prompts/provider");
+const trace_1 = __importDefault(require("../../bin/ai/trace"));
+const logger_1 = require("../../bin/logger");
+const html_1 = require("../../utils/html");
+const logger = new logger_1.CustomLogger();
 async function browsingAgent(task, page, options = {}) {
     const trace = new trace_1.default({ name: "browsing-agent" });
     const actions = new actions_1.PlaywrightActions(page);
@@ -54,14 +56,15 @@ async function browsingAgent(task, page, options = {}) {
             catch (e) {
                 // TODO: implement feedback loop to llm
                 lastActionErrors.push(e.message);
-                console.error(e);
+                logger.error(e);
             }
         }
         isTaskDone = actions.isComplete();
     }
     const code = actions.generateCode();
     trace.update({ input: { task }, output: { code } });
-    console.log("Successfully generated code for the given task", "[Trace:", trace.url, "]");
+    logger.success("Successfully generated code for the given task");
+    logger.log("Trace:", trace.url);
     return code;
 }
 exports.browsingAgent = browsingAgent;

package/dist/agent/browsing/run.d.ts

@@ -0,0 +1,2 @@
+export declare function generateTestsUsingBrowsingAgent(testFilePath: string): Promise<void>;
+//# sourceMappingURL=run.d.ts.map

package/dist/agent/browsing/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAOA,wBAAsB,+BAA+B,CAAC,YAAY,EAAE,MAAM,iBAwBzE"}

package/dist/agent/browsing/run.js

@@ -0,0 +1,38 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateTestsUsingBrowsingAgent = void 0;
+const detect_port_1 = __importDefault(require("detect-port"));
+const web_1 = require("../../bin/utils/platform/web");
+const server_1 = require("../../file/server");
+const exec_1 = require("../../utils/exec");
+const utils_1 = require("./utils");
+async function generateTestsUsingBrowsingAgent(testFilePath) {
+    (0, utils_1.canRunBrowsingAgent)(testFilePath);
+    const port = await (0, detect_port_1.default)(3030);
+    const fileService = new server_1.FileService({ port });
+    await fileService.startFileService();
+    try {
+        fileService.setFilePath(testFilePath);
+        //TODO: change this to per test
+        //TODO: support multiple projects (browsers)
+        const command = [`npx`, `playwright`, `test`, testFilePath];
+        if (!process.env.CI) {
+            command.push(`--headed`);
+        }
+        await (0, exec_1.cmd)(command, {
+            env: {
+                APP_PORT: port.toString(),
+                PW_TEST_HTML_REPORT_OPEN: "never",
+            },
+        });
+        await (0, web_1.removeTestOnly)(testFilePath);
+    }
+    catch (e) {
+        // TODO: better error handling -- e.g. undo preparation work
+        console.error(e);
+    }
+}
+exports.generateTestsUsingBrowsingAgent = generateTestsUsingBrowsingAgent;

package/dist/agent/browsing/utils.d.ts

@@ -0,0 +1,4 @@
+import { TestGenConfig } from "../../types";
+export declare function prepareFileForBrowsingAgent(genConfig: TestGenConfig): Promise<void>;
+export declare function canRunBrowsingAgent(filePath: string): void;
+//# sourceMappingURL=utils.d.ts.map

package/dist/agent/browsing/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAiBzE;AAWD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD"}

package/dist/agent/browsing/utils.js

@@ -0,0 +1,52 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.canRunBrowsingAgent = exports.prepareFileForBrowsingAgent = void 0;
+const child_process_1 = require("child_process");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const logger_1 = require("../../bin/logger");
+const web_1 = require("../../bin/utils/platform/web");
+async function prepareFileForBrowsingAgent(genConfig) {
+    const { specPath, scenarios } = genConfig;
+    const { name, steps, assert } = scenarios[0];
+    const mergedSteps = `${steps.join("\n")}\n${assert}`;
+    if (!fs_extra_1.default.existsSync(specPath)) {
+        await fs_extra_1.default.writeFile(specPath, (0, web_1.addNewImport)("", ["test", "expect"], "@playwright/test"));
+    }
+    // TODO: this assumes that test code repo has `page` as the main entrypoint fixture
+    const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
+    const newContents = `${existingContents}\n\n${createTestGenBlock(name, mergedSteps)}`;
+    await fs_extra_1.default.writeFile(specPath, (0, web_1.addNewImport)(newContents, ["createTest"], "@empiricalrun/test-gen"));
+}
+exports.prepareFileForBrowsingAgent = prepareFileForBrowsingAgent;
+function createTestGenBlock(name, steps) {
+    return `
+test.only("${name}", async ({page}) => {
+  await createTest(\`
+${steps}\`, page, test);
+});
+  `;
+}
+function canRunBrowsingAgent(filePath) {
+    if (!fs_extra_1.default.existsSync(filePath)) {
+        throw new Error(`File for browsing agent to run not found: ${filePath}`);
+    }
+    const { dependencies: installedPackages } = JSON.parse((0, child_process_1.execSync)("npm ls --json").toString());
+    const hasPwTestInstalled = installedPackages["@playwright/test"];
+    const testGenInstalled = installedPackages["@empiricalrun/test-gen"];
+    const linterInstalled = installedPackages["eslint"];
+    const testGenVersion = testGenInstalled.version;
+    const latestTestGenVersion = (0, child_process_1.execSync)("npm view @empiricalrun/test-gen version")
+        .toString()
+        .trim();
+    if (!hasPwTestInstalled || !testGenInstalled || !linterInstalled) {
+        throw new Error(`Missing packages: browsing agent requires playwright, linter, and @empiricalrun/test-gen to be installed`);
+    }
+    if (testGenVersion !== latestTestGenVersion) {
+        const logger = new logger_1.CustomLogger();
+        logger.warn(`Outdated @empiricalrun/test-gen package: expected ${latestTestGenVersion}, got ${testGenVersion}`);
+    }
+}
+exports.canRunBrowsingAgent = canRunBrowsingAgent;

package/dist/agent/codegen/run.d.ts

@@ -0,0 +1,3 @@
+import { Scenario } from "../../types";
+export declare function generateTest(scenarios: Scenario[], file: string, isUpdate: boolean): Promise<Scenario[]>;
+//# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAIvC,wBAAsB,YAAY,CAChC,SAAS,EAAE,QAAQ,EAAE,EACrB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC,QAAQ,EAAE,CAAC,CA0HrB"}

package/dist/agent/codegen/run.js

@@ -0,0 +1,127 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateTest = void 0;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const ai_1 = require("../../bin/ai");
+const provider_1 = require("../../bin/ai/prompts/provider");
+const trace_1 = __importDefault(require("../../bin/ai/trace"));
+const logger_1 = require("../../bin/logger");
+const context_1 = require("../../bin/utils/context");
+const web_1 = require("../../bin/utils/platform/web");
+const logger = new logger_1.CustomLogger();
+async function generateTest(scenarios, file, isUpdate) {
+    const context = await (0, context_1.contextForGeneration)(file);
+    const { codePrompt, pomPrompt, testFileContent } = context;
+    const generatedScenarios = [];
+    for (const i in scenarios) {
+        logger.logEmptyLine();
+        const trace = new trace_1.default({ name: "generate-test" });
+        trace.event({
+            name: "collate-files-as-text",
+            output: {
+                codePrompt,
+                pomPrompt,
+                testFileContent,
+            },
+        });
+        const scenario = scenarios[i];
+        trace.update({ input: { scenario } });
+        logger.log("Generating test for scenario:", scenario?.name);
+        if (!isUpdate && testFileContent.includes(`test("${scenario?.name}"`)) {
+            logger.success("Test already exists for this scenario");
+            trace.event({ name: "test-already-exists" });
+            continue;
+        }
+        const promptSpan = trace.startSpan(isUpdate ? "update-scenario-prompt" : "add-scenario-prompt");
+        const promptName = isUpdate ? "update-scenario" : "add-scenario";
+        const instruction = await (0, provider_1.getPrompt)(promptName, {
+            testFiles: codePrompt,
+            pageFiles: pomPrompt,
+            scenarioName: scenario.name,
+            scenarioSteps: scenario.steps.join("\n"),
+            assert: scenario.assert,
+            scenarioFile: file,
+        });
+        promptSpan.end({ output: { instruction } });
+        const firstShotMessage = await (0, ai_1.getLLMResult)({
+            messages: instruction,
+            trace,
+        });
+        let response = firstShotMessage?.content || "";
+        logger.success("Test generated successfully!");
+        const readWriteFileSpan = trace.startSpan("write-to-file");
+        let contents = fs_extra_1.default.readFileSync(file, "utf-8");
+        const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response);
+        let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
+        if (isUpdate) {
+            const testBlock = (0, web_1.getTypescriptTestBlock)(scenario?.name, contents);
+            contents = contents.replace(testBlock[0], `\n\n${strippedContent}`);
+            updatedContent = prependContent + contents;
+        }
+        await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
+        readWriteFileSpan.end({ output: { updatedContent } });
+        logger.log("Linting generated code...");
+        trace.event({ name: "lint-file" });
+        await (0, web_1.lintErrors)(file);
+        const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
+        logger.log("Validating types...");
+        let errors = (0, web_1.validateTypescript)(file);
+        validateTypesSpan.end({ output: { errors } });
+        if (!errors.length) {
+            logger.success("Found no type issues!");
+        }
+        const maxIteration = 2;
+        let counter = 0;
+        while (errors.length > 0) {
+            const fileContent = fs_extra_1.default.readFileSync(file, "utf-8");
+            counter += 1;
+            if (counter > maxIteration) {
+                trace.event({ name: "code-fix-iteration-max-out" });
+                logger.error([
+                    `Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors.`,
+                    `Run the test-gen command again, once errors are fixed.`,
+                    `Trace: ${trace.url}`,
+                ].join("\n"));
+                break;
+            }
+            trace.event({ name: "Found errors fixing" });
+            logger.warn("Found few errors while validating types:");
+            errors.forEach((e) => logger.warn(e));
+            logger.log("Trying to fix above errors...");
+            const promptSpan = trace.startSpan("fix-type-errors-prompt");
+            const instruction = await (0, provider_1.getPrompt)("fix-file-errors-ts", {
+                testFiles: codePrompt || "",
+                pageFiles: pomPrompt || "",
+                scenarioFile: file,
+                errors: errors,
+                fileContent: fileContent,
+                scenaioName: scenario.name,
+            });
+            promptSpan.end({ output: { instruction } });
+            const message = await (0, ai_1.getLLMResult)({ messages: instruction, trace });
+            response = message?.content || "";
+            const readWriteFileSpan = trace.startSpan("write-to-file");
+            await fs_extra_1.default.writeFile(file, response, "utf-8");
+            readWriteFileSpan.end({ output: { response } });
+            trace.event({ name: "lint-file" });
+            await (0, web_1.lintErrors)(file);
+            const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
+            errors = (0, web_1.validateTypescript)(file);
+            validateTypesSpan.end({ output: { errors } });
+            if (!errors.length) {
+                logger.success("Found no type issues!");
+            }
+        }
+        trace.event({ name: "format-file" });
+        await (0, web_1.formatCode)(file);
+        logger.success("File formatted successfully!");
+        logger.log(`Trace: ${trace.url}`);
+        generatedScenarios.push(scenario);
+        trace.update({ input: { scenario }, output: { response } });
+    }
+    return generatedScenarios;
+}
+exports.generateTest = generateTest;

package/dist/bin/index.js

@@ -4,195 +4,84 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const detect_port_1 = __importDefault(require("detect-port"));
 const dotenv_1 = __importDefault(require("dotenv"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
-const server_1 = require("../file/server");
-const exec_1 = require("../utils/exec");
-const ai_1 = require("./ai");
-const provider_1 = require("./ai/prompts/provider");
-const trace_1 = __importDefault(require("./ai/trace"));
+const run_1 = require("../agent/browsing/run");
+const utils_1 = require("../agent/browsing/utils");
+const run_2 = require("../agent/codegen/run");
 const logger_1 = require("./logger");
 const ci_1 = require("./reporter/ci");
 const scenarios_1 = require("./scenarios");
-const context_1 = require("./utils/context");
-const web_1 = require("./utils/platform/web");
 dotenv_1.default.config({
     path: [".env.local", ".env"],
 });
 const logger = new logger_1.CustomLogger();
-async function generateTest(scenarios, file, isUpdate) {
-    const context = await (0, context_1.contextForGeneration)(file);
-    const { codePrompt, pomPrompt, testFileContent } = context;
-    const generatedScenarios = [];
-    for (const i in scenarios) {
-        logger.logEmptyLine();
-        const trace = new trace_1.default({ name: "generate-test" });
-        trace.event({
-            name: "collate-files-as-text",
-            output: {
-                codePrompt,
-                pomPrompt,
-                testFileContent,
-            },
-        });
-        const scenario = scenarios[i];
-        trace.update({ input: { scenario } });
-        logger.log("Generating test for scenario:", scenario?.name);
-        if (!isUpdate && testFileContent.includes(`test("${scenario?.name}"`)) {
-            logger.success("Test already exists for this scenario");
-            trace.event({ name: "test-already-exists" });
-            continue;
+async function runAgent(sourceFile, isUpdate, testGenConfigs) {
+    const compatibleWithBrowsing = !sourceFile.endsWith(".yaml") &&
+        !sourceFile.startsWith("https://docs.google.com/spreadsheets") &&
+        !isUpdate;
+    if (compatibleWithBrowsing) {
+        const testGenConfig = testGenConfigs[0];
+        let fileForBrowsingAgent;
+        if (sourceFile.endsWith(".ts")) {
+            fileForBrowsingAgent = sourceFile;
         }
-        const promptSpan = trace.startSpan(isUpdate ? "update-scenario-prompt" : "add-scenario-prompt");
-        const promptName = isUpdate ? "update-scenario" : "add-scenario";
-        const instruction = await (0, provider_1.getPrompt)(promptName, {
-            testFiles: codePrompt,
-            pageFiles: pomPrompt,
-            scenarioName: scenario.name,
-            scenarioSteps: scenario.steps.join("\n"),
-            assert: scenario.assert,
-            scenarioFile: file,
-        });
-        promptSpan.end({ output: { instruction } });
-        const firstShotMessage = await (0, ai_1.getLLMResult)({
-            messages: instruction,
-            trace,
-        });
-        let response = firstShotMessage?.content || "";
-        logger.success("Test generated successfully!");
-        const readWriteFileSpan = trace.startSpan("write-to-file");
-        let contents = fs_extra_1.default.readFileSync(file, "utf-8");
-        const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response);
-        let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
-        if (isUpdate) {
-            const testBlock = (0, web_1.getTypescriptTestBlock)(scenario?.name, contents);
-            contents = contents.replace(testBlock[0], `\n\n${strippedContent}`);
-            updatedContent = prependContent + contents;
+        else {
+            fileForBrowsingAgent = testGenConfig.specPath;
+            await (0, utils_1.prepareFileForBrowsingAgent)(testGenConfig);
         }
-        await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
-        readWriteFileSpan.end({ output: { updatedContent } });
-        logger.log("Linting generated code...");
-        trace.event({ name: "lint-file" });
-        await (0, web_1.lintErrors)(file);
-        const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
-        logger.log("Validating types...");
-        let errors = (0, web_1.validateTypescript)(file);
-        validateTypesSpan.end({ output: { errors } });
-        if (!errors.length) {
-            logger.success("Found no type issues!");
+        if (fileForBrowsingAgent) {
+            logger.success("Generating tests using browsing agent");
+            await (0, run_1.generateTestsUsingBrowsingAgent)(fileForBrowsingAgent);
+            process.exit(0);
         }
-        const maxIteration = 2;
-        let counter = 0;
-        while (errors.length > 0) {
-            const fileContent = fs_extra_1.default.readFileSync(file, "utf-8");
-            counter += 1;
-            if (counter > maxIteration) {
-                trace.event({ name: "code-fix-iteration-max-out" });
-                logger.error([
-                    `Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors.`,
-                    `Run the test-gen command again, once errors are fixed.`,
-                    `[Trace: ${trace.url}]`,
-                ].join("\n"));
-                break;
-            }
-            trace.event({ name: "Found errors fixing" });
-            logger.warn("Found few errors while validating types:");
-            errors.forEach((e) => logger.warn(e));
-            logger.log("Trying to fix above errors...");
-            const promptSpan = trace.startSpan("fix-type-errors-prompt");
-            const instruction = await (0, provider_1.getPrompt)("fix-file-errors-ts", {
-                testFiles: codePrompt || "",
-                pageFiles: pomPrompt || "",
-                scenarioFile: file,
-                errors: errors,
-                fileContent: fileContent,
-                scenaioName: scenario.name,
-            });
-            promptSpan.end({ output: { instruction } });
-            const message = await (0, ai_1.getLLMResult)({ messages: instruction, trace });
-            response = message?.content || "";
-            const readWriteFileSpan = trace.startSpan("write-to-file");
-            await fs_extra_1.default.writeFile(file, response, "utf-8");
-            readWriteFileSpan.end({ output: { response } });
-            trace.event({ name: "lint-file" });
-            await (0, web_1.lintErrors)(file);
-            const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
-            errors = (0, web_1.validateTypescript)(file);
-            validateTypesSpan.end({ output: { errors } });
-            if (!errors.length) {
-                logger.success("Found no type issues!");
-            }
+    }
+    const generatedTestScenarios = [];
+    for (const testGenConfig of testGenConfigs) {
+        const { specPath, scenarios } = testGenConfig;
+        if (!fs_extra_1.default.existsSync(specPath)) {
+            logger.log(`Creating a new spec file: ${specPath}`);
+            fs_extra_1.default.createFileSync(specPath);
         }
-        trace.event({ name: "format-file" });
-        await (0, web_1.formatCode)(file);
-        logger.success("File formatted successfully!", `[Trace: ${trace.url}]`);
-        generatedScenarios.push(scenario);
-        trace.update({ input: { scenario }, output: { response } });
+        const gen = await (0, run_2.generateTest)(scenarios, specPath, isUpdate);
+        generatedTestScenarios.push(...gen);
     }
-    return generatedScenarios;
+    return generatedTestScenarios;
 }
-async function generateTestsUsingBrowsingAgent() {
-    const testFilePath = process.argv[2];
-    if (!testFilePath) {
-        console.error("test file path is required");
-        process.exit(1);
-    }
-    const port = await (0, detect_port_1.default)(3030);
-    const fileService = new server_1.FileService({ port });
-    await fileService.startFileService();
-    try {
-        fileService.setFilePath(testFilePath);
-        //TODO: change this to per test
-        await (0, exec_1.cmd)([`npx`, `playwright`, `test`, testFilePath, "--headed"], {
-            env: {
-                APP_PORT: port.toString(),
-                PW_TEST_HTML_REPORT_OPEN: "never",
-            },
-        });
-        process.exit(0);
+function filterConfigsByScenarioName(testGenConfigs, scenarioName) {
+    if (!scenarioName) {
+        return testGenConfigs;
     }
-    catch (e) {
-        console.error(e);
+    const filteredConfigs = testGenConfigs.filter((t) => t.scenarios.some((s) => s.name === scenarioName));
+    // if no config available then throw error
+    if (testGenConfigs.length == 0) {
+        logger.error("No scenarios found for the given scenario name", scenarioName);
         process.exit(1);
     }
+    filteredConfigs.map((config) => {
+        config.scenarios = config.scenarios.filter((s) => s.name === scenarioName);
+        return config;
+    });
+    return filteredConfigs;
+}
+async function parseArgs() {
+    const scenarioOrScenariosPath = process.argv[2];
+    const isUpdate = process.argv.includes("-u");
+    const scenarioToFilter = process.argv[3] === "-u" ? "" : process.argv[3];
+    const testGenConfigs = filterConfigsByScenarioName(await (0, scenarios_1.loadScenarios)(scenarioOrScenariosPath), scenarioToFilter);
+    return {
+        sourceFile: scenarioOrScenariosPath,
+        testGenConfigs,
+        isUpdate,
+    };
 }
 (async function main() {
     if (process.argv.length < 3) {
         logger.error("Please provide path to scenarios using command:", "npx @empiricalrun/test-gen <SCENARIOS_FILE_PATH> <SCENARIO_NAME> -u");
         process.exit(1);
     }
-    const scenariosPath = process.argv[2];
-    //TODO: come up with better way to handle this
-    if (scenariosPath.endsWith(".ts")) {
-        logger.success("Generating tests using browsing agent");
-        await generateTestsUsingBrowsingAgent();
-        return;
-    }
-    const scenario = process.argv[3] === "-u" ? "" : process.argv[3];
-    const isUpdate = process.argv.includes("-u");
-    let testGenConfigs = await (0, scenarios_1.loadScenarios)(scenariosPath);
-    if (scenario) {
-        // filter config
-        testGenConfigs = testGenConfigs.filter((t) => t.scenarios.some((s) => s.name === scenario));
-        // if no config available then throw error
-        if (testGenConfigs.length == 0) {
-            logger.error("No scenarios found for the given scenario name", scenario);
-            process.exit(1);
-        }
-        const [testGenConfig] = testGenConfigs;
-        // filter scenarios
-        testGenConfig.scenarios = testGenConfig.scenarios.filter((s) => s.name === scenario);
-    }
-    const generatedTestScenarios = [];
-    for (const testGenConfig of testGenConfigs) {
-        const { specPath, scenarios } = testGenConfig;
-        if (!fs_extra_1.default.existsSync(specPath)) {
-            logger.log(`Creating a new spec file: ${specPath}`);
-            fs_extra_1.default.createFileSync(specPath);
-        }
-        const gen = await generateTest(scenarios, specPath, isUpdate);
-        generatedTestScenarios.push(...gen);
-    }
-    await (0, ci_1.reportOnCI)(generatedTestScenarios);
+    const { sourceFile, testGenConfigs, isUpdate } = await parseArgs();
+    const generated = await runAgent(sourceFile, isUpdate, testGenConfigs);
+    // TODO: make this work with browsing agent
+    await (0, ci_1.reportOnCI)(generated);
 })();

package/dist/bin/utils/platform/web/index.d.ts

@@ -3,4 +3,6 @@ export declare function validateTypescript(filePath: string): string[];
 export declare function stripAndPrependImports(content: string): Promise<string[]>;
 export declare function lintErrors(filePath: string): Promise<void>;
 export declare function formatCode(filePath: string): Promise<void>;
+export declare function addNewImport(contents: string, modules: string[], pkg: string): string;
+export declare function removeTestOnly(filePath: string): Promise<void>;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,0BAG3E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwC7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAM3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAOhD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,0BAG3E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwC7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAM3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAOhD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD"}

package/dist/bin/utils/platform/web/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.getTypescriptTestBlock = void 0;
+exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.getTypescriptTestBlock = void 0;
 const eslint_1 = require("eslint");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const prettier_1 = __importDefault(require("prettier"));
@@ -78,3 +78,13 @@ async function formatCode(filePath) {
     fs_extra_1.default.writeFileSync(filePath, formattedContent);
 }
 exports.formatCode = formatCode;
+function addNewImport(contents, modules, pkg) {
+    return `import { ${modules.join(", ")} } from "${pkg}";\n${contents}`;
+}
+exports.addNewImport = addNewImport;
+async function removeTestOnly(filePath) {
+    const contents = await fs_extra_1.default.readFile(filePath, "utf8");
+    const updatedContent = contents.replace(/test\.only/, "test");
+    return fs_extra_1.default.writeFile(filePath, updatedContent);
+}
+exports.removeTestOnly = removeTestOnly;

package/dist/index.js

@@ -4,13 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTest = void 0;
-const agent_1 = require("./agent");
+const browsing_1 = require("./agent/browsing");
 const client_1 = __importDefault(require("./file/client"));
 async function createTest(task, page, test) {
     const port = process.env.APP_PORT || 3030;
     const fileService = new client_1.default(Number(port));
     test.setTimeout(900000);
-    const code = await (0, agent_1.browsingAgent)(task, page, {
+    const code = await (0, browsing_1.browsingAgent)(task, page, {
         htmlSanitize: {
             disallowedStrings: ["v-data-table__td v-data-table-column--align-start"],
         },

package/dist/types/index.d.ts

@@ -6,7 +6,7 @@ export type FileContent = {
 };
 export type TestGenConfig = {
     dir?: string;
-    specPath?: string;
+    specPath: string;
     scenarios: Scenario[];
 };
 export type Scenario = {

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CACP,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACtB,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;IACrD,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAC1B,MAAM,CAAC;CACb,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CACP,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACtB,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;IACrD,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAC1B,MAAM,CAAC;CACb,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"

package/dist/agent/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,GAAE;IACP,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACE,mBAgEP"}