@empiricalrun/test-gen 0.25.2 → 0.27.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @empiricalrun/test-gen
 
+## 0.27.0
+
+### Minor Changes
+
+- 88067ed: fix: create test script remain in code after test gen error
+
+## 0.26.0
+
+### Minor Changes
+
+- 5882a20: feat: add inline master agent support
+
+### Patch Changes
+
+- 5882a20: fix: improve code agent accuracy with updated prompt
+
 ## 0.25.2
 
 ### Patch Changes

package/dist/actions/index.js

@@ -43,7 +43,7 @@ class PlaywrightActions {
         }
         catch (e) {
             logger.log(`action: ${name} \nreason: ${args.reason}`);
-            throw Error(`Error executing ${name} action of playwright: ${e}`);
+            throw Error(`Error executing ${name} action: ${e.message}`);
         }
     }
     getActionSchemas() {

package/dist/agent/browsing/run.d.ts

@@ -1,2 +1,26 @@
-export declare function generateTestsUsingBrowsingAgent(testFilePath: string): Promise<void>;
+type GenerateTestsType = {
+    /**
+     * Path to the test case file being updated or created
+     *
+     * @type {string}
+     */
+    testFilePath: string;
+    /**
+     * File path being updated for the concerned test case
+     *
+     * @type {string}
+     */
+    filePathToUpdate: string;
+};
+/**
+ *
+ * Function to generate tests using master agent
+ * @export
+ * @param {GenerateTestsType} {
+ *   testFilePath,
+ *   filePathToUpdate,
+ * }
+ */
+export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, }: GenerateTestsType): Promise<void>;
+export {};
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAcA,wBAAsB,+BAA+B,CAAC,YAAY,EAAE,MAAM,iBA6BzE"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,GACjB,EAAE,iBAAiB,iBAqDnB"}

package/dist/agent/browsing/run.js

@@ -3,44 +3,70 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTestsUsingBrowsingAgent = void 0;
+exports.generateTestsUsingMasterAgent = void 0;
 const detect_port_1 = __importDefault(require("detect-port"));
-const logger_1 = require("../../bin/logger");
+const fs_extra_1 = __importDefault(require("fs-extra"));
 const utils_1 = require("../../bin/utils");
 const web_1 = require("../../bin/utils/platform/web");
 const server_1 = require("../../file/server");
-const reporter_1 = require("../../reporter");
 const exec_1 = require("../../utils/exec");
 const utils_2 = require("./utils");
-async function generateTestsUsingBrowsingAgent(testFilePath) {
-    const logger = new logger_1.CustomLogger({ useReporter: false });
-    (0, utils_2.canRunBrowsingAgent)(testFilePath);
+/**
+ *
+ * Function to generate tests using master agent
+ * @export
+ * @param {GenerateTestsType} {
+ *   testFilePath,
+ *   filePathToUpdate,
+ * }
+ */
+async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, }) {
+    // valiate if the file path and file to update are valid
+    // also warn users if they are on older version of test-gen
+    (0, utils_2.canRunMasterAgent)(testFilePath);
+    // detect available http port on the machine
     const port = await (0, detect_port_1.default)(3030);
+    // start a file service to handle file updates from agent
+    // - also update the file path with updates when agent is done spitting out code
     const fileService = new server_1.FileService({ port });
     await fileService.startFileService();
-    fileService.setFilePath(testFilePath);
+    fileService.setFilePath(filePathToUpdate);
     // read playwright config from ./playwright.config.ts of source repo
     const playwrightConfig = await (0, utils_2.readPlaywrightConfig)();
+    // detect the playwright project name for the given test file and playwright config
     const project = await (0, utils_2.detectProjectName)(testFilePath, playwrightConfig);
-    logger.log(`Detected playwright project name: ${project}`);
-    //TODO: change this to per test
+    console.log(`Detected playwright project name: ${project}`);
+    // run playwright test which will internally run the master agent
     let command = `npx playwright test ${testFilePath} --retries 0 --project ${project} --timeout 0`;
     if (!process.env.CI) {
         command = command.concat(` --headed`);
     }
+    let isError = false;
     try {
         await (0, exec_1.cmd)(command.split(" "), {
             env: {
                 APP_PORT: port.toString(),
                 PW_TEST_HTML_REPORT_OPEN: "never",
+                // pass the test gen token so that the agent has the same configuration as cli
                 TEST_GEN_TOKEN: (0, utils_1.getTestConfigCliArg)(),
             },
         });
     }
     catch (e) {
-        logger.error(e);
-        await new reporter_1.TestGenUpdatesReporter().sendMessage(e);
+        console.error(e);
+        isError = true;
+    }
+    if (isError) {
+        try {
+            const fileContent = await fs_extra_1.default.readFile(filePathToUpdate, "utf-8");
+            const updatedContent = (0, web_1.replaceCreateTestWithNewCode)(filePathToUpdate, fileContent, "");
+            await fs_extra_1.default.writeFile(filePathToUpdate, updatedContent, "utf-8");
+            await (0, web_1.lintErrors)(filePathToUpdate);
+        }
+        catch (e) {
+            console.error("Failed to remove extra scripts from files post test gen error", e);
+        }
     }
     await (0, web_1.removeTestOnly)(testFilePath);
 }
-exports.generateTestsUsingBrowsingAgent = generateTestsUsingBrowsingAgent;
+exports.generateTestsUsingMasterAgent = generateTestsUsingMasterAgent;

package/dist/agent/browsing/utils.d.ts

@@ -3,9 +3,23 @@ import { PlaywrightTestConfig } from "playwright/test";
 import { TestGenConfig } from "../../types";
 export declare function isRegExp(obj: any): obj is RegExp;
 export declare function prepareBrowsingAgentTask(steps: string[]): string;
-export declare function prepareFileForBrowsingAgent(genConfig: TestGenConfig): Promise<void>;
+/**
+ * Function to prepare test file for master agent to run
+ * @param {TestGenConfig} genConfig
+ * @return {*}  {Promise<string>}
+ */
+export declare function prepareFileForMasterAgent(genConfig: TestGenConfig): Promise<string>;
 export declare function injectPwLocatorGenerator(page: Page): Promise<void>;
-export declare function canRunBrowsingAgent(filePath: string): void;
+/**
+ * Function to validate if the test file path are valid.
+ * @throws if there are any missing dependencies for master agent to run.
+ * @param {string} filePath
+ */
+export declare function canRunMasterAgent(filePath: string): void;
+/**
+ * function to read playwright config from the source repo
+ * @return {*}  {Promise<PlaywrightTestConfig>}
+ */
 export declare function readPlaywrightConfig(): Promise<PlaywrightTestConfig>;
 /**
  * detect the project name for the given file in playwright test repo

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAUvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBA+CzE;AAwCD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAUvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAyFD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CAuBjB;AAwCD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA6BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}

package/dist/agent/browsing/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPromptForNextAction = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunBrowsingAgent = exports.injectPwLocatorGenerator = exports.prepareFileForBrowsingAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
+exports.getPromptForNextAction = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunMasterAgent = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const child_process_1 = require("child_process");
 const fs_extra_1 = __importDefault(require("fs-extra"));
@@ -11,6 +11,7 @@ const minimatch_1 = require("minimatch");
 const api_1 = __importDefault(require("tsx/cjs/api"));
 const logger_1 = require("../../bin/logger");
 const web_1 = require("../../bin/utils/platform/web");
+const update_flow_1 = require("../codegen/update-flow");
 function isRegExp(obj) {
     return (obj instanceof RegExp ||
         Object.prototype.toString.call(obj) === "[object RegExp]");
@@ -22,36 +23,91 @@ function prepareBrowsingAgentTask(steps) {
     return task;
 }
 exports.prepareBrowsingAgentTask = prepareBrowsingAgentTask;
-async function prepareFileForBrowsingAgent(genConfig) {
+/**
+ * Function to prepare test file for update scenarios for master agent to run
+ * @param {TestGenConfig} genConfig
+ */
+async function prepareFileForUpdateScenario(genConfig) {
+    const { specPath, testCase } = genConfig;
+    const { name, steps } = testCase;
+    // update the test case with appropriate location for createTest
+    // TODO: reduce the payload for this LLM call. Only provide test file and page files which are used in the test block
+    // this will help with faster response
+    const [suggestion] = await (0, update_flow_1.updateTest)({
+        ...testCase,
+        steps: [
+            ...steps,
+            `
+- You are given a method with interface await createTest(<task>, <playwright_page_instance>)
+- This method will help you execute given task.
+- Given the task you need to decide the right file path and code block to place this method.
+- The task should contain hints as in where to place this method.
+- If there is no hint provided in the task, you can assume the method needs to be placed at the end of the test block.
+- YOU NEED TO MANDATORILY USE "createTest" method to execute the task.
+- Once the methods is placed in the right location with a task and playwright page provided, assume the task is done.
+- You need to respond with file path, code block and updated code block where this method can be placed in order to accomplish the task`,
+        ],
+    }, specPath, genConfig.options, false);
+    const createTestFilePath = suggestion?.updatedFiles[0] || "";
+    console.log("appending to existing test block");
+    console.log("updated test file path", createTestFilePath);
+    // if change suggested by LLM is in spec file
+    const isChangeInSpecFile = createTestFilePath.includes("spec.ts");
+    // if the file is not a spec file, add import for the test-gen
+    if (!isChangeInSpecFile) {
+        await fs_extra_1.default.writeFile(createTestFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(createTestFilePath, "utf-8"), ["createTest"], "@empiricalrun/test-gen"));
+    }
+    const testFileContent = await fs_extra_1.default.readFile(specPath, "utf-8");
+    const { testBlock, testNode } = (0, web_1.getTypescriptTestBlock)(name, testFileContent);
+    const parentDescribe = (0, web_1.findFirstSerialDescribeBlock)(testNode);
+    // add test.only / describe.only to the spec file so that only that block is executed
+    const updatedTestFileContent = newContentsWithTestOnly(testFileContent, testBlock, testBlock, parentDescribe?.getText() || "");
+    await fs_extra_1.default.writeFile(specPath, isChangeInSpecFile
+        ? (0, web_1.addNewImport)(updatedTestFileContent, ["createTest"], "@empiricalrun/test-gen")
+        : updatedTestFileContent);
+}
+/**
+ * Function to prepare test file for new scenarios for master agent to run
+ * @param {TestGenConfig} genConfig
+ */
+async function prepareFileForNewScenario(genConfig) {
     const { specPath, testCase } = genConfig;
     const { name, steps } = testCase;
-    const logger = new logger_1.CustomLogger({ useReporter: false });
+    console.log("creating new test block");
+    const mergedSteps = prepareBrowsingAgentTask(steps);
+    // TODO: this assumes that test code repo has `page` as the main entrypoint fixture
+    const testGenCodeBlock = createTestBlockForCreateTest(name, mergedSteps);
+    const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
+    const newContents = `${existingContents}\n\n${testGenCodeBlock}`;
+    await fs_extra_1.default.writeFile(specPath, (0, web_1.addNewImport)(newContents, ["createTest"], "@empiricalrun/test-gen"));
+}
+/**
+ * Function to prepare test file for master agent to run
+ * @param {TestGenConfig} genConfig
+ * @return {*}  {Promise<string>}
+ */
+async function prepareFileForMasterAgent(genConfig) {
+    const { specPath, testCase } = genConfig;
+    const { name } = testCase;
+    // check if the spec file exists
+    // if no then create a new file with test and expect imports
     if (!fs_extra_1.default.existsSync(specPath)) {
         await fs_extra_1.default.createFile(specPath);
         const fileContentWithImports = (0, web_1.addNewImport)("", ["test", "expect"], (0, web_1.getFixtureImportPath)(specPath));
         await fs_extra_1.default.writeFile(specPath, fileContentWithImports, "utf-8");
     }
-    if (name && steps && steps.length) {
-        const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
-        const { testBlock, parentDescribe } = (0, web_1.getTypescriptTestBlock)(name, existingContents);
-        const mergedSteps = prepareBrowsingAgentTask(steps);
-        let newContents = existingContents;
-        if (testBlock) {
-            logger.log("appending to existing test block");
-            // test scenario is already present
-            const updatedTestBlock = (0, web_1.appendToTestBlock)(testBlock, createTestGenBlock(mergedSteps));
-            newContents = newContentsWithTestOnly(existingContents, testBlock, updatedTestBlock, parentDescribe);
-        }
-        else {
-            logger.log("creating new test block");
-            // TODO: this assumes that test code repo has `page` as the main entrypoint fixture
-            const testGenCodeBlock = createTestBlockForCreateTest(name, mergedSteps);
-            newContents = `${existingContents}\n\n${testGenCodeBlock}`;
-        }
-        await fs_extra_1.default.writeFile(specPath, (0, web_1.addNewImport)(newContents, ["createTest"], "@empiricalrun/test-gen"));
+    let createTestFilePath = specPath;
+    const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
+    const { testBlock } = (0, web_1.getTypescriptTestBlock)(name, existingContents);
+    if (testBlock) {
+        await prepareFileForUpdateScenario(genConfig);
+    }
+    else {
+        await prepareFileForNewScenario(genConfig);
     }
+    return createTestFilePath;
 }
-exports.prepareFileForBrowsingAgent = prepareFileForBrowsingAgent;
+exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
 function newContentsWithTestOnly(existingContents, originalTestBlock, updatedTestBlock, parentDescribeBlock) {
     if (!parentDescribeBlock) {
         const testMarkedAsOnly = updatedTestBlock.replace("test(", "test.only(");
@@ -95,9 +151,14 @@ async function injectPwLocatorGenerator(page) {
     await Promise.all(scripts.map((s) => page.addScriptTag({ content: s })));
 }
 exports.injectPwLocatorGenerator = injectPwLocatorGenerator;
-function canRunBrowsingAgent(filePath) {
+/**
+ * Function to validate if the test file path are valid.
+ * @throws if there are any missing dependencies for master agent to run.
+ * @param {string} filePath
+ */
+function canRunMasterAgent(filePath) {
     if (!fs_extra_1.default.existsSync(filePath)) {
-        throw new Error(`File for browsing agent to run not found: ${filePath}`);
+        throw new Error(`File for master agent to run not found: ${filePath}`);
     }
     const { dependencies: installedPackages } = JSON.parse((0, child_process_1.execSync)("npm ls --json").toString());
     const hasPwTestInstalled = installedPackages["@playwright/test"];
@@ -115,7 +176,11 @@ function canRunBrowsingAgent(filePath) {
         logger.warn(`Outdated @empiricalrun/test-gen package: expected ${latestTestGenVersion}, got ${testGenVersion}`);
     }
 }
-exports.canRunBrowsingAgent = canRunBrowsingAgent;
+exports.canRunMasterAgent = canRunMasterAgent;
+/**
+ * function to read playwright config from the source repo
+ * @return {*}  {Promise<PlaywrightTestConfig>}
+ */
 async function readPlaywrightConfig() {
     const [lastDir] = process.cwd().split("/").reverse();
     const playwrightConfig = (await api_1.default.require("./playwright.config.ts", `${process.cwd()}/${lastDir}`)).default;

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAmJrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAoJrB"}

package/dist/agent/codegen/run.js

@@ -142,6 +142,7 @@ async function generateTest(testCase, file, options) {
     logger.log(`Trace: ${trace.getTraceUrl()}`);
     generatedTestCases.push(testCase);
     trace.update({ input: { testCase }, output: { response } });
+    await (0, llm_1.flushAllTraces)();
     return generatedTestCases;
 }
 exports.generateTest = generateTest;

package/dist/agent/codegen/update-flow.d.ts

@@ -1,3 +1,7 @@
 import { TestCase, TestGenConfigOptions } from "../../types";
-export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions): Promise<TestCase[]>;
+type UpdatedTestCase = TestCase & {
+    updatedFiles: string[];
+};
+export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean): Promise<UpdatedTestCase[]>;
+export {};
 //# sourceMappingURL=update-flow.d.ts.map

package/dist/agent/codegen/update-flow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAoB7D,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAiGrB"}
+{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAqB7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,GACtB,OAAO,CAAC,eAAe,EAAE,CAAC,CAoH5B"}

package/dist/agent/codegen/update-flow.js

@@ -14,7 +14,8 @@ const constants_1 = require("../../constants");
 const session_1 = require("../../session");
 function extractInformation(input) {
     const result = [];
-    const regex = /<filepath>(.*?)<\/filepath.*?>[\s\S]*?<old_code_block>([\s\S]*?)<\/old_code_block>[\s\S]*?<new_code_block>([\s\S]*?)<\/new_code_block>[\s\S]*?<reason>([\s\S]*?)<\/reason>/g;
+    // TODO: use better structure for this. Do not kill me for this please.
+    const regex = /<file_path>(.*?)<\/file_path>[\s\S]*?<old_code_block>([\s\S]*?)<\/old_code_block>[\s\S]*?<new_code_block>([\s\S]*?)<\/new_code_block>[\s\S]*?<change>([\s\S]*?)<\/change>/g;
     let match;
     while ((match = regex.exec(input)) !== null) {
         const [, filePath, oldCode, newCode, reason] = match;
@@ -27,8 +28,8 @@ function extractInformation(input) {
     }
     return result;
 }
-async function updateTest(testCase, file, options) {
-    const logger = new logger_1.CustomLogger();
+async function updateTest(testCase, file, options, logging = true) {
+    const logger = new logger_1.CustomLogger({ useReporter: logging });
     const context = await (0, context_1.contextForGeneration)(file);
     const { codePrompt, pomPrompt, testFileContent } = context;
     const generatedTestCases = [];
@@ -38,7 +39,10 @@ async function updateTest(testCase, file, options) {
         name: "update-test",
         id: crypto_1.default.randomUUID(),
         release: session.version,
-        tags: [options.metadata.projectName, options.metadata.environment].filter((s) => !!s),
+        tags: [
+            options?.metadata.projectName || "",
+            options?.metadata.environment || "",
+        ].filter((s) => !!s),
     });
     trace.event({
         name: "collate-files-as-text",
@@ -59,30 +63,34 @@ async function updateTest(testCase, file, options) {
         scenarioName: testCase.name,
         scenarioSteps: testCase.steps.join("\n"),
         scenarioFile: file,
-    }, 8);
+    }, 14);
     promptSpan.end({ output: { instruction } });
     const llm = new llm_1.LLM({
         trace,
-        provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-        defaultModel: options.model || constants_1.DEFAULT_MODEL,
-        providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+        provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: options?.model || constants_1.DEFAULT_MODEL,
+        providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
     });
     const firstShotMessage = await llm.createChatCompletion({
         messages: instruction,
         modelParameters: {
             ...constants_1.DEFAULT_MODEL_PARAMETERS,
-            ...options.modelParameters,
+            ...options?.modelParameters,
         },
     });
     let response = firstShotMessage?.content || "";
     logger.success("Test generated successfully!");
     const fileChanges = extractInformation(response);
-    fileChanges.forEach(async (fileChange) => {
-        if (fileChange.filePath?.includes("spec.ts")) {
+    await Promise.allSettled(fileChanges.map(async (fileChange) => {
+        if (!fileChange.filePath) {
+            return;
+        }
+        const { testBlock: testBlockUpdate } = (0, web_1.getTypescriptTestBlock)(testCase?.name || "", fileChange.newCode || "");
+        if (testBlockUpdate) {
             // assuming the test case getting updated
             // maintaining the previous accuracy of the test case update
             const readWriteFileSpan = trace.span({ name: "write-to-file" });
-            let contents = fs_extra_1.default.readFileSync(fileChange.filePath, "utf-8");
+            let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
             const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(fileChange.newCode, testCase?.name);
             let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
             const { testBlock } = (0, web_1.getTypescriptTestBlock)(testCase?.name, contents);
@@ -91,25 +99,33 @@ async function updateTest(testCase, file, options) {
             await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
             readWriteFileSpan.end({ output: { updatedContent } });
             trace.event({ name: "format-file" });
+            await (0, web_1.lintErrors)(fileChange.filePath);
             await (0, web_1.formatCode)(fileChange.filePath);
-            logger.success("File formatted successfully!");
+            logger.success(`${fileChange.filePath} file formatted successfully!`);
         }
         else {
             // since we dont know what is getting updated,
-            // we believe that the patch is correct and contains few before and after lines to make the change unique
+            // we believe that the patch is correct and contains few before and after lines
+            // to make the change unique for search & replace
             const readWriteFileSpan = trace.span({ name: "write-to-file" });
-            let contents = fs_extra_1.default.readFileSync(fileChange.filePath, "utf-8");
+            let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
+            //TODO: move this to usage of ast blocks
             contents = contents.replace(fileChange.oldCode, `\n\n${fileChange.newCode}`);
             await fs_extra_1.default.writeFile(fileChange.filePath, contents, "utf-8");
             readWriteFileSpan.end({ output: { contents } });
             trace.event({ name: "format-file" });
+            await (0, web_1.lintErrors)(fileChange.filePath);
             await (0, web_1.formatCode)(fileChange.filePath);
-            logger.success("File formatted successfully!");
+            logger.success(`${fileChange.filePath} file formatted successfully!`);
         }
-    });
+    }));
     logger.log(`Trace: ${trace.getTraceUrl()}`);
-    generatedTestCases.push(testCase);
+    generatedTestCases.push({
+        ...testCase,
+        updatedFiles: fileChanges.map((f) => f.filePath),
+    });
     trace.update({ input: { testCase }, output: { response } });
+    await (0, llm_1.flushAllTraces)();
     return generatedTestCases;
 }
 exports.updateTest = updateTest;

package/dist/bin/index.js

@@ -27,8 +27,11 @@ async function runAgent(testGenConfig) {
     if (testGenConfig.options?.agent !== "code") {
         // this assumes we have only one scenario in test config
         logger.success(`Generating test using ${testGenConfig.options?.agent} agent`);
-        await (0, utils_1.prepareFileForBrowsingAgent)(testGenConfig);
-        await (0, run_1.generateTestsUsingBrowsingAgent)(specPath);
+        const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)(testGenConfig);
+        await (0, run_1.generateTestsUsingMasterAgent)({
+            testFilePath: specPath,
+            filePathToUpdate,
+        });
         await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
             projectRepoName: testGenConfig.options.metadata.projectRepoName,
             testName: testCase.name,

package/dist/bin/utils/context.js

@@ -11,14 +11,14 @@ async function contextForGeneration(file) {
     const ignoreFn = (0, ignore_1.default)();
     if (fs_extra_1.default.existsSync(".gitignore")) {
         // Not checking for nested gitignore
-        const gitignore = fs_extra_1.default.readFileSync(".gitignore").toString();
+        const gitignore = (await fs_extra_1.default.readFile(".gitignore")).toString();
         ignoreFn.add(gitignore);
     }
     const filter = ignoreFn.createFilter();
     return {
         codePrompt: await (0, fs_1.generatePromptFromDirectory)("./tests", filter),
         pomPrompt: await (0, fs_1.generatePromptFromDirectory)("./pages", filter),
-        testFileContent: fs_extra_1.default.readFileSync(file, "utf-8"),
+        testFileContent: await fs_extra_1.default.readFile(file, "utf-8"),
     };
 }
 exports.contextForGeneration = contextForGeneration;

package/dist/bin/utils/platform/web/index.d.ts

@@ -1,7 +1,39 @@
+import { Node } from "ts-morph";
+/**
+ * function to get the test block and test node for the scenario
+ * @export
+ * @param {string} scenarioName
+ * @param {string} content
+ * @return { testBlock: string; parentDescribe: string; } testBlock - the test block content, testNode - the test function node
+ */
 export declare function getTypescriptTestBlock(scenarioName: string, content: string): {
     testBlock: string | undefined;
-    parentDescribe: string | undefined;
+    testNode: Node | undefined;
 };
+/**
+ * Function to find the first 'describe' block configured with 'serial: true'
+ *
+ * e.g.
+ *
+ * test.describe("foo", () => {
+ *
+ *    test.describe.configure({ mode: "serial" });
+ *
+ *    test.describe("bar", () => {
+ *
+ *      test.describe.configure({ mode: "serial" });
+ *
+ *    })
+ *
+ * })
+ *
+ * for the above example,
+ * this function will return the first 'describe' block which is named "foo"
+ *
+ * @param {(Node | undefined)} node
+ * @return {(Node | undefined)}
+ */
+export declare function findFirstSerialDescribeBlock(node: Node | undefined): Node | undefined;
 export declare function appendToTestBlock(testBlock: string, content: string): string;
 export declare function validateTypescript(filePath: string): string[];
 export declare function stripAndPrependImports(content: string, testName: string): Promise<(string | undefined)[]>;

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;;;EAwB3E;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwD7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAUjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAqB,IAAI,EAAuB,MAAM,UAAU,CAAC;AAGxE;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd;IACD,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAeA;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwD7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAUjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB"}

package/dist/bin/utils/platform/web/index.js

@@ -3,28 +3,78 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.replaceCreateTestWithNewCode = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.getTypescriptTestBlock = void 0;
+exports.replaceCreateTestWithNewCode = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.findFirstSerialDescribeBlock = exports.getTypescriptTestBlock = void 0;
 const eslint_1 = require("eslint");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const prettier_1 = __importDefault(require("prettier"));
 const ts_morph_1 = require("ts-morph");
 const typescript_1 = __importDefault(require("typescript"));
+/**
+ * function to get the test block and test node for the scenario
+ * @export
+ * @param {string} scenarioName
+ * @param {string} content
+ * @return { testBlock: string; parentDescribe: string; } testBlock - the test block content, testNode - the test function node
+ */
 function getTypescriptTestBlock(scenarioName, content) {
     const project = new ts_morph_1.Project();
     const sourceFile = project.createSourceFile("test.ts", content);
     const testFunctionNode = sourceFile.getFirstDescendant((node) => !!(node.isKind(ts_morph_1.SyntaxKind.CallExpression) &&
         node.getExpression().getText() === "test" &&
         node.getArguments()[0]?.getText().includes(scenarioName)));
-    // TODO: support files that have 2 describe blocks with the same test name in them
-    const describeBlockNode = sourceFile.getFirstDescendant((node) => !!(node.isKind(ts_morph_1.SyntaxKind.CallExpression) &&
-        node.getExpression().getText() === "test.describe" &&
-        node.getText().includes(scenarioName)));
     return {
         testBlock: testFunctionNode?.getText(),
-        parentDescribe: describeBlockNode?.getText(),
+        testNode: testFunctionNode,
     };
 }
 exports.getTypescriptTestBlock = getTypescriptTestBlock;
+/**
+ * Function to find the first 'describe' block configured with 'serial: true'
+ *
+ * e.g.
+ *
+ * test.describe("foo", () => {
+ *
+ *    test.describe.configure({ mode: "serial" });
+ *
+ *    test.describe("bar", () => {
+ *
+ *      test.describe.configure({ mode: "serial" });
+ *
+ *    })
+ *
+ * })
+ *
+ * for the above example,
+ * this function will return the first 'describe' block which is named "foo"
+ *
+ * @param {(Node | undefined)} node
+ * @return {(Node | undefined)}
+ */
+function findFirstSerialDescribeBlock(node) {
+    let currentNode = node;
+    // Traverse upwards until we find a 'describe' block with 'serial: true'
+    while (currentNode) {
+        const parentDescribe = currentNode.getFirstAncestorByKind(ts_morph_1.SyntaxKind.CallExpression);
+        if (parentDescribe) {
+            const isDescribe = parentDescribe.getFirstChild()?.getText() === "test.describe";
+            if (isDescribe) {
+                const configureCall = parentDescribe
+                    .getDescendantsOfKind(ts_morph_1.SyntaxKind.CallExpression)
+                    .find((call) => call.getText().includes("configure"));
+                if (configureCall) {
+                    // Check if 'serial: true' exists
+                    if (configureCall.getText().includes("serial")) {
+                        return parentDescribe;
+                    }
+                }
+            }
+        }
+        currentNode = parentDescribe; // Move up the tree
+    }
+    return undefined; // Return undefined if no 'describe' with serial: true is found
+}
+exports.findFirstSerialDescribeBlock = findFirstSerialDescribeBlock;
 function appendToTestBlock(testBlock, content) {
     const updateTestBlock = testBlock.replace(/\}\)$/, `\n\n${content}\n\n })`);
     return updateTestBlock;
@@ -96,7 +146,7 @@ async function lintErrors(filePath) {
     });
     const [result] = await eslint.lintFiles(filePath);
     if (result?.output) {
-        fs_extra_1.default.writeFileSync(filePath, result.output);
+        await fs_extra_1.default.writeFile(filePath, result.output);
     }
 }
 exports.lintErrors = lintErrors;
@@ -107,7 +157,7 @@ async function formatCode(filePath) {
         ...prettierConfig,
         filepath: filePath,
     });
-    fs_extra_1.default.writeFileSync(filePath, formattedContent);
+    await fs_extra_1.default.writeFile(filePath, formattedContent);
 }
 exports.formatCode = formatCode;
 function addNewImport(contents, modules, pkg) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.25.2",
+  "version": "0.27.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"