@empiricalrun/test-gen 0.27.6 → 0.27.8

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @empiricalrun/test-gen
 
+## 0.27.8
+
+### Patch Changes
+
+- 849ec6a: fix: trace and video not playing after generation
+
+## 0.27.7
+
+### Patch Changes
+
+- ae6e28a: fix: handle different image input for different llms and handle escape characters in llm response
+- cf0a651: fix: master agent should generate code with correct page variable name
+- b1825cf: fix: handle typescript errors and add tests
+- Updated dependencies [ae6e28a]
+  - @empiricalrun/llm@0.9.2
+
 ## 0.27.6
 
 ### Patch Changes

package/dist/actions/assert.js

@@ -16,7 +16,7 @@ const assertTextVisibilityActionGenerator = (page) => {
         },
         // TODO: args transformer to be kept at a single place
         template: (args, options) => {
-            return `await expect(page.${options.locator}).toBeVisible();`;
+            return `await expect(${(0, utils_1.getPageVarName)()}.${options.locator}).toBeVisible();`;
         },
         name: exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME,
         schema: {

package/dist/actions/click.js

@@ -18,7 +18,7 @@ const clickActionGenerator = (page) => {
         },
         // TODO: args transformer to be kept at a single place
         template: (args, options) => {
-            return `await page.${options.locator}.click();`;
+            return `await ${(0, utils_1.getPageVarName)()}.${options.locator}.click();`;
         },
         name: exports.PLAYWRIGHT_CLICK_ACTION_NAME,
         schema: {

package/dist/actions/fill.js

@@ -17,7 +17,7 @@ const fillActionGenerator = (page) => {
         },
         // TODO: args transformer to be kept at a single place
         template: (args, options) => {
-            return `await page.${options.locator}.fill("${args.text}");`;
+            return `await ${(0, utils_1.getPageVarName)()}.${options.locator}.fill("${args.text}");`;
         },
         name: exports.PLAYWRIGHT_FILL_ACTION_NAME,
         schema: {

package/dist/actions/goto.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAqCjC,CAAC"}
+{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAqCjC,CAAC"}

package/dist/actions/goto.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.gotoActionGenerator = exports.PLAYWRIGHT_GOTO_ACTION_NAME = void 0;
 const utils_1 = require("../agent/browsing/utils");
 const constants_1 = require("./constants");
+const utils_2 = require("./utils");
 exports.PLAYWRIGHT_GOTO_ACTION_NAME = "page_goto";
 const gotoActionGenerator = (page) => {
     return {
@@ -15,7 +16,7 @@ const gotoActionGenerator = (page) => {
         // TODO: args transformer to be kept at a single place
         template: (args) => {
             const url = args.url;
-            const templ = `await page.goto("${url}");`;
+            const templ = `await ${(0, utils_2.getPageVarName)()}.goto("${url}");`;
             return templ;
         },
         name: exports.PLAYWRIGHT_GOTO_ACTION_NAME,

package/dist/actions/reload-page.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reload-page.d.ts","sourceRoot":"","sources":["../../src/actions/reload-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,6BAA6B,gBAAgB,CAAC;AAE3D,eAAO,MAAM,qBAAqB,EAAE,yBA+BnC,CAAC"}
+{"version":3,"file":"reload-page.d.ts","sourceRoot":"","sources":["../../src/actions/reload-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,6BAA6B,gBAAgB,CAAC;AAE3D,eAAO,MAAM,qBAAqB,EAAE,yBA+BnC,CAAC"}

package/dist/actions/reload-page.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.reloadActionGenerator = exports.PLAYWRIGHT_RELOAD_ACTION_NAME = void 0;
 const utils_1 = require("../agent/browsing/utils");
 const constants_1 = require("./constants");
+const utils_2 = require("./utils");
 exports.PLAYWRIGHT_RELOAD_ACTION_NAME = "page_reload";
 const reloadActionGenerator = (page) => {
     return {
@@ -12,7 +13,7 @@ const reloadActionGenerator = (page) => {
             await (0, utils_1.injectPwLocatorGenerator)(page);
         },
         template: () => {
-            const templ = `await page.reload();`;
+            const templ = `await ${(0, utils_2.getPageVarName)()}.reload();`;
             return templ;
         },
         name: exports.PLAYWRIGHT_RELOAD_ACTION_NAME,

package/dist/actions/utils/index.d.ts

@@ -6,4 +6,5 @@ declare global {
     }
 }
 export declare function getPlaywrightLocatorUsingCssSelector(cssSelector: string, page: Page): Promise<any>;
+export declare function getPageVarName(): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/actions/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,IAAI,gBAkBX"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,IAAI,gBAwBX;AAED,wBAAgB,cAAc,WAG7B"}

package/dist/actions/utils/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPlaywrightLocatorUsingCssSelector = void 0;
+exports.getPageVarName = exports.getPlaywrightLocatorUsingCssSelector = void 0;
 async function getPlaywrightLocatorUsingCssSelector(cssSelector, page) {
     // TODO: analyse other solutions than just css. Also find other solutions to support :has-text -> prompting ?
     // jquery doesnt support :has-text. neither css. Only playwright locator supports this selector though.
@@ -9,7 +9,13 @@ async function getPlaywrightLocatorUsingCssSelector(cssSelector, page) {
         .replaceAll(":text", ":contains");
     return await page.evaluate((locator) => {
         const elements = window.jQuery(locator.cssForJq);
-        let selectedElem = elements[0];
+        let elIdx = 0;
+        Array.from(elements).forEach((el, i) => {
+            if (window.jQuery(el).is(":visible")) {
+                elIdx = i;
+            }
+        });
+        let selectedElem = elements[elIdx];
         if (!selectedElem) {
             throw Error(`Unable to find element, css: ${locator.css}`);
         }
@@ -17,3 +23,8 @@ async function getPlaywrightLocatorUsingCssSelector(cssSelector, page) {
     }, { css: cssSelector, cssForJq: sanitizedCssSelectorForJQuery });
 }
 exports.getPlaywrightLocatorUsingCssSelector = getPlaywrightLocatorUsingCssSelector;
+function getPageVarName() {
+    // eslint-disable-next-line turbo/no-undeclared-env-vars
+    return process.env.PAGE_VAR_NAME || "page";
+}
+exports.getPageVarName = getPageVarName;

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,GACjB,EAAE,iBAAiB,iBAqDnB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAkBA,KAAK,iBAAiB,GAAG;IACvB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,GACjB,EAAE,iBAAiB,iBAuDnB"}

package/dist/agent/browsing/run.js

@@ -35,6 +35,7 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, }
     const playwrightConfig = await (0, utils_2.readPlaywrightConfig)();
     // detect the playwright project name for the given test file and playwright config
     const project = await (0, utils_2.detectProjectName)(testFilePath, playwrightConfig);
+    const pageVar = await (0, web_1.getPageVariableNameFromCreateTest)(testFilePath);
     console.log(`Detected playwright project name: ${project}`);
     // run playwright test which will internally run the master agent
     let command = `npx playwright test ${testFilePath} --retries 0 --project ${project} --timeout 0`;
@@ -49,6 +50,7 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, }
                 PW_TEST_HTML_REPORT_OPEN: "never",
                 // pass the test gen token so that the agent has the same configuration as cli
                 TEST_GEN_TOKEN: (0, utils_1.getTestConfigCliArg)(),
+                PAGE_VAR_NAME: pageVar || "page",
             },
         });
     }

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAUvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAyFD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CAuBjB;AAwCD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA6BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CA8CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAWvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAI5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAqFD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CAuBjB;AAwCD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA6BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CA8CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}

package/dist/agent/browsing/utils.js

@@ -10,7 +10,9 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 const minimatch_1 = require("minimatch");
 const api_1 = __importDefault(require("tsx/cjs/api"));
 const logger_1 = require("../../bin/logger");
+const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
+const fix_ts_errors_1 = require("../codegen/fix-ts-errors");
 const update_flow_1 = require("../codegen/update-flow");
 function isRegExp(obj) {
     return (obj instanceof RegExp ||
@@ -47,24 +49,25 @@ async function prepareFileForUpdateScenario(genConfig) {
 - Once the methods is placed in the right location with a task and playwright page provided, assume the task is done.
 - You need to respond with file path, code block and updated code block where this method can be placed in order to accomplish the task`,
         ],
-    }, specPath, genConfig.options, false);
+    }, specPath, genConfig.options, false, false);
     const createTestFilePath = suggestion?.updatedFiles[0] || "";
     console.log("appending to existing test block");
     console.log("updated test file path", createTestFilePath);
-    // if change suggested by LLM is in spec file
-    const isChangeInSpecFile = createTestFilePath.includes("spec.ts");
-    // if the file is not a spec file, add import for the test-gen
-    if (!isChangeInSpecFile) {
-        await fs_extra_1.default.writeFile(createTestFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(createTestFilePath, "utf-8"), ["createTest"], "@empiricalrun/test-gen"));
-    }
+    await fs_extra_1.default.writeFile(createTestFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(createTestFilePath, "utf-8"), ["createTest"], "@empiricalrun/test-gen"));
+    const { codePrompt, pomPrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
+    await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
+        file: createTestFilePath,
+        testCode: codePrompt,
+        pomCode: pomPrompt,
+        testCase: testCase,
+        options: genConfig.options,
+    });
     const testFileContent = await fs_extra_1.default.readFile(specPath, "utf-8");
     const { testBlock, testNode } = (0, web_1.getTypescriptTestBlock)(name, testFileContent);
     const parentDescribe = (0, web_1.findFirstSerialDescribeBlock)(testNode);
     // add test.only / describe.only to the spec file so that only that block is executed
     const updatedTestFileContent = newContentsWithTestOnly(testFileContent, testBlock, testBlock, parentDescribe?.getText() || "");
-    await fs_extra_1.default.writeFile(specPath, isChangeInSpecFile
-        ? (0, web_1.addNewImport)(updatedTestFileContent, ["createTest"], "@empiricalrun/test-gen")
-        : updatedTestFileContent);
+    await fs_extra_1.default.writeFile(specPath, updatedTestFileContent);
 }
 /**
  * Function to prepare test file for new scenarios for master agent to run

package/dist/agent/codegen/fix-ts-errors.d.ts

@@ -0,0 +1,13 @@
+import { TraceClient } from "@empiricalrun/llm";
+import { CustomLogger } from "../../bin/logger";
+import { TestCase, TestGenConfigOptions } from "../../types";
+export declare function validateAndFixTypescriptErrors({ trace, logger, file, testCode, pomCode, testCase, options, }: {
+    trace?: TraceClient;
+    logger?: CustomLogger;
+    file: string;
+    testCode: string;
+    pomCode: string;
+    testCase: TestCase;
+    options?: TestGenConfigOptions;
+}): Promise<void>;
+//# sourceMappingURL=fix-ts-errors.d.ts.map

package/dist/agent/codegen/fix-ts-errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAQhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,iBAoEA"}

package/dist/agent/codegen/fix-ts-errors.js

@@ -0,0 +1,76 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateAndFixTypescriptErrors = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const logger_1 = require("../../bin/logger");
+const web_1 = require("../../bin/utils/platform/web");
+const constants_1 = require("../../constants");
+async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.CustomLogger(), file, testCode, pomCode, testCase, options, }) {
+    const validateTypesSpan = trace?.span({ name: "detect-type-errors-in-file" });
+    logger.log("Validating types...");
+    let errors = (0, web_1.validateTypescript)(file);
+    validateTypesSpan?.end({ output: { errors } });
+    if (!errors.length) {
+        logger.success("Found no type issues!");
+    }
+    const maxIteration = 2;
+    let counter = 0;
+    while (errors.length > 0) {
+        const fileContent = await fs_extra_1.default.readFile(file, "utf-8");
+        counter += 1;
+        if (counter > maxIteration) {
+            trace?.event({ name: "code-fix-iteration-max-out" });
+            logger.error([
+                `Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors.`,
+                `Trace: ${trace?.getTraceUrl()}`,
+            ].join("\n"));
+            break;
+        }
+        trace?.event({ name: "Found errors fixing" });
+        logger.warn("Found few errors while validating types:");
+        errors.forEach((e) => logger.warn(e));
+        logger.log("Trying to fix above errors...");
+        const promptSpan = trace?.span({ name: "fix-type-errors-prompt" });
+        const instruction = await (0, llm_1.getPrompt)("fix-file-errors-ts", {
+            testFiles: testCode || "",
+            pageFiles: pomCode || "",
+            scenarioFile: file,
+            errors: errors,
+            fileContent: fileContent,
+            scenaioName: testCase.name,
+        });
+        promptSpan?.end({ output: { instruction } });
+        const llm = new llm_1.LLM({
+            trace,
+            provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+            defaultModel: options?.model || constants_1.DEFAULT_MODEL,
+            providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+        });
+        const message = await llm.createChatCompletion({
+            messages: instruction,
+            modelParameters: {
+                ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                ...options?.modelParameters,
+            },
+        });
+        const response = message?.content || "";
+        const readWriteFileSpan = trace?.span({ name: "write-to-file" });
+        await fs_extra_1.default.writeFile(file, response, "utf-8");
+        readWriteFileSpan?.end({ output: { response } });
+        trace?.event({ name: "lint-file" });
+        await (0, web_1.lintErrors)(file);
+        const validateTypesSpan = trace?.span({
+            name: "detect-type-errors-in-file",
+        });
+        errors = (0, web_1.validateTypescript)(file);
+        validateTypesSpan?.end({ output: { errors } });
+        if (!errors.length) {
+            logger.success("Found no type issues!");
+        }
+    }
+}
+exports.validateAndFixTypescriptErrors = validateAndFixTypescriptErrors;

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAoJrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CA+FrB"}

package/dist/agent/codegen/run.js

@@ -12,6 +12,7 @@ const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const session_1 = require("../../session");
+const fix_ts_errors_1 = require("./fix-ts-errors");
 const update_flow_1 = require("./update-flow");
 async function generateTest(testCase, file, options) {
     const logger = new logger_1.CustomLogger();
@@ -79,63 +80,15 @@ async function generateTest(testCase, file, options) {
     logger.log("Linting generated code...");
     trace.event({ name: "lint-file" });
     await (0, web_1.lintErrors)(file);
-    const validateTypesSpan = trace.span({ name: "detect-type-errors-in-file" });
-    logger.log("Validating types...");
-    let errors = (0, web_1.validateTypescript)(file);
-    validateTypesSpan.end({ output: { errors } });
-    if (!errors.length) {
-        logger.success("Found no type issues!");
-    }
-    const maxIteration = 2;
-    let counter = 0;
-    while (errors.length > 0) {
-        const fileContent = fs_extra_1.default.readFileSync(file, "utf-8");
-        counter += 1;
-        if (counter > maxIteration) {
-            trace.event({ name: "code-fix-iteration-max-out" });
-            logger.error([
-                `Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors.`,
-                `Run the test-gen command again, once errors are fixed.`,
-                `Trace: ${trace.getTraceUrl()}`,
-            ].join("\n"));
-            break;
-        }
-        trace.event({ name: "Found errors fixing" });
-        logger.warn("Found few errors while validating types:");
-        errors.forEach((e) => logger.warn(e));
-        logger.log("Trying to fix above errors...");
-        const promptSpan = trace.span({ name: "fix-type-errors-prompt" });
-        const instruction = await (0, llm_1.getPrompt)("fix-file-errors-ts", {
-            testFiles: codePrompt || "",
-            pageFiles: pomPrompt || "",
-            scenarioFile: file,
-            errors: errors,
-            fileContent: fileContent,
-            scenaioName: testCase.name,
-        });
-        promptSpan.end({ output: { instruction } });
-        const message = await llm.createChatCompletion({
-            messages: instruction,
-            modelParameters: {
-                ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                ...options.modelParameters,
-            },
-        });
-        response = message?.content || "";
-        const readWriteFileSpan = trace.span({ name: "write-to-file" });
-        await fs_extra_1.default.writeFile(file, response, "utf-8");
-        readWriteFileSpan.end({ output: { response } });
-        trace.event({ name: "lint-file" });
-        await (0, web_1.lintErrors)(file);
-        const validateTypesSpan = trace.span({
-            name: "detect-type-errors-in-file",
-        });
-        errors = (0, web_1.validateTypescript)(file);
-        validateTypesSpan.end({ output: { errors } });
-        if (!errors.length) {
-            logger.success("Found no type issues!");
-        }
-    }
+    await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
+        trace,
+        logger,
+        file,
+        testCode: codePrompt,
+        pomCode: pomPrompt,
+        testCase: testCase,
+        options,
+    });
     trace.event({ name: "format-file" });
     await (0, web_1.formatCode)(file);
     logger.success("File formatted successfully!");

package/dist/agent/codegen/update-flow.d.ts

@@ -2,6 +2,6 @@ import { TestCase, TestGenConfigOptions } from "../../types";
 type UpdatedTestCase = TestCase & {
     updatedFiles: string[];
 };
-export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean): Promise<UpdatedTestCase[]>;
+export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean, validate?: boolean): Promise<UpdatedTestCase[]>;
 export {};
 //# sourceMappingURL=update-flow.d.ts.map

package/dist/agent/codegen/update-flow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAqB7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,GACtB,OAAO,CAAC,eAAe,EAAE,CAAC,CA6I5B"}
+{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,GACvB,OAAO,CAAC,eAAe,EAAE,CAAC,CAoJ5B"}

package/dist/agent/codegen/update-flow.js

@@ -13,23 +13,9 @@ const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const session_1 = require("../../session");
-function extractInformation(input) {
-    const result = [];
-    // TODO: use better structure for this. Do not kill me for this please.
-    const regex = /<file_path>(.*?)<\/file_path>[\s\S]*?<old_code_block>([\s\S]*?)<\/old_code_block>[\s\S]*?<new_code_block>([\s\S]*?)<\/new_code_block>[\s\S]*?<change>([\s\S]*?)<\/change>/g;
-    let match;
-    while ((match = regex.exec(input)) !== null) {
-        const [, filePath, oldCode, newCode, reason] = match;
-        result.push({
-            filePath: filePath?.trim(),
-            oldCode: oldCode?.trim(),
-            newCode: newCode?.trim(),
-            reason: reason?.trim(),
-        });
-    }
-    return result;
-}
-async function updateTest(testCase, file, options, logging = true) {
+const fix_ts_errors_1 = require("./fix-ts-errors");
+const utils_1 = require("./utils");
+async function updateTest(testCase, file, options, logging = true, validate = true) {
     const logger = new logger_1.CustomLogger({ useReporter: logging });
     const context = await (0, context_1.contextForGeneration)(file);
     const { codePrompt, pomPrompt, testFileContent } = context;
@@ -81,7 +67,7 @@ async function updateTest(testCase, file, options, logging = true) {
     });
     let response = firstShotMessage?.content || "";
     logger.success("Test generated successfully!");
-    const fileChanges = extractInformation(response);
+    const fileChanges = (0, utils_1.extractTestUpdates)(response);
     await Promise.allSettled(fileChanges.map(async (fileChange) => {
         if (!fileChange.filePath) {
             return;
@@ -97,12 +83,8 @@ async function updateTest(testCase, file, options, logging = true) {
             const { testBlock } = (0, web_1.getTypescriptTestBlock)(testCase?.name, contents);
             contents = contents.replace(testBlock, `\n\n${strippedContent}`);
             updatedContent = prependContent + contents;
-            await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
+            await fs_extra_1.default.writeFile(fileChange.filePath, updatedContent, "utf-8");
             readWriteFileSpan.end({ output: { updatedContent } });
-            trace.event({ name: "format-file" });
-            await (0, web_1.lintErrors)(fileChange.filePath);
-            await (0, web_1.formatCode)(fileChange.filePath);
-            logger.success(`${fileChange.filePath} file formatted successfully!`);
         }
         else {
             const readWriteFileSpan = trace.span({ name: "write-to-file" });
@@ -129,11 +111,22 @@ async function updateTest(testCase, file, options, logging = true) {
             }
             await fs_extra_1.default.writeFile(fileChange.filePath, contents, "utf-8");
             readWriteFileSpan.end({ output: { contents } });
-            trace.event({ name: "format-file" });
-            await (0, web_1.lintErrors)(fileChange.filePath);
-            await (0, web_1.formatCode)(fileChange.filePath);
-            logger.success(`${fileChange.filePath} file formatted successfully!`);
         }
+        // format and validate file change
+        if (validate) {
+            await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
+                trace,
+                logger,
+                file: fileChange.filePath,
+                testCode: codePrompt,
+                pomCode: pomPrompt,
+                testCase: testCase,
+                options,
+            });
+        }
+        trace.event({ name: "format-file" });
+        await (0, web_1.formatCode)(fileChange.filePath);
+        logger.success(`${fileChange.filePath} file formatted successfully!`);
     }));
     logger.log(`Trace: ${trace.getTraceUrl()}`);
     generatedTestCases.push({

package/dist/agent/codegen/utils.d.ts

@@ -0,0 +1,19 @@
+/**
+ *
+ * method to extract file path and code updates for the LLM response of update flow
+ * @export
+ * @param {string} input
+ * @return {*}  {({
+ *   filePath: string | undefined;
+ *   oldCode: string | undefined;
+ *   newCode: string | undefined;
+ *   reason: string | undefined;
+ * }[])}
+ */
+export declare function extractTestUpdates(input: string): {
+    filePath: string | undefined;
+    oldCode: string | undefined;
+    newCode: string | undefined;
+    reason: string | undefined;
+}[];
+//# sourceMappingURL=utils.d.ts.map

package/dist/agent/codegen/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAqBF"}

package/dist/agent/codegen/utils.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractTestUpdates = void 0;
+/**
+ *
+ * method to extract file path and code updates for the LLM response of update flow
+ * @export
+ * @param {string} input
+ * @return {*}  {({
+ *   filePath: string | undefined;
+ *   oldCode: string | undefined;
+ *   newCode: string | undefined;
+ *   reason: string | undefined;
+ * }[])}
+ */
+function extractTestUpdates(input) {
+    const result = [];
+    // TODO: use better structure for this. Do not kill me for this please.
+    const regex = /<file_path>(.*?)<\/file_path>[\s\S]*?<old_code_block>([\s\S]*?)<\/old_code_block>[\s\S]*?<new_code_block>([\s\S]*?)<\/new_code_block>[\s\S]*?<change>([\s\S]*?)<\/change>/g;
+    let match;
+    // sometimes LLM responds with single backslashes instead of double backslashe for escapes
+    // we only have a scenario of double quotes being escaped.
+    // eslint-disable-next-line no-useless-escape, prettier/prettier
+    input = input.replaceAll(' \"', ' \\"').replaceAll('\" ', '\\" ');
+    while ((match = regex.exec(input)) !== null) {
+        const [, filePath, oldCode, newCode, reason] = match;
+        result.push({
+            filePath: filePath?.trim(),
+            oldCode: oldCode?.trim(),
+            newCode: newCode?.trim(),
+            reason: reason?.trim(),
+        });
+    }
+    return result;
+}
+exports.extractTestUpdates = extractTestUpdates;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAMlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAE1D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,eAAe,EAAE,MAAM,EAAE,EACzB,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,oBAAoB,gBAsE9B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAE1D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,eAAe,EAAE,MAAM,EAAE,EACzB,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,oBAAoB,gBAyE9B"}

package/dist/agent/master/run.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.masterAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
+const vision_1 = require("@empiricalrun/llm/vision");
 const done_1 = require("../../actions/done");
 const next_task_1 = require("../../actions/next-task");
 const constants_1 = require("../../constants");
@@ -12,7 +13,7 @@ async function masterAgent(task, page, executedActions, trace, llm, options) {
     const buffer = await page.screenshot({ fullPage: true });
     const testGenReporter = new reporter_1.TestGenUpdatesReporter();
     const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
-    const pageScreenshot = `data:image/png;base64,${buffer.toString("base64")}`;
+    const pageScreenshot = buffer.toString("base64");
     const promptMessages = await (0, llm_1.getPrompt)("test-gen", {
         task,
         executedActions: executedActions.map((a) => a).join("\n"),
@@ -29,7 +30,7 @@ async function masterAgent(task, page, executedActions, trace, llm, options) {
         {
             type: "image_url",
             image_url: {
-                url: pageScreenshot,
+                url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
             },
         },
     ];

package/dist/bin/index.js

@@ -61,7 +61,12 @@ async function runAgent(testGenConfig) {
         sessionId: testGenConfig.options?.metadata.testSessionId,
         generationId: testGenConfig.options?.metadata.generationId,
     });
-    await runAgent(testGenConfig);
+    try {
+        await runAgent(testGenConfig);
+    }
+    catch (e) {
+        logger.error("Failed to run agent for the scenario", e);
+    }
     // TODO: move these reporters to a better lifecycle
     await (0, ci_1.reportOnCI)(testGenConfig.testCase);
     await (0, llm_1.flushAllTraces)();

package/dist/bin/utils/platform/web/index.d.ts

@@ -42,5 +42,6 @@ export declare function formatCode(filePath: string): Promise<void>;
 export declare function addNewImport(contents: string, modules: string[], pkg: string): string;
 export declare function removeTestOnly(filePath: string): Promise<void>;
 export declare function getFixtureImportPath(filePath: string): string;
+export declare function getPageVariableNameFromCreateTest(filePath: string): Promise<string | undefined>;
 export declare function replaceCreateTestWithNewCode(filePath: string, contents: string, generatedCode: string): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAqB,IAAI,EAAuB,MAAM,UAAU,CAAC;AAGxE;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd;IACD,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAeA;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwD7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAUjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAqB,IAAI,EAAuB,MAAM,UAAU,CAAC;AAGxE;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd;IACD,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAeA;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAUjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAmBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB"}

package/dist/bin/utils/platform/web/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.replaceCreateTestWithNewCode = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.findFirstSerialDescribeBlock = exports.getTypescriptTestBlock = void 0;
+exports.replaceCreateTestWithNewCode = exports.getPageVariableNameFromCreateTest = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.findFirstSerialDescribeBlock = exports.getTypescriptTestBlock = void 0;
 const eslint_1 = require("eslint");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const prettier_1 = __importDefault(require("prettier"));
@@ -96,16 +96,6 @@ function validateTypescript(filePath) {
     const syntacticDiagnostics = program.getSyntacticDiagnostics(sourceFile);
     if (syntacticDiagnostics.length > 0) {
         syntacticDiagnostics.forEach((diagnostic) => {
-            // const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(
-            //   diagnostic.start,
-            // );
-            // const message = ts.flattenDiagnosticMessageText(
-            //   diagnostic.messageText,
-            //   "\n",
-            // );
-            // logger.log(
-            //   `${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`,
-            // );
             if (typeof diagnostic.messageText === "string") {
                 errors.push(diagnostic.messageText);
             }
@@ -188,6 +178,22 @@ function getFixtureImportPath(filePath) {
     return fixturesPath;
 }
 exports.getFixtureImportPath = getFixtureImportPath;
+async function getPageVariableNameFromCreateTest(filePath) {
+    const contents = await fs_extra_1.default.readFile(filePath, "utf-8");
+    const project = new ts_morph_1.Project();
+    const sourceFile = project.createSourceFile("test.ts", contents);
+    const createTestNode = sourceFile.getFirstDescendant((node) => !!(node.isKind(ts_morph_1.SyntaxKind.CallExpression) &&
+        node.getExpression().getText() === "createTest"));
+    if (!createTestNode) {
+        throw new Error("createTest not found in file");
+    }
+    const descendants = createTestNode?.getDescendants();
+    const descendentTexts = descendants.map((d) => d.getText());
+    const parameterSeparatorIdx = descendentTexts.findIndex((s) => s === ",");
+    const pageVariableName = descendentTexts[parameterSeparatorIdx + 1];
+    return pageVariableName;
+}
+exports.getPageVariableNameFromCreateTest = getPageVariableNameFromCreateTest;
 function replaceCreateTestWithNewCode(filePath, contents, generatedCode) {
     const project = new ts_morph_1.Project();
     const sourceFile = project.createSourceFile("test.ts", contents);

package/dist/reporter/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAa5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAkCK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2B9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAUlD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAa5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IA4CK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAUlD"}

package/dist/reporter/index.js

@@ -6,7 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.TestGenUpdatesReporter = exports.setReporterConfig = exports.getReporter = void 0;
 const r2_uploader_1 = require("@empiricalrun/r2-uploader");
 const reporter_1 = require("@empiricalrun/reporter");
-const promises_1 = __importDefault(require("fs/promises"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 const logger_1 = require("../bin/logger");
 const uploader_1 = require("../uploader");
@@ -61,13 +61,23 @@ class TestGenUpdatesReporter {
                 videoUrls,
             };
             await Promise.allSettled([
-                reporter?.report(new reporter_1.ProcessLogMessageBuilder({ message: JSON.stringify(message) })),
-                reporter?.report(new reporter_1.ProcessLogMessageBuilder({
-                    message: JSON.stringify({
-                        type: "trace",
-                        traceFiles,
-                    }),
-                })),
+                ...(videoUrls.length
+                    ? [
+                        reporter?.report(new reporter_1.ProcessLogMessageBuilder({
+                            message: JSON.stringify(message),
+                        })),
+                    ]
+                    : []),
+                ...(traceFiles.length
+                    ? [
+                        reporter?.report(new reporter_1.ProcessLogMessageBuilder({
+                            message: JSON.stringify({
+                                type: "trace",
+                                traceFiles,
+                            }),
+                        })),
+                    ]
+                    : []),
             ]);
         }
         catch (err) {
@@ -77,8 +87,10 @@ class TestGenUpdatesReporter {
     }
     async sendCurrentView(buffer) {
         // upload current screenshot to r2 and report it to reporter
-        await promises_1.default.mkdir((process.cwd(), "gen-assets"));
-        await promises_1.default.writeFile(path_1.default.join(process.cwd(), "gen-assets", `current-view-${Date.now()}.png`), buffer);
+        if (!fs_extra_1.default.existsSync(path_1.default.join(process.cwd(), "gen-assets"))) {
+            await fs_extra_1.default.mkdir((process.cwd(), "gen-assets"));
+        }
+        await fs_extra_1.default.writeFile(path_1.default.join(process.cwd(), "gen-assets", `current-view-${Date.now()}.png`), buffer);
         const uploadDir = (0, uploader_1.getUploadPathForRun)(reporterConfig?.projectRepoName);
         const files = await (0, r2_uploader_1.uploadDirectory)({
             sourceDir: path_1.default.join(process.cwd(), "gen-assets"),
@@ -91,7 +103,7 @@ class TestGenUpdatesReporter {
         await getReporter()?.report(new reporter_1.ProcessLogMessageBuilder({
             message: JSON.stringify({ type: "current-view", url }),
         }));
-        await promises_1.default.rmdir((process.cwd(), "gen-assets"), { recursive: true });
+        await fs_extra_1.default.rmdir((process.cwd(), "gen-assets"), { recursive: true });
     }
     async sendMessage(message) {
         const reporter = getReporter();

package/dist/uploader/index.d.ts

@@ -1,6 +1,7 @@
 export declare const UPLOAD_BUCKET = "test-report";
 export declare const UPLOAD_DOMAIN = "https://reports.empirical.run";
 export declare function getFullUploadPath(filePath: string, uploadDir: string): string;
+export declare function getRelativeUploadPath(filePath: string): string;
 /**
  * Function to upload test results to R2 using the project repo name and test name.
  * This function uploads both the JSON summary of test results and associated video files.

package/dist/uploader/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/uploader/index.ts"],"names":[],"mappings":"AAWA,eAAO,MAAM,aAAa,gBAAgB,CAAC;AAC3C,eAAO,MAAM,aAAa,kCAAkC,CAAC;AAG7D,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,UAUpE;AAED;;;;;;;;;GASG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,eAAe,EACf,QAAQ,GACT,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC;IACV,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB,CAAC,CAmED;AAED,wBAAgB,mBAAmB,CAAC,eAAe,EAAE,MAAM,UAM1D;AAED,wBAAgB,2BAA2B,uBAQ1C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/uploader/index.ts"],"names":[],"mappings":"AAWA,eAAO,MAAM,aAAa,gBAAgB,CAAC;AAC3C,eAAO,MAAM,aAAa,kCAAkC,CAAC;AAG7D,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,UAIpE;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,UAErD;AAED;;;;;;;;;GASG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,eAAe,EACf,QAAQ,GACT,EAAE;IACD,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC;IACV,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB,CAAC,CAwDD;AAED,wBAAgB,mBAAmB,CAAC,eAAe,EAAE,MAAM,UAM1D;AAED,wBAAgB,2BAA2B,uBAQ1C"}

package/dist/uploader/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.checkIfResultsUploadAllowed = exports.getUploadPathForRun = exports.uploadTestResultsUsingPrjRepo = exports.getFullUploadPath = exports.UPLOAD_DOMAIN = exports.UPLOAD_BUCKET = void 0;
+exports.checkIfResultsUploadAllowed = exports.getUploadPathForRun = exports.uploadTestResultsUsingPrjRepo = exports.getRelativeUploadPath = exports.getFullUploadPath = exports.UPLOAD_DOMAIN = exports.UPLOAD_BUCKET = void 0;
 const r2_uploader_1 = require("@empiricalrun/r2-uploader");
 const reporter_1 = require("@empiricalrun/reporter");
 const path_1 = __importDefault(require("path"));
@@ -18,13 +18,14 @@ exports.UPLOAD_DOMAIN = "https://reports.empirical.run"; // domain based on buck
 const uploadId = crypto.randomUUID();
 function getFullUploadPath(filePath, uploadDir) {
     // remove the source dir from the file path - only keep the relative path
-    // old path for data: ~/source-repo/test-results/testName/
-    // new path for data: ~/source-repo/playwright-report/data/testName
-    const relativeFilePath = filePath.replace(path_1.default.join(process.cwd(), PLAYWRIGHT_REPORT_DATA), "");
-    // new relativeFilePath will look like
+    const relativeFilePath = getRelativeUploadPath(filePath);
     return `${exports.UPLOAD_DOMAIN}/${uploadDir}${relativeFilePath}`;
 }
 exports.getFullUploadPath = getFullUploadPath;
+function getRelativeUploadPath(filePath) {
+    return filePath.replace(path_1.default.join(process.cwd(), PLAYWRIGHT_REPORT_DATA), "");
+}
+exports.getRelativeUploadPath = getRelativeUploadPath;
 /**
  * Function to upload test results to R2 using the project repo name and test name.
  * This function uploads both the JSON summary of test results and associated video files.
@@ -36,27 +37,21 @@ exports.getFullUploadPath = getFullUploadPath;
  * @returns {string} returns.summaryUrl - URL of the uploaded summary JSON file.
  */
 async function uploadTestResultsUsingPrjRepo({ projectRepoName, testName, }) {
-    // project repo name is the github repo name
-    // test-generation/<project-name>/<random-uuid>
     const uploadDir = getUploadPathForRun(projectRepoName);
-    // upload test assets
     const files = await (0, r2_uploader_1.uploadDirectory)({
-        // ~/source-repo/playwright-report -> has index.html, data, trace(they are assets for index.html, not sure why they are called trace), and summary.json
-        sourceDir: path_1.default.join(process.cwd(), PLAYWRIGHT_REPORT_DATA), // upload data dir
+        sourceDir: path_1.default.join(process.cwd(), PLAYWRIGHT_REPORT_DATA),
         destinationDir: uploadDir,
         uploadBucket: exports.UPLOAD_BUCKET,
     });
     // upload summary.json
     await (0, r2_uploader_1.uploadDirectory)({
-        // ~/source-repo/playwright-report -> has index.html, data, trace(they are assets for index.html, not sure why they are called trace), and summary.json
-        sourceDir: path_1.default.join(process.cwd(), "playwright-report"), // upload data dir
+        sourceDir: path_1.default.join(process.cwd(), "playwright-report"),
         fileList: [path_1.default.join(process.cwd(), "playwright-report", "summary.json")],
         destinationDir: uploadDir,
         uploadBucket: exports.UPLOAD_BUCKET,
     });
-    const fileNames = Object.keys(files); // fileNames are absolute paths of the input files
-    // old default location: ..../source-repo/test-results/summary.json
-    // new default location: ..../source-repo/playwright-report/summary.json
+    // fileNames are relative upload paths
+    const fileNames = Object.keys(files);
     const defaultLocation = path_1.default.join(process.cwd(), "playwright-report", "summary.json");
     const results = (0, reporter_1.parseJsonReport)(defaultLocation);
     const flatTestsList = (0, reporter_1.getFlattenedTestList)(results.suites);
@@ -68,24 +63,23 @@ async function uploadTestResultsUsingPrjRepo({ projectRepoName, testName, }) {
                     // results array is basically made by retries
                     for (const attachments of test.tests[0].results[0].attachments) {
                         if (attachments.path) {
-                            testAttachmentPaths.push(attachments.path);
+                            testAttachmentPaths.push(getRelativeUploadPath(attachments.path));
                         }
                     }
                 }
             }
         }
     }
-    // current assumption
-    // - test gen will only run on a single spec file
-    // - the video files are of the format - <some-directory>/video.webm
     const videoFiles = fileNames.filter((fileName) => fileName.endsWith(".webm") && testAttachmentPaths.includes(fileName));
     const traceFiles = fileNames.filter((fileName) => fileName.endsWith(".zip") && testAttachmentPaths.includes(fileName));
     return {
-        videoUrls: videoFiles.map((fileName) => getFullUploadPath(fileName, uploadDir)),
-        // uploaded separately
+        videoUrls: videoFiles
+            .map((fileName) => getFullUploadPath(fileName, uploadDir))
+            .filter((url) => !!url),
         summaryUrl: `${exports.UPLOAD_DOMAIN}/${uploadDir}/summary.json`,
-        // summaryUrl: getFullUploadPath("/test-results/summary.json", uploadDir),
-        traceFiles: traceFiles.map((fileName) => getFullUploadPath(fileName, uploadDir)),
+        traceFiles: traceFiles
+            .map((fileName) => getFullUploadPath(fileName, uploadDir))
+            .filter((url) => !!url),
     };
 }
 exports.uploadTestResultsUsingPrjRepo = uploadTestResultsUsingPrjRepo;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.27.6",
+  "version": "0.27.8",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.1",
+    "@empiricalrun/llm": "^0.9.2",
     "@empiricalrun/r2-uploader": "^0.2.0",
     "@empiricalrun/reporter": "^0.18.0"
   },