@empiricalrun/test-gen 0.42.25 → 0.42.27

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.42.27
+
+### Patch Changes
+
+- 9fc1b40: test: increase timeout for lexical test
+- 1f18628: fix: separate out create test and repo edit LLM calls and remove str replace flag
+
+## 0.42.26
+
+### Patch Changes
+
+- 070c16a: feat: config for master agent sanity test suite
+
 ## 0.42.25
 
 ### Patch Changes

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAI7E,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuHxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAI7E,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -164,7 +164,11 @@ function newContentsWithTestOnly(existingContents, originalTestBlock, updatedTes
     }
 }
 async function injectPwLocatorGenerator(page) {
-    const pathToInstalledTestGen = require.resolve(".").split("dist")[0];
+    let pathToInstalledTestGen = require.resolve(".").split("dist")[0];
+    // eslint-disable-next-line turbo/no-undeclared-env-vars
+    if (process.env.RUNNING_BROWSER_TESTS_FOR_TEST_GEN) {
+        pathToInstalledTestGen = process.cwd();
+    }
     const annotateElementPath = path_1.default.join(pathToInstalledTestGen, "dist", "browser-injected-scripts", "annotate-elements.js");
     if (!fs_extra_1.default.existsSync(annotateElementPath)) {
         throw new Error(`annotate-elements.js not found at path: ${annotateElementPath}`);

package/dist/agent/codegen/generate-code-apply-changes.js

@@ -130,7 +130,7 @@ function getPlannerToolCalls() {
 }
 async function getPlanForCodeEditorAgent({ prompt, trace, }) {
     const promptSpan = trace?.span({
-        name: "planner-agent",
+        name: "repo-edit-planner-agent",
         input: {
             prompt,
         },

package/dist/agent/codegen/repo-edit.d.ts

@@ -2,22 +2,20 @@ import { TraceClient } from "@empiricalrun/llm";
 import { ChatCompletionMessageParam } from "openai/resources/index.mjs";
 import { CustomLogger } from "../../bin/logger";
 import { CodeUpdate } from "./types";
-export declare function generateCodeUsingRepoAgent({ task, trace, repoFiles, useStrReplace, logger, }: {
+export declare function generateCodeUsingRepoAgent({ task, trace, repoFiles, logger, }: {
     trace?: TraceClient;
     task: string;
     repoFiles: string;
-    useStrReplace?: boolean;
     logger?: CustomLogger;
 }): Promise<{
     prompt: ChatCompletionMessageParam[];
     agentResponse: string;
     fileChanges: CodeUpdate[];
 }>;
-export declare function repoEditAgent({ trace, task, logger, useStrReplace, }: {
+export declare function repoEditAgent({ trace, task, logger, }: {
     trace?: TraceClient;
     task: string;
     logger?: CustomLogger;
-    useStrReplace?: boolean;
 }): Promise<{
     prompt: ChatCompletionMessageParam[];
     fileChanges: CodeUpdate[];

package/dist/agent/codegen/repo-edit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAGxE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAUhD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAOrC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,KAAK,EACL,SAAS,EACT,aAAa,EACb,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B,CAAC,CAwJD;AAED,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,EACJ,MAAM,EACN,aAAa,GACd,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B,CAAC,CA0ED"}
+{"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAExE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAQhD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAGrC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,KAAK,EACL,SAAS,EAET,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B,CAAC,CA6CD;AAED,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,EACJ,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B,CAAC,CAqCD"}

package/dist/agent/codegen/repo-edit.js

@@ -4,31 +4,25 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.repoEditAgent = exports.generateCodeUsingRepoAgent = void 0;
-const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
-const path_1 = require("path");
 const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
-const constants_1 = require("../../constants");
 const reporter_1 = require("../../reporter");
 const generate_code_apply_changes_1 = require("./generate-code-apply-changes");
-const test_update_feedback_1 = require("./test-update-feedback");
 const utils_1 = require("./utils");
-async function generateCodeUsingRepoAgent({ task, trace, repoFiles, useStrReplace, logger, }) {
+async function generateCodeUsingRepoAgent({ task, trace, repoFiles, logger, }) {
     const repoEditSpan = trace?.span({
         name: "repo-edit",
         input: { task },
     });
-    // TODO: remove this once we test this flow
-    if (useStrReplace) {
-        const prompt = [
-            {
-                role: "system",
-                content: (0, generate_code_apply_changes_1.systemPromptBuilderForRepoEdit)(repoFiles),
-            },
-            {
-                role: "user",
-                content: `
+    const prompt = [
+        {
+            role: "system",
+            content: (0, generate_code_apply_changes_1.systemPromptBuilderForRepoEdit)(repoFiles),
+        },
+        {
+            role: "user",
+            content: `
 You need to work towards completing the task provided to you.
 
 Task: 
@@ -43,127 +37,25 @@ In order to execute the task, FOLLOW BELOW STEPS:
 
 If all the changes are done, respond with "exit" tool call otherwise respond with "change-plan" tool call.
   `,
-            },
-        ];
-        const fileChanges = await (0, generate_code_apply_changes_1.generateCodeAndApplyChanges)({
-            task,
-            trace: repoEditSpan,
-            getRelevantFiles: async () => await (0, context_1.generateTxtForRepository)(),
-            logger,
-        });
-        repoEditSpan?.end({
-            output: { fileChanges },
-        });
-        return {
-            prompt,
-            fileChanges,
-            agentResponse: "",
-        };
-    }
-    const prompt = [
-        {
-            role: "system",
-            content: `
-You are a software test engineer who is given a task to update code in a Playwright test repository. 
-You will be provided with current test files, fixtures and page object models for you to use and update code as per the task provided to you.
-You need to respond with file path and updated code block inside the file.
-
-Directory structure for the repository:
-└── /
-    ├── tests/
-    ├────test-data/
-    │    └── index.ts
-    ├── pages/
-    ├── playwright.config.ts
-    ├── app_knowledge.md
-    ├── .eslintrc.js
-    ├── package.json
-    └── tsconfig.json
-
-Explanation of repository structure:
-- tests: this is a directory where all tests and fixtures are kept. All spec files here end with ".spec.ts" as naming convention. There is an exception for Playwright fixtures file. Fixtures file is named as "fixtures.ts"
-- test-data: this is a directory where all the test data are kept.
-- pages: this is a directory where all reusable page object model methods are kept. Page object model methods are written in a pure functional convention. These methods are reusable methods created to be used across tests. This is also a directory where all the utility methods are kept which can be executed independent of the Playwright tests.
-
-Coding principles and guidelines:
-
-## Code Style and Structure
-- Write concise, maintainable, and technically accurate TypeScript code with relevant examples.
-- Use functional and declarative programming patterns; avoid classes.
-- Use types to describe the shape of data and the behavior of functions.
-- Favour iteration and modularisation to adhere to DRY principles and avoid code duplication.
-- Use descriptive variable names with auxiliary verbs (e.g., isLoading, hasError).
-- Organise files systematically: each file should contain only related content, such as exported components, subcomponents, helpers, static content, and types.
-- Page object models are pure functions and are stateless.
-
-## Naming Conventions
-- Use lowercase with dashes for directories (e.g., components/auth-wizard).
-- Favour named exports for functions.
-
-## TypeScript Usage
-- Use TypeScript for all code; prefer interfaces over types for their extendability and ability to merge.
-- Avoid enums; use maps instead for better type safety and flexibility.
-- Use functional components with TypeScript interfaces.
-
-## Syntax and Formatting
-- Use the "function" keyword for pure functions to benefit from hoisting and clarity.
-
-Here is the list of files:
-${repoFiles}  
-`,
-        },
-        {
-            role: "user",
-            content: `
-Task: ${task}
-
-
-## Follow these instructions before responding with output:
-- Read the code line by line and ensure that achieve the task provided to you
-- Read the dependencies of the code block by scanning through file paths and file provided to you. refer the same file path while responding with output.
-- Focus only on the test case provided and associated JS methods called from the test case.
-- Since the response will be used to search and replace blocks, always respond with output which includes the full lexical scope surrounding the modified code.
-- If there are any updates inside test code block, ensure responding with full test block with unchanged code as well
-- Each code block should contain edits to only one code block in file path
-- DO NOT respond with any backticks or markdown syntax
-- Respond only with file path where the code block to be updated is present, old code block, new code block and a one liner reason for the change
-- Respond with <file_path></file_path>, <old_code_block></old_code_block>, <new_code_block></new_code_block> and <change></change> as xml tags
-- The reason for change should adhere to coding principles provided and review if the updated code is present in the file path mentioned
-- The code change should belong to the right file path
-- The response must start with <file_path>
-- Use the repository structure provided,  whenever needed create directories which are not present
-    `,
         },
     ];
-    const llm = new llm_1.LLM({
-        trace: repoEditSpan,
-        provider: "anthropic",
-        defaultModel: "claude-3-5-sonnet-20240620",
-        providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
-    });
-    const updatedUsageExampleMessage = await llm.createChatCompletion({
-        messages: prompt,
+    const fileChanges = await (0, generate_code_apply_changes_1.generateCodeAndApplyChanges)({
+        task,
         trace: repoEditSpan,
-        traceName: "repo-edit-agent-llm",
-        modelParameters: {
-            ...constants_1.DEFAULT_MODEL_PARAMETERS,
-        },
+        getRelevantFiles: async () => await (0, context_1.generateTxtForRepository)(),
+        logger,
     });
-    const updates = (0, utils_1.extractTestUpdates)(updatedUsageExampleMessage?.content);
     repoEditSpan?.end({
-        output: { updates },
-        input: {
-            task,
-        },
+        output: { fileChanges },
     });
     return {
         prompt,
-        agentResponse: updatedUsageExampleMessage?.content,
-        fileChanges: updates,
+        fileChanges,
+        agentResponse: "",
     };
 }
 exports.generateCodeUsingRepoAgent = generateCodeUsingRepoAgent;
-async function repoEditAgent({ trace, task, logger, useStrReplace, }) {
+async function repoEditAgent({ trace, task, logger, }) {
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
     logger?.log(`Starting repo agent: [trace](${trace?.getTraceUrl()})`);
@@ -172,44 +64,14 @@ async function repoEditAgent({ trace, task, logger, useStrReplace, }) {
         task,
         trace,
         repoFiles: repoFiles,
-        useStrReplace,
         logger,
     });
     const updates = repoAgentOutput.fileChanges;
-    if (!useStrReplace) {
-        const fileUpdateResponses = await (0, utils_1.applyFileChangesForRepoEdit)({
-            trace,
-            fileChanges: updates.filter((f) => f.filePath && fs_extra_1.default.existsSync(f.filePath)),
-        });
-        const errorResponses = fileUpdateResponses.filter((f) => f?.error);
-        if (errorResponses.length > 0) {
-            const updatedFileChanges = await (0, test_update_feedback_1.applyTestUpdateFeedbacks)({
-                trace,
-                oldPrompt: repoAgentOutput.prompt,
-                agentResponse: repoAgentOutput.agentResponse,
-                feedbacks: errorResponses.map((e) => ({
-                    filePath: e?.filePath,
-                    errorMessage: e?.errorMessage,
-                })),
-            });
-            await (0, utils_1.applyFileChangesForRepoEdit)({
-                trace,
-                fileChanges: updatedFileChanges,
-            });
-        }
-        const newFileUpdates = updates.filter((f) => !fs_extra_1.default.existsSync(f.filePath));
-        await Promise.all(newFileUpdates.map((f) => {
-            return (async () => {
-                await fs_extra_1.default.mkdir((0, path_1.dirname)(f.filePath), { recursive: true });
-                await fs_extra_1.default.writeFile(f.filePath, f.newCode, "utf-8");
-            })();
-        }));
-    }
     await (0, utils_1.validateTypesAndFormatCode)({
         validateTypes: true,
         trace,
         testCase: {},
-        fileChanges: updates,
+        filePaths: updates.map((f) => f.filePath),
     });
     await Promise.all(updates.map((f) => {
         return (async () => {

package/dist/agent/codegen/run.d.ts

@@ -1,4 +1,16 @@
 import { TraceClient } from "@empiricalrun/llm";
 import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
+import { CustomLogger } from "../../bin/logger";
+export declare function createTestWithCodeAgent({ testCase, file, repoFiles, trace, }: {
+    testCase: TestCase;
+    file: string;
+    repoFiles: string | undefined;
+    trace?: TraceClient;
+    logger?: CustomLogger;
+}): Promise<{
+    prompt: import("openai/resources/index.mjs").ChatCompletionMessageParam[];
+    agentResponse: string;
+    fileChanges: import("./types").CreateTestCodeUpdate[];
+}>;
 export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[] | void>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAUpC,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAsC5B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAmBhD,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;;;;GAuDA;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CA8D5B"}

package/dist/agent/codegen/run.js

@@ -3,15 +3,66 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTest = void 0;
+exports.generateTest = exports.createTestWithCodeAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../../bin/logger");
 const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
-const repo_edit_1 = require("./repo-edit");
+const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write test cases in a Playwright test repository.\nYou will be provided with current test files, fixtures and page object models for you to use and update code as per the\ntask provided to you.\nYou need to respond with file path, the new test code block to be updated in the file and the reason for the change.\n\nHere is the list of files:\n\n{{repoFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nTask: {{task}}\n\n## Follow these instructions before responding with output:\n- Respond with <file_path></file_path>, <code_block></code_block> and <change></change> as xml tags\n- Only respond with single code_block in the response containing only the test code block to be updated\n- If the test already exists, do not update the file\n{{/section}}";
+const reporter_1 = require("../../reporter");
 const update_flow_1 = require("./update-flow");
 const utils_1 = require("./utils");
+async function createTestWithCodeAgent({ testCase, file, repoFiles, trace, }) {
+    const task = `
+Create a new test case with name '${testCase.name}' at file path '${file}'
+${testCase.suites.length
+        ? `
+Put this test case inside describe blocks named: ${testCase.suites.join(" > ")}`
+        : ""}
+
+In the above test, write code that performs these actions and ONLY these actions:
+${testCase.steps.join("\n")}`;
+    const prompt = (0, llm_1.compilePrompt)(promptTemplate_0, {
+        task,
+        repoFiles,
+    });
+    const createTestSpan = trace?.span({
+        name: "create-test",
+        input: {
+            testCase,
+            repoFiles,
+            testFilePath: file,
+            task,
+        },
+    });
+    const llm = new llm_1.LLM({
+        trace,
+        provider: "anthropic",
+        defaultModel: "claude-3-5-sonnet-20240620",
+        providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
+    });
+    const updatedUsageExampleMessage = await llm.createChatCompletion({
+        messages: prompt,
+        trace,
+        traceName: "create-test-llm",
+        modelParameters: {
+            ...constants_1.DEFAULT_MODEL_PARAMETERS,
+        },
+    });
+    const updates = (0, utils_1.extractCreateTestUpdates)(updatedUsageExampleMessage?.content);
+    createTestSpan?.end({
+        output: { updates },
+    });
+    console.log("Updates: ", updates);
+    return {
+        prompt,
+        agentResponse: updatedUsageExampleMessage?.content,
+        fileChanges: updates,
+    };
+}
+exports.createTestWithCodeAgent = createTestWithCodeAgent;
 async function generateTest(testCase, file, options, trace) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(file)) {
@@ -19,32 +70,48 @@ async function generateTest(testCase, file, options, trace) {
         fs_extra_1.default.createFileSync(file);
     }
     const context = await (0, context_1.contextForGeneration)(file);
-    const { codePrompt, pomPrompt, testFileContent } = context;
+    const { testFileContent } = context;
     const { testBlock } = (0, web_1.getTypescriptTestBlock)({
         scenarioName: testCase?.name,
         content: testFileContent,
         suites: testCase?.suites,
     });
+    // Update the test if it already exists
     const isUpdate = !!testBlock;
     if (isUpdate) {
         logger.log(`Updating the test '${testCase.name}': [View trace](${trace?.getTraceUrl()})`);
         return await (0, update_flow_1.updateTest)(testCase, file, options, true, true, trace);
     }
-    const createTestSpan = trace?.span({
-        name: "create-test",
-        input: {
-            testCase,
-            testFiles: codePrompt,
-            pageFiles: pomPrompt,
-            testFilePath: file,
-        },
-    });
-    const repoAgentResponse = await (0, repo_edit_1.repoEditAgent)({
+    const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
+    void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
+    logger?.log(`Creating a new test case '${testCase.name}': [trace](${trace?.getTraceUrl()})`);
+    // Create the test if it doesn't exist
+    const { prompt: repoFiles } = await (0, context_1.generateTxtForRepository)();
+    const updates = await createTestWithCodeAgent({
+        testCase,
+        file,
+        repoFiles,
         trace,
-        task: (0, utils_1.getTaskForCreateTest)({ testCase, file }),
         logger,
     });
-    createTestSpan?.end({ output: repoAgentResponse.fileChanges });
+    await (0, utils_1.applyFileChangesForCreateTest)({
+        trace,
+        fileChanges: updates.fileChanges,
+        testgenUpdatesReporter,
+    });
+    await (0, utils_1.validateTypesAndFormatCode)({
+        validateTypes: true,
+        trace,
+        testCase: {},
+        filePaths: updates.fileChanges.map((f) => f.filePath),
+    });
+    await Promise.all(updates.fileChanges.map((f) => {
+        return (async () => {
+            if (fs_extra_1.default.existsSync(f.filePath)) {
+                await (0, web_1.lintErrors)(f.filePath);
+            }
+        })();
+    }));
     await (0, llm_1.flushAllTraces)();
 }
 exports.generateTest = generateTest;

package/dist/agent/codegen/types.d.ts

@@ -5,6 +5,11 @@ export type CodeUpdate = {
     newCode: string | undefined;
     reason: string | undefined;
 };
+export type CreateTestCodeUpdate = {
+    filePath: string | undefined;
+    code: string | undefined;
+    reason: string | undefined;
+};
 export type FileUpdateResponse = {
     error: boolean;
     errorMessage: string;

package/dist/agent/codegen/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAGF,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,oBAAY,kBAAkB;IAC5B,WAAW,gBAAgB;IAC3B,WAAW,gBAAgB;CAC5B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAGF,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,oBAAY,kBAAkB;IAC5B,WAAW,gBAAgB;IAC3B,WAAW,gBAAgB;CAC5B"}

package/dist/agent/codegen/update-flow.js

@@ -134,7 +134,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
         validateTypes: validate,
         trace: updateTestSpan,
         testCase,
-        fileChanges,
+        filePaths: fileChanges.map((f) => f.filePath),
         logger,
         testGenOptions: options,
         pomPrompt: pomPrompt,
@@ -280,7 +280,7 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
     await (0, utils_1.validateTypesAndFormatCode)({
         trace,
         testCase,
-        fileChanges,
+        filePaths: fileChanges.map((f) => f.filePath),
         logger,
         testGenOptions: options,
         pomPrompt: pomPrompt,

package/dist/agent/codegen/utils.d.ts

@@ -1,7 +1,8 @@
 import { TraceClient } from "@empiricalrun/llm";
 import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
 import { CustomLogger } from "../../bin/logger";
-import { CodeUpdate, FileUpdateResponse } from "./types";
+import { TestGenUpdatesReporter } from "../../reporter";
+import { CodeUpdate, CreateTestCodeUpdate, FileUpdateResponse } from "./types";
 /**
  *
  * method to extract file path and code updates for the LLM response of update flow
@@ -15,6 +16,18 @@ import { CodeUpdate, FileUpdateResponse } from "./types";
  * }[])}
  */
 export declare function extractTestUpdates(input: string): CodeUpdate[];
+/**
+ *
+ * method to extract file path and code updates for the LLM response of create flow
+ * @export
+ * @param {string} input
+ * @return {*}  {({
+ *   filePath: string | undefined;
+ *   code: string | undefined;
+ *   reason: string | undefined;
+ * }[])}
+ */
+export declare function extractCreateTestUpdates(input: string): CreateTestCodeUpdate[];
 /**
  *
  * method to extract append create test updates
@@ -44,20 +57,16 @@ export declare function extractTestStepsSuggestions(input: string): {
     reason: string;
     methodName: string;
 }[];
-export declare function validateTypesAndFormatCode({ validateTypes, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }: {
+export declare function validateTypesAndFormatCode({ validateTypes, trace, testCase, filePaths, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }: {
     validateTypes?: boolean;
     trace?: TraceClient;
     testCase: TestCase;
-    fileChanges: CodeUpdate[];
+    filePaths: (string | undefined)[];
     logger?: CustomLogger;
     testGenOptions?: TestGenConfigOptions;
     pomPrompt?: string;
     nonSpecFilePrompt?: string;
 }): Promise<void>;
-export declare function getTaskForCreateTest({ testCase, file, }: {
-    testCase: TestCase;
-    file: string;
-}): string;
 export declare function applyFileChangesUsingStrReplace({ trace, fileChanges, logger, }: {
     trace?: TraceClient;
     fileChanges: CodeUpdate[];
@@ -75,20 +84,11 @@ export declare function searchAndReplaceCodeUsingStrReplace({ logger, fileChange
     result: FileUpdateResponse;
     updatedContent: string;
 }>;
-export declare function applyFileChangesForRepoEdit({ trace, fileChanges, logger, }: {
+export declare function applyFileChangesForCreateTest({ trace, fileChanges, testgenUpdatesReporter, }: {
     trace?: TraceClient;
-    fileChanges: {
-        filePath: string | undefined;
-        oldCode: string | undefined;
-        newCode: string | undefined;
-        reason: string | undefined;
-    }[];
-    logger?: CustomLogger;
-}): Promise<{
-    error: boolean;
-    errorMessage: string;
-    filePath: string;
-}[]>;
+    fileChanges: CreateTestCodeUpdate[];
+    testgenUpdatesReporter?: TestGenUpdatesReporter;
+}): Promise<void>;
 export declare function searchAndReplaceCode({ logger, fileChange, }: {
     fileChange: {
         filePath: string | undefined;

package/dist/agent/codegen/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAOhD,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEzD;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,EAAE,CAiB9D;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA6BA;AAED,wBAAgB,oBAAoB,CAAC,EACnC,QAAQ,EACR,IAAI,GACL,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd,UAaA;AAED,wBAAsB,+BAA+B,CAAC,EACpD,KAAK,EACL,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA6ChC;AAED,wBAAsB,mCAAmC,CAAC,EACxD,MAAM,EACN,UAAU,GACX,EAAE;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,kBAAkB,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CAuCD;AAED,wBAAsB,2BAA2B,CAAC,EAChD,KAAK,EACL,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAgDxE;AAED,wBAAsB,oBAAoB,CAAC,EACzC,MAAM,EACN,UAAU,GACX,EAAE;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACnE,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CA6BD;AAED,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAkIxE"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAE/E;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,EAAE,CAiB9D;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,GACZ,oBAAoB,EAAE,CAgBxB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,SAAS,EACT,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC;IAClC,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA0BA;AAED,wBAAsB,+BAA+B,CAAC,EACpD,KAAK,EACL,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA6ChC;AAED,wBAAsB,mCAAmC,CAAC,EACxD,MAAM,EACN,UAAU,GACX,EAAE;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,kBAAkB,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CAuCD;AAED,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,WAAW,EACX,sBAAsB,GACvB,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD,iBAoDA;AAED,wBAAsB,oBAAoB,CAAC,EACzC,MAAM,EACN,UAAU,GACX,EAAE;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACnE,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CA6BD;AAED,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAkIxE"}

package/dist/agent/codegen/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.applyFileChanges = exports.searchAndReplaceCode = exports.applyFileChangesForRepoEdit = exports.searchAndReplaceCodeUsingStrReplace = exports.applyFileChangesUsingStrReplace = exports.getTaskForCreateTest = exports.validateTypesAndFormatCode = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractTestUpdates = void 0;
+exports.applyFileChanges = exports.searchAndReplaceCode = exports.applyFileChangesForCreateTest = exports.searchAndReplaceCodeUsingStrReplace = exports.applyFileChangesUsingStrReplace = exports.validateTypesAndFormatCode = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractCreateTestUpdates = exports.extractTestUpdates = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const ts_morph_1 = require("ts-morph");
@@ -38,6 +38,33 @@ function extractTestUpdates(input) {
     return result;
 }
 exports.extractTestUpdates = extractTestUpdates;
+/**
+ *
+ * method to extract file path and code updates for the LLM response of create flow
+ * @export
+ * @param {string} input
+ * @return {*}  {({
+ *   filePath: string | undefined;
+ *   code: string | undefined;
+ *   reason: string | undefined;
+ * }[])}
+ */
+function extractCreateTestUpdates(input) {
+    const result = [];
+    // TODO: use better structure for this. Do not kill me for this please.
+    const regex = /<file_path>(.*?)<\/file_path>[\s\S]*?<code_block>([\s\S]*?)<\/code_block>[\s\S]*?<change>([\s\S]*?)<\/change>/g;
+    let match;
+    while ((match = regex.exec(input)) !== null) {
+        const [, filePath, code, reason] = match;
+        result.push({
+            filePath: filePath?.trim(),
+            code: code?.trim(),
+            reason: reason?.trim(),
+        });
+    }
+    return result;
+}
+exports.extractCreateTestUpdates = extractCreateTestUpdates;
 /**
  *
  * method to extract append create test updates
@@ -89,9 +116,9 @@ function extractTestStepsSuggestions(input) {
     return result.filter((r) => !!r.filePath && !!r.usageExample);
 }
 exports.extractTestStepsSuggestions = extractTestStepsSuggestions;
-async function validateTypesAndFormatCode({ validateTypes = true, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }) {
-    for (let fileChange of fileChanges) {
-        if (!fileChange.filePath) {
+async function validateTypesAndFormatCode({ validateTypes = true, trace, testCase, filePaths, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }) {
+    for (let filePath of filePaths) {
+        if (!filePath) {
             continue;
         }
         try {
@@ -99,7 +126,7 @@ async function validateTypesAndFormatCode({ validateTypes = true, trace, testCas
                 await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
                     trace,
                     logger,
-                    file: fileChange.filePath,
+                    file: filePath,
                     pomCode: pomPrompt ?? "",
                     nonSpecFileCode: nonSpecFilePrompt ?? "",
                     testCase: testCase,
@@ -107,29 +134,16 @@ async function validateTypesAndFormatCode({ validateTypes = true, trace, testCas
                 });
             }
             trace?.event({ name: "format-file" });
-            await (0, web_1.formatCode)(fileChange.filePath, trace);
-            logger?.success(`${fileChange.filePath} file formatted successfully!`);
+            await (0, web_1.formatCode)(filePath, trace);
+            logger?.success(`${filePath} file formatted successfully!`);
         }
         catch (e) {
-            console.error(`Error while formatting the file ${fileChange.filePath}`, e);
+            console.error(`Error while formatting the file ${filePath}`, e);
         }
     }
     await (0, llm_1.flushAllTraces)();
 }
 exports.validateTypesAndFormatCode = validateTypesAndFormatCode;
-function getTaskForCreateTest({ testCase, file, }) {
-    return `
-Create a new test case with name '${testCase.name}' at file path '${file}'
-${testCase.suites.length
-        ? `
-Put this test case inside describe blocks named: ${testCase.suites.join(" > ")}`
-        : ""}
-
-In the above test, write code that performs these actions and ONLY these actions:
-${testCase.steps.join("\n")}
-  `;
-}
-exports.getTaskForCreateTest = getTaskForCreateTest;
 async function applyFileChangesUsingStrReplace({ trace, fileChanges, logger, }) {
     const repoEditFileChangesSpan = trace?.span({
         name: "repo-edit-file-changes",
@@ -211,11 +225,10 @@ async function searchAndReplaceCodeUsingStrReplace({ logger, fileChange, }) {
     };
 }
 exports.searchAndReplaceCodeUsingStrReplace = searchAndReplaceCodeUsingStrReplace;
-async function applyFileChangesForRepoEdit({ trace, fileChanges, logger, }) {
+async function applyFileChangesForCreateTest({ trace, fileChanges, testgenUpdatesReporter, }) {
     const repoEditFileChangesSpan = trace?.span({
-        name: "repo-edit-file-changes",
+        name: "create-test-file-changes",
     });
-    const results = [];
     for (const fileChange of fileChanges) {
         if (!fileChange.filePath) {
             continue;
@@ -227,33 +240,33 @@ async function applyFileChangesForRepoEdit({ trace, fileChanges, logger, }) {
                     fileChange,
                 },
             });
-            const { result, updatedContent } = await searchAndReplaceCode({
-                logger,
-                fileChange,
-            });
-            if (result.error) {
-                logger?.error(`Unable to find the code to update in ${result.filePath}`);
-            }
-            else {
-                await fs_extra_1.default.writeFile(fileChange.filePath, updatedContent, "utf-8");
-                readWriteFileSpan?.end({ output: { updatedContent } });
+            if (fileChange.code) {
+                const currentContent = fs_extra_1.default.existsSync(fileChange.filePath)
+                    ? await fs_extra_1.default.readFile(fileChange.filePath, "utf-8")
+                    : "";
+                await fs_extra_1.default.writeFile(fileChange.filePath, currentContent
+                    ? `${currentContent}\n\n${fileChange.code}`
+                    : fileChange.code, "utf-8");
+                const updatedFileContent = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
+                readWriteFileSpan?.end({
+                    output: { fileChange, updatedFile: updatedFileContent },
+                });
             }
-            results.push(result);
         }
         catch (e) {
             trace?.event({
-                name: "repo-edit-file-changes-error",
+                name: "create-test-file-changes-error",
                 output: {
                     filePath: fileChange.filePath,
                     error: e,
                 },
             });
+            void testgenUpdatesReporter?.sendMessage(`Error while applying changes to file ${fileChange.filePath}, error: ${e}`);
             console.error(`Error while applying changes to file ${fileChange.filePath}`, e);
         }
     }
-    return results;
 }
-exports.applyFileChangesForRepoEdit = applyFileChangesForRepoEdit;
+exports.applyFileChangesForCreateTest = applyFileChangesForCreateTest;
 async function searchAndReplaceCode({ logger, fileChange, }) {
     let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
     // since we dont know what is getting updated,

package/dist/agent/master/browser-tests/index.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.spec.d.ts.map

package/dist/agent/master/browser-tests/index.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.spec.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/browser-tests/index.spec.ts"],"names":[],"mappings":""}

package/dist/agent/master/browser-tests/index.spec.js

@@ -0,0 +1,65 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const test_1 = require("@playwright/test");
+const http_server_1 = __importDefault(require("http-server"));
+const path_1 = __importDefault(require("path"));
+const run_1 = require("../run");
+let server;
+let PORT = 2345;
+test_1.test.beforeAll(async () => {
+    server = http_server_1.default.createServer({
+        root: path_1.default.join(process.cwd(), "test-data"),
+    });
+    // Start the server
+    await new Promise((resolve) => {
+        server.listen(PORT, () => {
+            console.log(`Server running at http://localhost:${PORT}`);
+            resolve(true);
+        });
+    });
+});
+test_1.test.afterAll(() => {
+    if (server) {
+        server.close();
+    }
+});
+(0, test_1.test)("able to scroll and interact with elements", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `fill test@test.com into the email field and click the submit`,
+        page,
+        options: {},
+    });
+    console.log(response);
+    (0, test_1.expect)(response.importPaths.length).toBe(0);
+    (0, test_1.expect)(response.code).toContain("await page.getByPlaceholder('Enter your email').fill(\"test@test.com\")");
+    (0, test_1.expect)(response.code).toContain("await page.getByRole('button', { name: 'Subscribe' }).click()");
+});
+test_1.test.skip("scroll when element does not exist", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `click search button`,
+        page,
+        options: {},
+    });
+    console.log(response);
+    (0, test_1.expect)(response.importPaths.length).toBe(0);
+    (0, test_1.expect)(response.code.length).toBe(0);
+});
+(0, test_1.test)("agent can click icons accurately", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/icons-navbar.html`);
+    await (0, test_1.expect)(page.getByText("select an icon")).toBeVisible();
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `click on the gear icon`,
+        page,
+        options: {},
+    });
+    await (0, test_1.expect)(page.getByText("you clicked Gear")).toBeVisible();
+    console.log(response);
+    (0, test_1.expect)(response.importPaths.length).toBe(0);
+    (0, test_1.expect)(response.code).toContain("page.getByRole");
+    (0, test_1.expect)(response.code).toContain("click()");
+});

package/dist/bin/index.js

@@ -79,7 +79,6 @@ async function runAgent(testGenConfig) {
             trace,
             task: testGenConfig.testCase.steps.join("\n"),
             logger,
-            useStrReplace: testGenConfig.options?.useStrReplace,
         });
         return;
     }

package/dist/browser-injected-scripts/annotate-elements.spec.js

@@ -180,7 +180,6 @@ const action_tool_calls_1 = require("../agent/master/action-tool-calls");
             href: config.node.href,
         }));
     }, preference);
-    console.log("Annotations:", annotations);
     test_1.test.expect(annotations.length).toBe(1);
     test_1.test.expect(annotations[0]?.tagName).toBe("DIV");
 });

package/dist/browser-injected-scripts/annotate-elements.spec.ts

@@ -296,8 +296,6 @@ test("should only annotate given text on quizziz page", async ({ page }) => {
       href: config.node.href,
     }));
   }, preference);
-
-  console.log("Annotations:", annotations);
   test.expect(annotations.length).toBe(1);
   test.expect(annotations[0]?.tagName).toBe("DIV");
 });

package/dist/evals/add-scenario-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAqCpC,QAAA,MAAM,4BAA4B,EAAE,UAqBnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
+{"version":3,"file":"add-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/add-scenario-agent.evals.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AA2BpC,QAAA,MAAM,4BAA4B,EAAE,UAsBnC,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/add-scenario-agent.evals.js

@@ -4,8 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const path_1 = __importDefault(require("path"));
-const repo_edit_1 = require("../agent/codegen/repo-edit");
-const utils_1 = require("../agent/codegen/utils");
+const run_1 = require("../agent/codegen/run");
 function evaluateEqualityScore({ currentOutput, expectedOutput, }) {
     for (const expectedFile of expectedOutput) {
         if (!currentOutput.find((output) => {
@@ -23,8 +22,9 @@ function evaluateEqualityScore({ currentOutput, expectedOutput, }) {
 const addScenarioCodeAgentEvaluate = async ({ item, trace }) => {
     const { testCase, testFilePath, pageFiles, testFiles } = item.input;
     const repoFiles = pageFiles + testFiles;
-    const repoAgentOutput = await (0, repo_edit_1.generateCodeUsingRepoAgent)({
-        task: (0, utils_1.getTaskForCreateTest)({ testCase, file: testFilePath }),
+    const repoAgentOutput = await (0, run_1.createTestWithCodeAgent)({
+        testCase,
+        file: testFilePath,
         trace,
         repoFiles,
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.42.25",
+  "version": "0.42.27",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -71,22 +71,24 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
+    "@empiricalrun/reporter": "^0.23.1",
     "@empiricalrun/llm": "^0.9.34",
-    "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/reporter": "^0.23.1"
+    "@empiricalrun/r2-uploader": "^0.3.8"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",
     "@types/detect-port": "^1.3.5",
     "@types/express": "^4.17.21",
     "@types/fs-extra": "^11.0.4",
+    "@types/http-server": "^0.12.4",
     "@types/js-levenshtein": "^1.1.3",
     "@types/lodash.isequal": "^4.5.8",
     "@types/md5": "^2.3.5",
+    "http-server": "^14.1.1",
     "js-levenshtein": "^1.1.6",
     "playwright": "1.47.1",
     "ts-patch": "^3.3.0",
-    "@empiricalrun/shared-types": "0.0.2"
+    "@empiricalrun/shared-types": "0.0.3"
   },
   "scripts": {
     "dev": "tspc --build --watch",
@@ -94,7 +96,7 @@
     "clean": "tspc --build --clean",
     "lint": "eslint .",
     "test": "vitest run",
-    "test-browser": "npx playwright test",
+    "test-browser": "pnpm build && RUNNING_BROWSER_TESTS_FOR_TEST_GEN=1 npx playwright test",
     "test:watch": "vitest",
     "test:watch-files": "vitest $0 --watch"
   }

package/playwright.config.ts

@@ -1,5 +1,12 @@
 import { defineConfig } from "@playwright/test";
 
+// To run master agent sanity tests, we want our typescript
+// transformer to be used while compiling the tests.
+// Hence, we bypass Playwright's default TS compiler, and have
+// it run compiled test files from /dist/
 export default defineConfig({
-  testDir: "./src/browser-injected-scripts",
+  testDir: "./dist/.",
+  testMatch: "**/*.spec.js",
+  // Setting large timeout as master agent takes time to run.
+  timeout: 120_000,
 });