@empiricalrun/test-gen 0.47.3 → 0.47.4

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @empiricalrun/test-gen
 
+## 0.47.4
+
+### Patch Changes
+
+- 007ef65: feat: master agent tool call returns git patch on successful runs
+- c4ddc2c: feat: chat can use codegen agent as a tool call
+- e1903b7: feat: add master agent as a tool call for chat
+- 4b18733: chore: simplify interface to prepare for tool call
+- 6328aaf: feat: test-gen can use test-run package as a tool call
+- e4bf218: fix: update append-create-test prompt to emphasize task without location hints
+- 7748c7b: chore: remove options from prepare file workflow
+- 72a8cf2: feat: chat agent only needs prompt string
+- d995e4c: feat: enable conversation along with test-run tool call
+- 848cb88: fix: trace hierarchy in prep for master agent
+- Updated dependencies [72a8cf2]
+- Updated dependencies [242f659]
+- Updated dependencies [6328aaf]
+- Updated dependencies [d995e4c]
+  - @empiricalrun/test-run@0.7.1
+
 ## 0.47.3
 
 ### Patch Changes

package/dist/agent/browsing/run.d.ts

@@ -14,6 +14,9 @@ type GenerateTestsType = {
  *   filePathToUpdate,
  * }
  */
-export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, }: GenerateTestsType): Promise<void>;
+export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, }: GenerateTestsType): Promise<{
+    isError: boolean;
+    error: string;
+}>;
 export {};
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAkBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB,iBA6EnB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB;;;GAkFnB"}

package/dist/agent/browsing/run.js

@@ -20,9 +20,9 @@ const utils_1 = require("./utils");
  * }
  */
 async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, }) {
-    // valiate if the file path and file to update are valid
-    // also warn users if they are on older version of test-gen
-    (0, utils_1.canRunMasterAgent)(testFilePath);
+    if (!fs_extra_1.default.existsSync(testFilePath)) {
+        throw new Error(`File for master agent to run not found: ${testFilePath}`);
+    }
     // detect available http port on the machine
     const port = await (0, detect_port_1.default)(3030);
     // start a file service to handle file updates from agent
@@ -84,5 +84,10 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
         // throw the error because of which test gen failed
         throw Error(error);
     }
+    await fileService.stop();
+    return {
+        isError,
+        error,
+    };
 }
 exports.generateTestsUsingMasterAgent = generateTestsUsingMasterAgent;

package/dist/agent/browsing/utils.d.ts

@@ -1,22 +1,15 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestGenConfig } from "@empiricalrun/shared-types";
+import type { TestCase } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
 import { PlaywrightTestConfig } from "playwright/test";
 export declare function isRegExp(obj: any): obj is RegExp;
 export declare function prepareBrowsingAgentTask(steps: string[]): string;
-/**
- * Function to prepare test file for master agent to run
- * @param {TestGenConfig} genConfig
- * @return {*}  {Promise<string>}
- */
-export declare function prepareFileForMasterAgent(genConfig: TestGenConfig, trace?: TraceClient): Promise<string>;
+export declare function prepareFileForMasterAgent({ testCase, specPath, trace, }: {
+    testCase: TestCase;
+    specPath: string;
+    trace?: TraceClient;
+}): Promise<string>;
 export declare function injectPwLocatorGenerator(page: Page): Promise<void>;
-/**
- * Function to validate if the test file path are valid.
- * @throws if there are any missing dependencies for master agent to run.
- * @param {string} filePath
- */
-export declare function canRunMasterAgent(filePath: string): void;
 /**
  * function to read playwright config from the source repo
  * @return {*}  {Promise<PlaywrightTestConfig>}

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAI7E,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAwGD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CAM/B;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAIxE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA+FD,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CAM/B;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TeardownManager = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunMasterAgent = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
+exports.TeardownManager = exports.detectProjectName = exports.readPlaywrightConfig = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const minimatch_1 = require("minimatch");
 const path_1 = __importDefault(require("path"));
@@ -37,12 +37,7 @@ async function addImportForCreateTest(testFilePath) {
     }
     await fs_extra_1.default.writeFile(testFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(testFilePath, "utf-8"), ["createTest"], importSource));
 }
-/**
- * Function to prepare test file for update scenarios for master agent to run
- * @param {TestGenConfig} genConfig
- */
-async function prepareFileForUpdateScenario(genConfig, trace) {
-    const { specPath, testCase } = genConfig;
+async function prepareFileForUpdateScenario({ testCase, specPath, trace, }) {
     const { name, suites } = testCase;
     await (0, web_1.addUserContextFixture)({
         scenarioName: name,
@@ -55,7 +50,6 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
         testCase,
         file: specPath,
         validateTypes: false,
-        options: genConfig.options,
     });
     const createTestFilePath = suggestion?.updatedFiles[0] || "";
     console.log("appending to existing test block");
@@ -78,15 +72,7 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
         },
     });
     await (0, web_1.appendScopeToCreateTest)(createTestFilePath, scopeVariables);
-    // extract the createTest function and pass the scoped vars
-    const updateFileSpan = trace?.span({
-        name: "update-file-span",
-        input: {
-            createTestFilePath,
-        },
-    });
     await addImportForCreateTest(createTestFilePath);
-    updateFileSpan?.end();
     const { pomPrompt, nonSpecFilePrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
     await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
         trace,
@@ -94,7 +80,6 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
         pomCode: pomPrompt,
         nonSpecFileCode: nonSpecFilePrompt,
         testCase: testCase,
-        options: genConfig.options,
     });
     const testFileContent = await fs_extra_1.default.readFile(specPath, "utf-8");
     const { testBlock, testNode } = (0, web_1.getTypescriptTestBlock)({
@@ -111,17 +96,11 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
     }
     return createTestFilePath;
 }
-/**
- * Function to prepare test file for master agent to run
- * @param {TestGenConfig} genConfig
- * @return {*}  {Promise<string>}
- */
-async function prepareFileForMasterAgent(genConfig, trace) {
+async function prepareFileForMasterAgent({ testCase, specPath, trace, }) {
     const prepareFileSpan = trace?.span({
         name: "prepare-file-for-master-agent",
     });
     new logger_1.CustomLogger({ useReporter: true }).log(`Preparing file for master agent. [view trace](${trace?.getTraceUrl()})`);
-    const { specPath, testCase } = genConfig;
     const { name, suites } = testCase;
     // check if the spec file exists
     // if no then create a new file with test and expect imports
@@ -144,9 +123,8 @@ async function prepareFileForMasterAgent(genConfig, trace) {
     if (!testBlock) {
         const newTestBlock = await (0, create_test_block_1.createEmptyTestCaseBlock)({
             trace: prepareFileSpan,
-            testCase: genConfig.testCase,
+            testCase,
             file: specPath,
-            options: genConfig.options,
         });
         await fs_extra_1.default.writeFile(specPath, `${existingContents} \n\n ${newTestBlock}`, "utf-8");
         const updatedContent = (0, web_1.injectCodeSnippetBySuiteChain)({
@@ -156,7 +134,11 @@ async function prepareFileForMasterAgent(genConfig, trace) {
         });
         await fs_extra_1.default.writeFile(specPath, updatedContent, "utf-8");
     }
-    const updatePath = await prepareFileForUpdateScenario(genConfig, prepareFileSpan);
+    const updatePath = await prepareFileForUpdateScenario({
+        testCase,
+        specPath,
+        trace,
+    });
     return updatePath;
 }
 exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
@@ -280,17 +262,6 @@ async function injectPwLocatorGenerator(page) {
     }
 }
 exports.injectPwLocatorGenerator = injectPwLocatorGenerator;
-/**
- * Function to validate if the test file path are valid.
- * @throws if there are any missing dependencies for master agent to run.
- * @param {string} filePath
- */
-function canRunMasterAgent(filePath) {
-    if (!fs_extra_1.default.existsSync(filePath)) {
-        throw new Error(`File for master agent to run not found: ${filePath}`);
-    }
-}
-exports.canRunMasterAgent = canRunMasterAgent;
 /**
  * function to read playwright config from the source repo
  * @return {*}  {Promise<PlaywrightTestConfig>}

package/dist/agent/chat.d.ts

@@ -0,0 +1,7 @@
+import { TraceClient } from "@empiricalrun/llm";
+import { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
+export declare function chatAgent({ prompt, }: {
+    prompt: string;
+    trace?: TraceClient;
+}): Promise<ChatCompletionMessageParam[]>;
+//# sourceMappingURL=chat.d.ts.map

package/dist/agent/chat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,0BAA0B,EAE3B,MAAM,uCAAuC,CAAC;AAsC/C,wBAAsB,SAAS,CAAC,EAC9B,MAAM,GACP,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,yCA6DA"}

package/dist/agent/chat.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chatAgent = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const human_in_the_loop_1 = require("../human-in-the-loop");
+const browser_agent_1 = require("../tools/browser-agent");
+const codegen_agent_1 = require("../tools/codegen-agent");
+const test_run_1 = require("../tools/test-run");
+const systemPrompt = `
+You are a helpful assistant that can answer questions and help with tasks.
+You are given a set to tools to use to fulfill the user's request.
+
+For example, if the user asks you to run a test, you should use the runTest tool.
+Once the test is run, you will receive the results in the form of a JSON object.
+Summarize the results in a few sentences.
+
+Or if the user asks you to modify a test, you should use the
+generateTestWithCodegen or the generateTestWithBrowserAgent tool. Read their
+descriptions to understand what each tool does.
+`;
+const getToolExecutor = (toolName) => {
+    if (toolName === "runTest") {
+        return test_run_1.runTestTool;
+    }
+    else if (toolName === "generateTestWithCodegen") {
+        return codegen_agent_1.generateTestWithCodegenTool;
+    }
+    else if (toolName === "generateTestWithBrowserAgent") {
+        return browser_agent_1.generateTestWithBrowserAgentTool;
+    }
+    else {
+        throw new Error(`Tool name ${toolName} not found`);
+    }
+};
+async function chatAgent({ prompt, }) {
+    let userPrompt = prompt;
+    let chatHistory = [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: userPrompt },
+    ];
+    const llm = new llm_1.LLM({ provider: "openai" });
+    let response;
+    let shouldAskUserForInput = false;
+    let pendingToolCalls = [];
+    while (!userPrompt.toLowerCase().includes("stop")) {
+        if (pendingToolCalls.length > 0) {
+            const toolCall = pendingToolCalls.shift();
+            if (!toolCall) {
+                throw new Error("No tool call found");
+            }
+            console.log("Executing tool call:", toolCall.function.name, "with args:", toolCall.function.arguments);
+            const toolExecutor = getToolExecutor(toolCall.function.name);
+            const toolArgs = JSON.parse(toolCall.function.arguments);
+            const toolResult = await toolExecutor(toolArgs);
+            chatHistory.push({
+                role: "tool",
+                content: JSON.stringify(toolResult),
+                tool_call_id: toolCall.id,
+            });
+            continue;
+        }
+        if (shouldAskUserForInput) {
+            userPrompt = await human_in_the_loop_1.humanLoop.getFeedback({
+                message: "Your response?",
+            });
+            chatHistory.push({ role: "user", content: userPrompt });
+            shouldAskUserForInput = false;
+            continue;
+        }
+        response = await llm.createChatCompletion({
+            model: "gpt-4o",
+            messages: chatHistory,
+            tools: [test_run_1.schema, codegen_agent_1.schema, browser_agent_1.schema],
+        });
+        if (!response) {
+            throw new Error("No response from LLM");
+        }
+        chatHistory.push(response);
+        if (response.tool_calls) {
+            pendingToolCalls.push(...response.tool_calls);
+        }
+        else if (response.content) {
+            console.log("Assistant response:", response.content);
+            shouldAskUserForInput = true;
+        }
+    }
+    return chatHistory;
+}
+exports.chatAgent = chatAgent;

package/dist/agent/codegen/create-test-block.d.ts

@@ -1,9 +1,8 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
-export declare function createEmptyTestCaseBlock({ testCase, file, options, trace, }: {
+import type { TestCase } from "@empiricalrun/shared-types";
+export declare function createEmptyTestCaseBlock({ testCase, file, trace, }: {
     testCase: TestCase;
     file: string;
-    options?: TestGenConfigOptions;
     trace?: TraceClient;
 }): Promise<string | undefined>;
 //# sourceMappingURL=create-test-block.d.ts.map

package/dist/agent/codegen/create-test-block.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAcpC,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAqDA"}
+{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAc3D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BA+CA"}

package/dist/agent/codegen/create-test-block.js

@@ -8,7 +8,7 @@ const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write an empty test block.\nBased on the inputs you need to create an empty playwright test block with correctly imported fixture.\n\nThe test will contain a test name which you will need to use to build the empty test case block.\n\nYou will be provided with current tests, fixtures and page object models for you to use and create test case block as\nper the task provided to you.\n\nBefore responding you need to ensure that the code change is minimal and the change is reusable across tests. You need\nto ensure the code follows DRY principle.\n\nHere is the list of current tests and fixtures:\n\n{{testFiles}}\n\nHere is the list of current page object models:\n\n{{pageFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nFollowing is the test scenario for which you need to write the empty test case block:\ntest name:\n{{scenarioName}}\n\ntask:\ncreate an empty test case block for the following test steps:\n{{scenario}}\n\ntest file path: {{scenarioFile}}\n\n------\n\nYou also need to ensure that the empty test case block has a starting page to begin test.\n\nIn order to identify the right page with which the test should start, follow the steps:\n- based on the similarities with other test cases mentioned in the file, identify the right page fixture to be imported\n- Read the page fixture methods step by step. Identify whether the fixture handles navigating to a page.\n- Identify whether other tests using the page fixture had to add separate steps for navigation or not\n- Based on the above analysis there will be following cases and choose either for the given test scenario:\n-- Case 1: if the test case scenario provided inside the task mentions about page navigation, then use that page\nnavigation. skip other cases if this case is satisfied.\n-- Case 2: refer other test cases which import similar fixtures and infer the first page navigation of this test case.\nYou should prefer tests which are in the same file. Tests within same file have higher overlaps in first page\nnavigation.\n- Once the page fixture is decided, look for userContext fixture in files. If its available then add \"userContext\" to\nthe test case block\n\n\n\nFollow these instructions before responding with output:\n- Read the code line by line and achieve the task provided to you\n- Read the dependencies of the code block by scanning through file paths and file provided to you. refer the same file\npath while responding with update\n- Focus only on the test case provided and associated JS methods called from the test case.\n- Respond only with the new empty test case block to be created and nothing else.\n- DO NOT respond with any backticks or markdown syntax\n- If \"userContext\" fixture is available in fixtures file, ensure importing that fixture in the test case block.\n- Provide a reason based on the test steps provided to you on why you chose the fixture or page.goto statement. The\nreason should be one of the list steps provided to you and mention why the case was chosen\n{{/section}}";
 const session_1 = require("../../session");
-async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
+async function createEmptyTestCaseBlock({ testCase, file, trace, }) {
     const logger = new logger_1.CustomLogger({ useReporter: false });
     logger.log("Creating new test block");
     const context = await (0, context_1.contextForGeneration)(file);
@@ -20,10 +20,6 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
                 name: "create-empty-test-block",
                 id: crypto.randomUUID(),
                 release: session.version,
-                tags: [
-                    options?.metadata.projectName || "",
-                    options?.metadata.environment || "",
-                ].filter((s) => !!s),
             });
     const promptSpan = trace?.span({
         name: "build-create-empty-test-case-prompt",
@@ -38,15 +34,14 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
     promptSpan?.end({ output: { prompt } });
     const llm = new llm_1.LLM({
         trace,
-        provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-        defaultModel: options?.model || constants_1.DEFAULT_MODEL,
-        providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+        provider: constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: constants_1.DEFAULT_MODEL,
+        providerApiKey: constants_1.MODEL_API_KEYS[constants_1.DEFAULT_MODEL_PROVIDER],
     });
     const firstShotMessage = await llm.createChatCompletion({
         messages: prompt,
         modelParameters: {
             ...constants_1.DEFAULT_MODEL_PARAMETERS,
-            ...options?.modelParameters,
         },
     });
     const markdownRemoverSpan = trace?.span({

package/dist/agent/codegen/fix-ts-errors.d.ts

@@ -1,13 +1,12 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
+import type { TestCase } from "@empiricalrun/shared-types";
 import { CustomLogger } from "../../bin/logger";
-export declare function validateAndFixTypescriptErrors({ trace, logger, file, pomCode, nonSpecFileCode, testCase, options, }: {
+export declare function validateAndFixTypescriptErrors({ trace, logger, file, pomCode, nonSpecFileCode, testCase, }: {
     trace?: TraceClient;
     logger?: CustomLogger;
     file: string;
     pomCode: string;
     nonSpecFileCode: string;
     testCase: TestCase;
-    options?: TestGenConfigOptions;
 }): Promise<void>;
 //# sourceMappingURL=fix-ts-errors.d.ts.map

package/dist/agent/codegen/fix-ts-errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAUhD,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,OAAO,EACP,eAAe,EACf,QAAQ,EACR,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,iBAwEA"}
+{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAUhD,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,OAAO,EACP,eAAe,EACf,QAAQ,GACT,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;CACpB,iBAsEA"}

package/dist/agent/codegen/fix-ts-errors.js

@@ -10,7 +10,7 @@ const logger_1 = require("../../bin/logger");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software engineer who is given a task to fix semantic and syntactical errors in a typescript file\n'{{scenarioFile}}' provided to you.\nYou will be provided with fixtures and page object models to use and fix errors.\n\nHere is the list of fixtures available:\n\n{{fixtureFiles}}\n\nHere is the list of current page object models available to you:\n\n{{pageFiles}}\n\nUse the above files to fix the errors.\n\nFollow these guidelines before responding with output\n- Ensure there are no type issues in the given {{scenarioFile}} file\n- For the given file respond with only the code\n- Do not respond with markdown syntax or backticks\n- Do not modify anything else apart from the code required to fix typescript error\n- Do not modify any other scenarios apart from the provided scenario name\n- Do not respond with any explanation. Respond only with the updated code.\n{{/section}}\n\n{{#section \"user\"}}\n'{{scenarioFile}}' with scenario name '{{scenarioName}}', has following typescript errors which you need to fix:\n{{errors}}\n\nHere is the content of the '{{scenarioFile}}':\n\n{{fileContent}}\n{{/section}}";
-async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.CustomLogger(), file, pomCode, nonSpecFileCode, testCase, options, }) {
+async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.CustomLogger(), file, pomCode, nonSpecFileCode, testCase, }) {
     const validateTypesSpan = trace?.span({ name: "detect-type-errors-in-file" });
     logger.log("Validating types...");
     let errors = (0, web_1.validateTypescript)(file);
@@ -46,15 +46,14 @@ async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.Cus
         promptSpan?.end({ output: { instruction } });
         const llm = new llm_1.LLM({
             trace,
-            provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-            defaultModel: options?.model || constants_1.DEFAULT_MODEL,
-            providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+            provider: constants_1.DEFAULT_MODEL_PROVIDER,
+            defaultModel: constants_1.DEFAULT_MODEL,
+            providerApiKey: constants_1.MODEL_API_KEYS[constants_1.DEFAULT_MODEL_PROVIDER],
         });
         const message = await llm.createChatCompletion({
             messages: instruction,
             modelParameters: {
                 ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                ...options?.modelParameters,
             },
         });
         const response = message?.content || "";

package/dist/agent/codegen/run.d.ts

@@ -1,18 +1,20 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
+import type { TestCase } from "@empiricalrun/shared-types";
 import OpenAI from "openai";
-import { CustomLogger } from "../../bin/logger";
 import { CreateTestCodeUpdate } from "./types";
 export declare function createTestWithCodeAgent({ testCase, file, repoFiles, trace, }: {
     testCase: TestCase;
     file: string;
     repoFiles: string | undefined;
     trace?: TraceClient;
-    logger?: CustomLogger;
 }): Promise<{
     prompt: OpenAI.Chat.Completions.ChatCompletionMessageParam[];
     agentResponse: string;
     fileChanges: CreateTestCodeUpdate[];
 }>;
-export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[] | void>;
+export declare function generateTestWithCodegen({ testCase, file, trace, }: {
+    testCase: TestCase;
+    file: string;
+    trace?: TraceClient;
+}): Promise<TestCase[] | void>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAYhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAQ/C,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC;IAC7D,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,oBAAoB,EAAE,CAAC;CACrC,CAAC,CAyDD;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CA8D5B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAc5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAQ/C,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC;IAC7D,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,oBAAoB,EAAE,CAAC;CACrC,CAAC,CAyDD;AAED,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAgE7B"}

package/dist/agent/codegen/run.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTest = exports.createTestWithCodeAgent = void 0;
+exports.generateTestWithCodegen = exports.createTestWithCodeAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../../bin/logger");
@@ -65,7 +65,7 @@ ${testCase.steps.join("\n")}`;
     };
 }
 exports.createTestWithCodeAgent = createTestWithCodeAgent;
-async function generateTest(testCase, file, options, trace) {
+async function generateTestWithCodegen({ testCase, file, trace, }) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(file)) {
         logger.log(`Creating a new spec file: ${file}`);
@@ -82,7 +82,11 @@ async function generateTest(testCase, file, options, trace) {
     const isUpdate = !!testBlock;
     if (isUpdate) {
         logger.log(`Updating the test '${testCase.name}': [View trace](${trace?.getTraceUrl()})`);
-        return await (0, update_flow_1.updateTest)(testCase, file, options, true, true, trace);
+        return await (0, update_flow_1.updateTest)({
+            testCase,
+            file,
+            trace,
+        });
     }
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
@@ -94,7 +98,6 @@ async function generateTest(testCase, file, options, trace) {
         file,
         repoFiles,
         trace,
-        logger,
     });
     await (0, utils_1.applyFileChangesForCreateTest)({
         trace,
@@ -114,6 +117,5 @@ async function generateTest(testCase, file, options, trace) {
             }
         })();
     }));
-    await (0, llm_1.flushAllTraces)();
 }
-exports.generateTest = generateTest;
+exports.generateTestWithCodegen = generateTestWithCodegen;

package/dist/agent/codegen/update-flow.d.ts

@@ -15,19 +15,21 @@ export declare function getUpdateTestCodeCompletion({ testCase, testFileContent,
     agentResponse: string;
     fileChanges: CodeUpdate[];
 }>;
-export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean, validate?: boolean, trace?: TraceClient): Promise<UpdatedTestCase[]>;
-export declare function getAppendCreateTestBlockCompletion({ testFiles, pageFiles, testCase, testFilePath, options, trace, }: {
+export declare function updateTest({ testCase, file, trace, }: {
+    testCase: TestCase;
+    file: string;
+    trace?: TraceClient;
+}): Promise<UpdatedTestCase[]>;
+export declare function getAppendCreateTestBlockCompletion({ testFiles, pageFiles, testCase, testFilePath, trace, }: {
     trace?: TraceClient;
     pageFiles?: string;
     testFiles: string;
     testCase: TestCase;
-    options?: TestGenConfigOptions;
     testFilePath: string;
 }): Promise<string>;
-export declare function appendCreateTestBlock({ testCase, file, options, trace, validateTypes, }: {
+export declare function appendCreateTestBlock({ testCase, file, trace, validateTypes, }: {
     testCase: TestCase;
     file: string;
-    options?: TestGenConfigOptions;
     trace?: TraceClient;
     validateTypes?: boolean;
 }): Promise<UpdatedTestCase[]>;

package/dist/agent/codegen/update-flow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAexE,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAQtD,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B,CAAC,CAkDD;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA+F5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBAmGA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAqD7B"}
+{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAcxE,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAQtD,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B,CAAC,CAkDD;AAED,wBAAsB,UAAU,CAAC,EAC/B,QAAQ,EACR,IAAI,EACJ,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA8E7B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,mBAkFA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA8C7B"}