@empiricalrun/test-gen 0.47.3 → 0.48.0

package/CHANGELOG.md

@@ -1,5 +1,46 @@
 # @empiricalrun/test-gen
 
+## 0.48.0
+
+### Minor Changes
+
+- fc952c9: feat: Add new tool call to fetch diagnosis details
+
+### Patch Changes
+
+- 32eaf6f: feat: add repo tree in ascii to chat system prompt
+- 6cc97ca: feat: use claude3.7 in chat agent for diagnosis auto-fix scenarios
+- 2e28c20: fix: master agent should not throw when used in tool call
+- f03ff97: feat: update master agent tool call to use TODO(agent) preparation method
+- ed273c2: fix: disable skills usage in master agent
+- 049102a: fix: support suites in agent tool calls
+- Updated dependencies [6cc97ca]
+- Updated dependencies [0fee9bf]
+- Updated dependencies [1f95e4b]
+- Updated dependencies [049102a]
+  - @empiricalrun/llm@0.10.0
+  - @empiricalrun/test-run@0.7.2
+
+## 0.47.4
+
+### Patch Changes
+
+- 007ef65: feat: master agent tool call returns git patch on successful runs
+- c4ddc2c: feat: chat can use codegen agent as a tool call
+- e1903b7: feat: add master agent as a tool call for chat
+- 4b18733: chore: simplify interface to prepare for tool call
+- 6328aaf: feat: test-gen can use test-run package as a tool call
+- e4bf218: fix: update append-create-test prompt to emphasize task without location hints
+- 7748c7b: chore: remove options from prepare file workflow
+- 72a8cf2: feat: chat agent only needs prompt string
+- d995e4c: feat: enable conversation along with test-run tool call
+- 848cb88: fix: trace hierarchy in prep for master agent
+- Updated dependencies [72a8cf2]
+- Updated dependencies [242f659]
+- Updated dependencies [6328aaf]
+- Updated dependencies [d995e4c]
+  - @empiricalrun/test-run@0.7.1
+
 ## 0.47.3
 
 ### Patch Changes

package/dist/agent/browsing/run.d.ts

@@ -5,15 +5,9 @@ type GenerateTestsType = {
     testGenToken: string;
     repoDir: string;
 };
-/**
- *
- * Function to generate tests using master agent
- * @export
- * @param {GenerateTestsType} {
- *   testFilePath,
- *   filePathToUpdate,
- * }
- */
-export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, }: GenerateTestsType): Promise<void>;
+export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, }: GenerateTestsType): Promise<{
+    isError: boolean;
+    error: string;
+}>;
 export {};
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAkBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB,iBA6EnB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB;;;GA8EnB"}

package/dist/agent/browsing/run.js

@@ -10,19 +10,10 @@ const web_1 = require("../../bin/utils/platform/web");
 const server_1 = require("../../file/server");
 const exec_1 = require("../../utils/exec");
 const utils_1 = require("./utils");
-/**
- *
- * Function to generate tests using master agent
- * @export
- * @param {GenerateTestsType} {
- *   testFilePath,
- *   filePathToUpdate,
- * }
- */
 async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, }) {
-    // valiate if the file path and file to update are valid
-    // also warn users if they are on older version of test-gen
-    (0, utils_1.canRunMasterAgent)(testFilePath);
+    if (!fs_extra_1.default.existsSync(testFilePath)) {
+        throw new Error(`File for master agent to run not found: ${testFilePath}`);
+    }
     // detect available http port on the machine
     const port = await (0, detect_port_1.default)(3030);
     // start a file service to handle file updates from agent
@@ -45,7 +36,8 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
         await teardowns.skipAll();
     }
     const command = `npx playwright test ${testFilePath} --retries 0 --project ${project} --timeout 0 --headed`;
-    let isError = false, error = "";
+    let isError = false;
+    let error = "";
     try {
         await (0, exec_1.cmd)(command.split(" "), {
             env: {
@@ -80,9 +72,10 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
     }
     // remove the test only from the file
     await (0, web_1.removeTestOnly)(testFilePath);
-    if (isError) {
-        // throw the error because of which test gen failed
-        throw Error(error);
-    }
+    await fileService.stop();
+    return {
+        isError,
+        error,
+    };
 }
 exports.generateTestsUsingMasterAgent = generateTestsUsingMasterAgent;

package/dist/agent/browsing/utils.d.ts

@@ -1,22 +1,25 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestGenConfig } from "@empiricalrun/shared-types";
+import type { TestCase } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
 import { PlaywrightTestConfig } from "playwright/test";
 export declare function isRegExp(obj: any): obj is RegExp;
 export declare function prepareBrowsingAgentTask(steps: string[]): string;
-/**
- * Function to prepare test file for master agent to run
- * @param {TestGenConfig} genConfig
- * @return {*}  {Promise<string>}
- */
-export declare function prepareFileForMasterAgent(genConfig: TestGenConfig, trace?: TraceClient): Promise<string>;
+export declare function replaceTodoWithCreateTest({ testFilePath, testCaseName, testCaseSuites, }: {
+    testFilePath: string;
+    testCaseName: string;
+    testCaseSuites: string[];
+}): Promise<void>;
+export declare function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }: {
+    testCaseName: string;
+    testCaseSuites: string[];
+    specPath: string;
+}): Promise<void>;
+export declare function prepareFileForMasterAgent({ testCase, specPath, trace, }: {
+    testCase: TestCase;
+    specPath: string;
+    trace?: TraceClient;
+}): Promise<string>;
 export declare function injectPwLocatorGenerator(page: Page): Promise<void>;
-/**
- * Function to validate if the test file path are valid.
- * @throws if there are any missing dependencies for master agent to run.
- * @param {string} filePath
- */
-export declare function canRunMasterAgent(filePath: string): void;
 /**
  * function to read playwright config from the source repo
  * @return {*}  {Promise<PlaywrightTestConfig>}

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAI7E,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAwGD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CAM/B;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAIxE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAiFD,wBAAsB,yBAAyB,CAAC,EAC9C,YAAY,EACZ,YAAY,EACZ,cAAc,GACf,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B,iBAkBA;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CAM/B;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TeardownManager = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunMasterAgent = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
+exports.TeardownManager = exports.detectProjectName = exports.readPlaywrightConfig = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.markTestAsOnly = exports.replaceTodoWithCreateTest = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const minimatch_1 = require("minimatch");
 const path_1 = __importDefault(require("path"));
@@ -37,12 +37,7 @@ async function addImportForCreateTest(testFilePath) {
     }
     await fs_extra_1.default.writeFile(testFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(testFilePath, "utf-8"), ["createTest"], importSource));
 }
-/**
- * Function to prepare test file for update scenarios for master agent to run
- * @param {TestGenConfig} genConfig
- */
-async function prepareFileForUpdateScenario(genConfig, trace) {
-    const { specPath, testCase } = genConfig;
+async function prepareFileForUpdateScenario({ testCase, specPath, trace, }) {
     const { name, suites } = testCase;
     await (0, web_1.addUserContextFixture)({
         scenarioName: name,
@@ -55,7 +50,6 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
         testCase,
         file: specPath,
         validateTypes: false,
-        options: genConfig.options,
     });
     const createTestFilePath = suggestion?.updatedFiles[0] || "";
     console.log("appending to existing test block");
@@ -78,15 +72,7 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
         },
     });
     await (0, web_1.appendScopeToCreateTest)(createTestFilePath, scopeVariables);
-    // extract the createTest function and pass the scoped vars
-    const updateFileSpan = trace?.span({
-        name: "update-file-span",
-        input: {
-            createTestFilePath,
-        },
-    });
     await addImportForCreateTest(createTestFilePath);
-    updateFileSpan?.end();
     const { pomPrompt, nonSpecFilePrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
     await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
         trace,
@@ -94,13 +80,33 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
         pomCode: pomPrompt,
         nonSpecFileCode: nonSpecFilePrompt,
         testCase: testCase,
-        options: genConfig.options,
     });
+    await markTestAsOnly({
+        testCaseName: name,
+        testCaseSuites: suites,
+        specPath,
+    });
+    return createTestFilePath;
+}
+async function replaceTodoWithCreateTest({ testFilePath, testCaseName, testCaseSuites, }) {
+    // This method is an alternative to prepareFileForUpdateScenario
+    // TODO: Does not support multiple pages, scoped variables, updates in POM files
+    const fileContent = await fs_extra_1.default.readFile(testFilePath, "utf-8");
+    await fs_extra_1.default.writeFile(testFilePath, fileContent.replace(/\/\/ TODO\(agent\): (.*)/, (_, todoText) => `await createTest("${todoText.replace(/"/g, '\\"')}", page)`));
+    await addImportForCreateTest(testFilePath);
+    await markTestAsOnly({
+        testCaseName,
+        testCaseSuites,
+        specPath: testFilePath,
+    });
+}
+exports.replaceTodoWithCreateTest = replaceTodoWithCreateTest;
+async function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }) {
     const testFileContent = await fs_extra_1.default.readFile(specPath, "utf-8");
     const { testBlock, testNode } = (0, web_1.getTypescriptTestBlock)({
-        scenarioName: name,
+        scenarioName: testCaseName,
         content: testFileContent,
-        suites,
+        suites: testCaseSuites,
     });
     const parentDescribe = (0, web_1.findFirstSerialDescribeBlock)(testNode);
     const isFileMarkedSerial = await (0, web_1.hasTopLevelDescribeConfigureWithSerialMode)(specPath);
@@ -109,19 +115,13 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
         const updatedTestFileContent = newContentsWithTestOnly(testFileContent, testBlock, testBlock, parentDescribe?.getText() || "");
         await fs_extra_1.default.writeFile(specPath, updatedTestFileContent);
     }
-    return createTestFilePath;
 }
-/**
- * Function to prepare test file for master agent to run
- * @param {TestGenConfig} genConfig
- * @return {*}  {Promise<string>}
- */
-async function prepareFileForMasterAgent(genConfig, trace) {
+exports.markTestAsOnly = markTestAsOnly;
+async function prepareFileForMasterAgent({ testCase, specPath, trace, }) {
     const prepareFileSpan = trace?.span({
         name: "prepare-file-for-master-agent",
     });
     new logger_1.CustomLogger({ useReporter: true }).log(`Preparing file for master agent. [view trace](${trace?.getTraceUrl()})`);
-    const { specPath, testCase } = genConfig;
     const { name, suites } = testCase;
     // check if the spec file exists
     // if no then create a new file with test and expect imports
@@ -144,9 +144,8 @@ async function prepareFileForMasterAgent(genConfig, trace) {
     if (!testBlock) {
         const newTestBlock = await (0, create_test_block_1.createEmptyTestCaseBlock)({
             trace: prepareFileSpan,
-            testCase: genConfig.testCase,
+            testCase,
             file: specPath,
-            options: genConfig.options,
         });
         await fs_extra_1.default.writeFile(specPath, `${existingContents} \n\n ${newTestBlock}`, "utf-8");
         const updatedContent = (0, web_1.injectCodeSnippetBySuiteChain)({
@@ -156,7 +155,11 @@ async function prepareFileForMasterAgent(genConfig, trace) {
         });
         await fs_extra_1.default.writeFile(specPath, updatedContent, "utf-8");
     }
-    const updatePath = await prepareFileForUpdateScenario(genConfig, prepareFileSpan);
+    const updatePath = await prepareFileForUpdateScenario({
+        testCase,
+        specPath,
+        trace,
+    });
     return updatePath;
 }
 exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
@@ -280,17 +283,6 @@ async function injectPwLocatorGenerator(page) {
     }
 }
 exports.injectPwLocatorGenerator = injectPwLocatorGenerator;
-/**
- * Function to validate if the test file path are valid.
- * @throws if there are any missing dependencies for master agent to run.
- * @param {string} filePath
- */
-function canRunMasterAgent(filePath) {
-    if (!fs_extra_1.default.existsSync(filePath)) {
-        throw new Error(`File for master agent to run not found: ${filePath}`);
-    }
-}
-exports.canRunMasterAgent = canRunMasterAgent;
 /**
  * function to read playwright config from the source repo
  * @return {*}  {Promise<PlaywrightTestConfig>}

package/dist/agent/chat.d.ts

@@ -0,0 +1,7 @@
+import { TraceClient } from "@empiricalrun/llm";
+import type { Anthropic } from "@empiricalrun/llm/claude";
+export declare function chatAgent({ prompt, }: {
+    prompt: string;
+    trace?: TraceClient;
+}): Promise<Anthropic.Messages.MessageParam[]>;
+//# sourceMappingURL=chat.d.ts.map

package/dist/agent/chat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AA6D1D,wBAAsB,SAAS,CAAC,EAC9B,MAAM,GACP,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,8CAmEA"}

package/dist/agent/chat.js

@@ -0,0 +1,111 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chatAgent = void 0;
+const claude_1 = require("@empiricalrun/llm/claude");
+const path_1 = __importDefault(require("path"));
+const human_in_the_loop_1 = require("../human-in-the-loop");
+const browser_agent_1 = require("../tools/browser-agent");
+const diagnosis_fetcher_1 = require("../tools/diagnosis-fetcher");
+const test_run_1 = require("../tools/test-run");
+const repo_tree_1 = require("../utils/repo-tree");
+const systemPrompt = `
+You are a helpful assistant that can answer questions and help with tasks.
+You are given a set of tools to use to fulfill the user's request. Read their descriptions to
+understand what each tool does.
+
+For example, if the user asks you to run a test, you could use the runTest tool.
+Once the test is run, you will receive the results in the form of a JSON object.
+Summarize the results in a few sentences.
+
+If the user provides a diagnosis URL, you can use the fetchDiagnosisDetails tool
+to get more information about the test case and its results.
+
+Or if the user asks you to modify a test, you could use the generateTestWithBrowserAgent tool. If you suspect
+that a UI selector needs to be updated, using the browser agent is a good idea.
+
+Before using generateTestWithBrowserAgent, you need to prepare the test code for the browser agent.
+You can do this by using the str_replace_editor tool to add a TODO comment to the test code. This 
+comment should explain to the browser agent what to do.
+
+For example, if the expected modification is to click on a login button, you could add the following comment. 
+
+// TODO(agent): Click on the login button
+
+The position of the comment is important: the browser agent will look for this comment and replace it with 
+the actual code to click on the login button. If you are fixing a failing test, your comment should be 
+around the failing line of code, so that it can be replaced/modified.
+
+You are running as a CLI tool inside the directory of the repo where this test file is located. Here is
+the repo directory structure:
+
+${(0, repo_tree_1.generateAsciiTree)(process.cwd())}
+
+While specifying paths to files, use relative paths from the current working directory. For example:
+- Correct path: "tests/lesson.spec.ts"
+- Incorrect path: "/repo/tests/lesson.spec.ts" or "${path_1.default.basename(process.cwd())}/tests/lesson.spec.ts"
+`;
+const tools = [test_run_1.runTestTool, browser_agent_1.browserAgentTool, diagnosis_fetcher_1.diagnosisTool];
+const toolExecutors = {
+    ...Object.fromEntries(tools.map((tool) => [tool.schema.function.name, tool.execute])),
+    str_replace_editor: claude_1.strReplaceEditorTool,
+};
+async function chatAgent({ prompt, }) {
+    let userPrompt = prompt;
+    let chatState = new claude_1.ChatState();
+    chatState.pushTextMessage({ message: { role: "user", content: userPrompt } });
+    let shouldAskUserForInput = false;
+    while (!userPrompt.toLowerCase().includes("stop")) {
+        const toolUse = chatState.getPendingToolCall();
+        if (toolUse) {
+            console.log("Executing tool:", toolUse.name, "with args:", toolUse.input);
+            const toolExecutor = toolExecutors[toolUse.name];
+            if (!toolExecutor) {
+                throw new Error(`Tool ${toolUse.name} not found`);
+            }
+            const toolResult = await toolExecutor(toolUse.input);
+            chatState.pushToolResultToMessages({
+                toolCall: toolUse,
+                isError: toolResult.isError,
+                result: toolResult.result,
+            });
+            continue;
+        }
+        if (shouldAskUserForInput) {
+            userPrompt = await human_in_the_loop_1.humanLoop.getFeedback({
+                message: "Your response?",
+            });
+            chatState.pushTextMessage({
+                message: { role: "user", content: userPrompt },
+            });
+            shouldAskUserForInput = false;
+            continue;
+        }
+        const response = await (0, claude_1.createChatCompletion)(systemPrompt, chatState.getMessages(), [
+            ...tools.map((tool) => (0, claude_1.convertOpenAISchemaToAnthropic)(tool.schema)),
+            {
+                type: "text_editor_20250124",
+                name: "str_replace_editor",
+            },
+        ]);
+        if (!response) {
+            throw new Error("No response from LLM");
+        }
+        chatState.pushTextMessage({
+            message: { role: "assistant", content: response.content },
+        });
+        const textBlock = response.content.find((b) => b.type === "text");
+        const toolUseBlock = response.content.find((b) => b.type === "tool_use");
+        console.log("Assistant response:", textBlock?.text);
+        if (toolUseBlock) {
+            chatState.addPendingToolCall({ toolCall: toolUseBlock });
+        }
+        else {
+            shouldAskUserForInput = true;
+        }
+    }
+    return chatState.getMessages();
+}
+exports.chatAgent = chatAgent;

package/dist/agent/codegen/create-test-block.d.ts

@@ -1,9 +1,8 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
-export declare function createEmptyTestCaseBlock({ testCase, file, options, trace, }: {
+import type { TestCase } from "@empiricalrun/shared-types";
+export declare function createEmptyTestCaseBlock({ testCase, file, trace, }: {
     testCase: TestCase;
     file: string;
-    options?: TestGenConfigOptions;
     trace?: TraceClient;
 }): Promise<string | undefined>;
 //# sourceMappingURL=create-test-block.d.ts.map

package/dist/agent/codegen/create-test-block.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAcpC,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAqDA"}
+{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAc3D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BA+CA"}

package/dist/agent/codegen/create-test-block.js

@@ -8,7 +8,7 @@ const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write an empty test block.\nBased on the inputs you need to create an empty playwright test block with correctly imported fixture.\n\nThe test will contain a test name which you will need to use to build the empty test case block.\n\nYou will be provided with current tests, fixtures and page object models for you to use and create test case block as\nper the task provided to you.\n\nBefore responding you need to ensure that the code change is minimal and the change is reusable across tests. You need\nto ensure the code follows DRY principle.\n\nHere is the list of current tests and fixtures:\n\n{{testFiles}}\n\nHere is the list of current page object models:\n\n{{pageFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nFollowing is the test scenario for which you need to write the empty test case block:\ntest name:\n{{scenarioName}}\n\ntask:\ncreate an empty test case block for the following test steps:\n{{scenario}}\n\ntest file path: {{scenarioFile}}\n\n------\n\nYou also need to ensure that the empty test case block has a starting page to begin test.\n\nIn order to identify the right page with which the test should start, follow the steps:\n- based on the similarities with other test cases mentioned in the file, identify the right page fixture to be imported\n- Read the page fixture methods step by step. Identify whether the fixture handles navigating to a page.\n- Identify whether other tests using the page fixture had to add separate steps for navigation or not\n- Based on the above analysis there will be following cases and choose either for the given test scenario:\n-- Case 1: if the test case scenario provided inside the task mentions about page navigation, then use that page\nnavigation. skip other cases if this case is satisfied.\n-- Case 2: refer other test cases which import similar fixtures and infer the first page navigation of this test case.\nYou should prefer tests which are in the same file. Tests within same file have higher overlaps in first page\nnavigation.\n- Once the page fixture is decided, look for userContext fixture in files. If its available then add \"userContext\" to\nthe test case block\n\n\n\nFollow these instructions before responding with output:\n- Read the code line by line and achieve the task provided to you\n- Read the dependencies of the code block by scanning through file paths and file provided to you. refer the same file\npath while responding with update\n- Focus only on the test case provided and associated JS methods called from the test case.\n- Respond only with the new empty test case block to be created and nothing else.\n- DO NOT respond with any backticks or markdown syntax\n- If \"userContext\" fixture is available in fixtures file, ensure importing that fixture in the test case block.\n- Provide a reason based on the test steps provided to you on why you chose the fixture or page.goto statement. The\nreason should be one of the list steps provided to you and mention why the case was chosen\n{{/section}}";
 const session_1 = require("../../session");
-async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
+async function createEmptyTestCaseBlock({ testCase, file, trace, }) {
     const logger = new logger_1.CustomLogger({ useReporter: false });
     logger.log("Creating new test block");
     const context = await (0, context_1.contextForGeneration)(file);
@@ -20,10 +20,6 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
                 name: "create-empty-test-block",
                 id: crypto.randomUUID(),
                 release: session.version,
-                tags: [
-                    options?.metadata.projectName || "",
-                    options?.metadata.environment || "",
-                ].filter((s) => !!s),
             });
     const promptSpan = trace?.span({
         name: "build-create-empty-test-case-prompt",
@@ -38,15 +34,14 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
     promptSpan?.end({ output: { prompt } });
     const llm = new llm_1.LLM({
         trace,
-        provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-        defaultModel: options?.model || constants_1.DEFAULT_MODEL,
-        providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+        provider: constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: constants_1.DEFAULT_MODEL,
+        providerApiKey: constants_1.MODEL_API_KEYS[constants_1.DEFAULT_MODEL_PROVIDER],
     });
     const firstShotMessage = await llm.createChatCompletion({
         messages: prompt,
         modelParameters: {
             ...constants_1.DEFAULT_MODEL_PARAMETERS,
-            ...options?.modelParameters,
         },
     });
     const markdownRemoverSpan = trace?.span({

package/dist/agent/codegen/fix-ts-errors.d.ts

@@ -1,13 +1,12 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
+import type { TestCase } from "@empiricalrun/shared-types";
 import { CustomLogger } from "../../bin/logger";
-export declare function validateAndFixTypescriptErrors({ trace, logger, file, pomCode, nonSpecFileCode, testCase, options, }: {
+export declare function validateAndFixTypescriptErrors({ trace, logger, file, pomCode, nonSpecFileCode, testCase, }: {
     trace?: TraceClient;
     logger?: CustomLogger;
     file: string;
     pomCode: string;
     nonSpecFileCode: string;
     testCase: TestCase;
-    options?: TestGenConfigOptions;
 }): Promise<void>;
 //# sourceMappingURL=fix-ts-errors.d.ts.map

package/dist/agent/codegen/fix-ts-errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAUhD,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,OAAO,EACP,eAAe,EACf,QAAQ,EACR,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,iBAwEA"}
+{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAUhD,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,OAAO,EACP,eAAe,EACf,QAAQ,GACT,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;CACpB,iBAsEA"}

package/dist/agent/codegen/fix-ts-errors.js

@@ -10,7 +10,7 @@ const logger_1 = require("../../bin/logger");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software engineer who is given a task to fix semantic and syntactical errors in a typescript file\n'{{scenarioFile}}' provided to you.\nYou will be provided with fixtures and page object models to use and fix errors.\n\nHere is the list of fixtures available:\n\n{{fixtureFiles}}\n\nHere is the list of current page object models available to you:\n\n{{pageFiles}}\n\nUse the above files to fix the errors.\n\nFollow these guidelines before responding with output\n- Ensure there are no type issues in the given {{scenarioFile}} file\n- For the given file respond with only the code\n- Do not respond with markdown syntax or backticks\n- Do not modify anything else apart from the code required to fix typescript error\n- Do not modify any other scenarios apart from the provided scenario name\n- Do not respond with any explanation. Respond only with the updated code.\n{{/section}}\n\n{{#section \"user\"}}\n'{{scenarioFile}}' with scenario name '{{scenarioName}}', has following typescript errors which you need to fix:\n{{errors}}\n\nHere is the content of the '{{scenarioFile}}':\n\n{{fileContent}}\n{{/section}}";
-async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.CustomLogger(), file, pomCode, nonSpecFileCode, testCase, options, }) {
+async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.CustomLogger(), file, pomCode, nonSpecFileCode, testCase, }) {
     const validateTypesSpan = trace?.span({ name: "detect-type-errors-in-file" });
     logger.log("Validating types...");
     let errors = (0, web_1.validateTypescript)(file);
@@ -46,15 +46,14 @@ async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.Cus
         promptSpan?.end({ output: { instruction } });
         const llm = new llm_1.LLM({
             trace,
-            provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-            defaultModel: options?.model || constants_1.DEFAULT_MODEL,
-            providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+            provider: constants_1.DEFAULT_MODEL_PROVIDER,
+            defaultModel: constants_1.DEFAULT_MODEL,
+            providerApiKey: constants_1.MODEL_API_KEYS[constants_1.DEFAULT_MODEL_PROVIDER],
         });
         const message = await llm.createChatCompletion({
             messages: instruction,
             modelParameters: {
                 ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                ...options?.modelParameters,
             },
         });
         const response = message?.content || "";

package/dist/agent/codegen/run.d.ts

@@ -1,18 +1,20 @@
 import { TraceClient } from "@empiricalrun/llm";
-import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
+import type { TestCase } from "@empiricalrun/shared-types";
 import OpenAI from "openai";
-import { CustomLogger } from "../../bin/logger";
 import { CreateTestCodeUpdate } from "./types";
 export declare function createTestWithCodeAgent({ testCase, file, repoFiles, trace, }: {
     testCase: TestCase;
     file: string;
     repoFiles: string | undefined;
     trace?: TraceClient;
-    logger?: CustomLogger;
 }): Promise<{
     prompt: OpenAI.Chat.Completions.ChatCompletionMessageParam[];
     agentResponse: string;
     fileChanges: CreateTestCodeUpdate[];
 }>;
-export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[] | void>;
+export declare function generateTestWithCodegen({ testCase, file, trace, }: {
+    testCase: TestCase;
+    file: string;
+    trace?: TraceClient;
+}): Promise<TestCase[] | void>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAYhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAQ/C,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC;IAC7D,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,oBAAoB,EAAE,CAAC;CACrC,CAAC,CAyDD;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CA8D5B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAc5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAQ/C,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC;IAC7D,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,oBAAoB,EAAE,CAAC;CACrC,CAAC,CAyDD;AAED,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAgE7B"}

package/dist/agent/codegen/run.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTest = exports.createTestWithCodeAgent = void 0;
+exports.generateTestWithCodegen = exports.createTestWithCodeAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../../bin/logger");
@@ -65,7 +65,7 @@ ${testCase.steps.join("\n")}`;
     };
 }
 exports.createTestWithCodeAgent = createTestWithCodeAgent;
-async function generateTest(testCase, file, options, trace) {
+async function generateTestWithCodegen({ testCase, file, trace, }) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(file)) {
         logger.log(`Creating a new spec file: ${file}`);
@@ -82,7 +82,11 @@ async function generateTest(testCase, file, options, trace) {
     const isUpdate = !!testBlock;
     if (isUpdate) {
         logger.log(`Updating the test '${testCase.name}': [View trace](${trace?.getTraceUrl()})`);
-        return await (0, update_flow_1.updateTest)(testCase, file, options, true, true, trace);
+        return await (0, update_flow_1.updateTest)({
+            testCase,
+            file,
+            trace,
+        });
     }
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
@@ -94,7 +98,6 @@ async function generateTest(testCase, file, options, trace) {
         file,
         repoFiles,
         trace,
-        logger,
     });
     await (0, utils_1.applyFileChangesForCreateTest)({
         trace,
@@ -114,6 +117,5 @@ async function generateTest(testCase, file, options, trace) {
             }
         })();
     }));
-    await (0, llm_1.flushAllTraces)();
 }
-exports.generateTest = generateTest;
+exports.generateTestWithCodegen = generateTestWithCodegen;