@empiricalrun/test-gen 0.57.2 → 0.59.0

package/CHANGELOG.md

@@ -1,5 +1,74 @@
 # @empiricalrun/test-gen
 
+## 0.59.0
+
+### Minor Changes
+
+- e2b0318: feat: dashboard session migration from CLI working
+- 894c5b5: fix: stricter types for tool call args in canonical message
+- 919e7df: fix: session create, duplicate user message, add Model Options
+- 045e370: feat: stateless build download for tool execution
+- f2d5859: feat: canonical migration for claude and openai
+- e2b0318: feat: migrate dashboard chat sessions to canonical chat state
+- 4d3b46a: feat: migrate to new chat state version in cli run
+
+### Patch Changes
+
+- 72f4577: feat: stop marking files as only and move to cmd builder approach
+- 82b3373: test: remove irrelevant or flaky tests
+- 3dbd4ff: fix: remove teardown skip since we interrupt agent with SIGINT
+- 7c77600: refactor: move test-gen file changes to prepare for removal
+- 562c25d: fix: update chat agent system prompt for using try/catch
+- 43656bf: refactor: split tool call execute into 2 methods
+- 51a9975: fix: unit tests for test-gen tool call
+- 772b93c: feat: gemini message transformer into canonical message type
+- 0c935ce: chore: chat model types are moved to shared-types pkg
+- 050fbd3: fix: model selection is ignored while creating a new session
+- ddfd009: fix: refactor chat agent and session list UI
+- 28682e2: feat: use test-run cmd builder in test-gen, remove test.only marking
+- 6b2ed6d: chore: use function args to tell if tool execution is remote
+- 511dbb1: test: skip tests for the old master agent
+- 7145e34: test: add new agent-loop test for openai canonical model
+- 43fb70f: feat: add retry functionality for chat sessions and update ChatAgent …
+- d1d1055: feat: integrate askUserForInput state into session handling
+- ef491c0: fix: incorrect build url getter
+- Updated dependencies [72f4577]
+- Updated dependencies [97cef16]
+- Updated dependencies [e2b0318]
+- Updated dependencies [772b93c]
+- Updated dependencies [894c5b5]
+- Updated dependencies [0c935ce]
+- Updated dependencies [919e7df]
+- Updated dependencies [045e370]
+- Updated dependencies [f2d5859]
+- Updated dependencies [919e7df]
+- Updated dependencies [ddfd009]
+- Updated dependencies [e2b0318]
+- Updated dependencies [28682e2]
+- Updated dependencies [b415e29]
+- Updated dependencies [09291a8]
+- Updated dependencies [aab69ae]
+- Updated dependencies [4d3b46a]
+- Updated dependencies [bc5996d]
+  - @empiricalrun/test-run@0.9.0
+  - @empiricalrun/llm@0.16.0
+
+## 0.58.0
+
+### Minor Changes
+
+- 8c71fd1: feat: pass file info to system prompt to construct repo context
+
+### Patch Changes
+
+- 637c97f: chore: moved model creation logic from test-gen to llm
+- Updated dependencies [0634233]
+- Updated dependencies [637c97f]
+- Updated dependencies [21b0768]
+  - @empiricalrun/test-run@0.8.5
+  - @empiricalrun/llm@0.15.4
+  - @empiricalrun/r2-uploader@0.3.9
+
 ## 0.57.2
 
 ### Patch Changes

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAEL,UAAU,EACV,YAAY,EAEb,MAAM,UAAU,CAAC;AAWlB,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,UAAU,EAChB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IA0D9B,wBAAwB,IAAI,YAAY,EAAE;IAoB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAEL,UAAU,EACV,YAAY,EAEb,MAAM,UAAU,CAAC;AAWlB,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,EAAE,MAAM,YAAK,EACjB,IAAI,EAAE,UAAU,EAChB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IA0D9B,wBAAwB,IAAI,YAAY,EAAE;IAoB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}

package/dist/actions/skill.js

@@ -3,7 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.skillActionGenerator = exports.extractSkillFromArgs = exports.testCaseSkills = exports.SKILL_USAGE = void 0;
+exports.skillActionGenerator = exports.testCaseSkills = exports.SKILL_USAGE = void 0;
+exports.extractSkillFromArgs = extractSkillFromArgs;
 const promises_1 = __importDefault(require("fs/promises"));
 const api_1 = __importDefault(require("tsx/cjs/api"));
 const use_skill_1 = require("../agent/codegen/use-skill");
@@ -28,7 +29,6 @@ function extractSkillFromArgs(args) {
     const [skillDetails] = availableSkills.filter((skillDetails) => skillDetails.testStep === skill);
     return { skillDetails };
 }
-exports.extractSkillFromArgs = extractSkillFromArgs;
 const skillActionGenerator = (page, options) => {
     const availableSkills = exports.testCaseSkills.getAvailableSkills();
     return {

package/dist/actions/utils/index.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPageVarName = exports.getPlaywrightLocatorUsingCssSelector = void 0;
+exports.getPlaywrightLocatorUsingCssSelector = getPlaywrightLocatorUsingCssSelector;
+exports.getPageVarName = getPageVarName;
 async function getPlaywrightLocatorUsingCssSelector(cssSelector, xpath, page, elementAnnotation) {
     // TODO: analyse other solutions than just css. Also find other solutions to support :has-text -> prompting ?
     // jquery doesnt support :has-text. neither css. Only playwright locator supports this selector though.
@@ -101,9 +102,7 @@ async function getPlaywrightLocatorUsingCssSelector(cssSelector, xpath, page, el
         elementAnnotation,
     });
 }
-exports.getPlaywrightLocatorUsingCssSelector = getPlaywrightLocatorUsingCssSelector;
 function getPageVarName() {
     // eslint-disable-next-line turbo/no-undeclared-env-vars
     return process.env.PAGE_VAR_NAME || "page";
 }
-exports.getPageVarName = getPageVarName;

package/dist/agent/browsing/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.executeTaskUsingBrowsingAgent = void 0;
+exports.executeTaskUsingBrowsingAgent = executeTaskUsingBrowsingAgent;
 const llm_1 = require("@empiricalrun/llm");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a browser automation agent who is given a task to generate code for navigation and assertion. This task is your\ngoal and you must achieve it.\n\nYou will be provided with already executed actions and basis that you need to pick the next step to achieve the task.\nRemember that the goal must be achieved.\n\nYou will be provided with the web page snapshot in the form of Document Object Model. Based on the goal and available\ntool calls you need to pick the appropriate tool call.\n\nInstructions:\n- Take actions one at a time. Do not try to take multiple actions\n- You can respond with multiple assertions in one shot\n- Do not repeat the same actions again otherwise your response will be marked INVALID\n- Avoid repeating errors which we got while executing the last action\n- Stick to the task provided to you and mark the task done once the task is complete\n- Do not execute any action which is not mentioned in the task\n- Do not repeat actions which are already executed more than twice otherwise your response will be marked INVALID\n- Always refer to \"Executed actions\" before deciding your next action for completion of the task.\n- End the task done if all actions required for task are executed\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\nCurrent page snapshot:\n{{pageSnapshot}}\n{{/section}}";
@@ -64,4 +64,3 @@ async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace
     toolCallsSpan?.end({ output: { toolCalls, generatedCodeSteps } });
     return generatedCodeSteps;
 }
-exports.executeTaskUsingBrowsingAgent = executeTaskUsingBrowsingAgent;

package/dist/agent/browsing/run.d.ts

@@ -1,4 +1,6 @@
 type GenerateTestsType = {
+    testCaseName: string;
+    testCaseSuites: string[];
     testFilePath: string;
     filePathToUpdate: string;
     pwProjectsFilter: string[];
@@ -7,7 +9,7 @@ type GenerateTestsType = {
     traceId?: string;
     editFileWithGeneratedCode: boolean;
 };
-export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, traceId, editFileWithGeneratedCode, }: GenerateTestsType): Promise<{
+export declare function generateTestsUsingMasterAgent({ testCaseName, testCaseSuites, testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, traceId, editFileWithGeneratedCode, }: GenerateTestsType): Promise<{
     isError: boolean;
     error: string;
     actionsSummary?: string;

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,EACP,OAAO,EACP,yBAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC,CAuFD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAcA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,EACP,OAAO,EACP,yBAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC,CAsFD"}

package/dist/agent/browsing/run.js

@@ -3,14 +3,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTestsUsingMasterAgent = void 0;
+exports.generateTestsUsingMasterAgent = generateTestsUsingMasterAgent;
+const test_run_1 = require("@empiricalrun/test-run");
 const detect_port_1 = __importDefault(require("detect-port"));
 const fs_1 = __importDefault(require("fs"));
 const web_1 = require("../../bin/utils/platform/web");
 const server_1 = require("../../file/server");
 const exec_1 = require("../../utils/exec");
 const utils_1 = require("./utils");
-async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, traceId, editFileWithGeneratedCode, }) {
+async function generateTestsUsingMasterAgent({ testCaseName, testCaseSuites, testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, traceId, editFileWithGeneratedCode, }) {
     if (!fs_1.default.existsSync(testFilePath)) {
         throw new Error(`File for master agent to run not found: ${testFilePath}`);
     }
@@ -32,24 +33,27 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
     const project = await (0, utils_1.detectProjectName)(testFilePath, playwrightConfig, pwProjectsFilter);
     const pageVar = await (0, web_1.getPageVariableNameFromCreateTest)(filePathToUpdate);
     console.log(`Detected playwright project name: ${project}`);
-    // run playwright test which will internally run the master agent
-    const teardownFileRegex = /.*\.teardown\.ts/;
-    const testsDirectory = `${repoDir}/tests`;
-    const isTestRunTriggeredForTeardown = teardownFileRegex.test(testFilePath);
-    const teardowns = new utils_1.TeardownManager(testsDirectory);
-    let removeListeners;
-    const command = `npx playwright test ${testFilePath} --retries 0 --project ${project} --timeout 0`;
-    let isError = false;
-    let error = "";
+    const testsDir = "tests";
+    const cmd = await (0, test_run_1.runSpecificTestsCmd)({
+        tests: [
+            {
+                name: testCaseName,
+                dir: testsDir,
+                filePath: testFilePath,
+                suites: testCaseSuites,
+            },
+        ],
+        projects: [project],
+        passthroughArgs: "--retries 0 --timeout 0",
+        //@ts-ignore
+        platform: "web",
+    });
+    let error = undefined;
     try {
-        if (!isTestRunTriggeredForTeardown) {
-            removeListeners = await teardowns.skipAll();
-        }
-        await pm.execute(command.split(" "), {
+        await pm.execute(cmd.command, cmd.args, {
             env: {
                 IPC_FILE_SERVICE_PORT: availablePort.toString(),
                 PW_TEST_HTML_REPORT_OPEN: "never",
-                // pass the test gen token so that the agent has the same configuration as cli
                 TEST_GEN_TOKEN: testGenToken,
                 PAGE_VAR_NAME: pageVar || "page",
                 DISPLAY: ":99",
@@ -60,15 +64,9 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
     catch (e) {
         error = `Failed to generate test using master agent ${e}`;
         console.error(error);
-        isError = true;
-    }
-    finally {
-        // Remove process listeners before unskipping files
-        removeListeners?.();
-        teardowns.unskipAll();
     }
-    // clean up the file if there is any error
-    if (isError) {
+    if (error) {
+        // clean up the file if there is any error
         try {
             const fileContent = fs_1.default.readFileSync(filePathToUpdate, "utf-8");
             const updatedContent = (0, web_1.replaceCreateTestWithNewCode)(filePathToUpdate, fileContent, "");
@@ -83,9 +81,8 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
     await (0, web_1.removeTestOnly)(testFilePath);
     await fileServer.stop();
     return {
-        isError,
-        error,
+        isError: !!error,
+        error: error || "",
         actionsSummary: fileServer.getActionsSummary(),
     };
 }
-exports.generateTestsUsingMasterAgent = generateTestsUsingMasterAgent;

package/dist/agent/browsing/utils.d.ts

@@ -4,11 +4,7 @@ import { Page } from "playwright";
 import { PlaywrightTestConfig } from "playwright/test";
 export declare function isRegExp(obj: any): obj is RegExp;
 export declare function prepareBrowsingAgentTask(steps: string[]): string;
-export declare function replaceTodoWithCreateTest({ testFilePath, testCaseName, testCaseSuites, }: {
-    testFilePath: string;
-    testCaseName: string;
-    testCaseSuites: string[];
-}): Promise<void>;
+export declare function replaceTodoWithCreateTest(testFilePath: string): Promise<void>;
 export declare function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }: {
     testCaseName: string;
     testCaseSuites: string[];
@@ -33,13 +29,4 @@ export declare function getValidProjectNames(playwrightConfig: PlaywrightTestCon
  * @returns
  */
 export declare function detectProjectName(testFilePath: string, playwrightConfig: PlaywrightTestConfig, pwProjectsFilter?: string[]): Promise<string>;
-export declare class TeardownManager {
-    private directory;
-    constructor(directory: string);
-    private teardownFiles;
-    private getAllTeardownFiles;
-    private skipTeardownFile;
-    skipAll(): Promise<() => void>;
-    unskipAll(): void;
-}
 //# sourceMappingURL=utils.d.ts.map

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAIxE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAwBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAiFD,wBAAsB,yBAAyB,CAAC,EAC9C,YAAY,EACZ,YAAY,EACZ,cAAc,GACf,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B,iBA2BA;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CA2B/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAoBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAiFD,wBAAsB,yBAAyB,CAAC,YAAY,EAAE,MAAM,iBAsBnE;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CA2B/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB"}

package/dist/agent/browsing/utils.js

@@ -3,16 +3,22 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TeardownManager = exports.detectProjectName = exports.getValidProjectNames = exports.readPlaywrightConfig = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.markTestAsOnly = exports.replaceTodoWithCreateTest = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
+exports.isRegExp = isRegExp;
+exports.prepareBrowsingAgentTask = prepareBrowsingAgentTask;
+exports.replaceTodoWithCreateTest = replaceTodoWithCreateTest;
+exports.markTestAsOnly = markTestAsOnly;
+exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
+exports.injectPwLocatorGenerator = injectPwLocatorGenerator;
+exports.readPlaywrightConfig = readPlaywrightConfig;
+exports.getValidProjectNames = getValidProjectNames;
+exports.detectProjectName = detectProjectName;
 const fs_1 = __importDefault(require("fs"));
 const minimatch_1 = require("minimatch");
 const path_1 = __importDefault(require("path"));
-const ts_morph_1 = require("ts-morph");
 // For TypeScript type safety
 let tsxImport = null;
 const logger_1 = require("../../bin/logger");
 const context_1 = require("../../bin/utils/context");
-const fs_2 = require("../../bin/utils/fs");
 const web_1 = require("../../bin/utils/platform/web");
 const create_test_block_1 = require("../codegen/create-test-block");
 const fix_ts_errors_1 = require("../codegen/fix-ts-errors");
@@ -22,13 +28,11 @@ function isRegExp(obj) {
     return (obj instanceof RegExp ||
         Object.prototype.toString.call(obj) === "[object RegExp]");
 }
-exports.isRegExp = isRegExp;
 function prepareBrowsingAgentTask(steps) {
     const sanitizedSteps = steps.map((step) => step.replace(/`/g, "\\`"));
     const task = `${sanitizedSteps.join("\n")}\n`;
     return task;
 }
-exports.prepareBrowsingAgentTask = prepareBrowsingAgentTask;
 async function addImportForCreateTest(testFilePath) {
     // Instead of using "@empiricalrun/test-gen", we use the local dist file
     // This is to avoid assuming that the test-gen package is installed in the project
@@ -89,7 +93,7 @@ async function prepareFileForUpdateScenario({ testCase, specPath, trace, }) {
     });
     return createTestFilePath;
 }
-async function replaceTodoWithCreateTest({ testFilePath, testCaseName, testCaseSuites, }) {
+async function replaceTodoWithCreateTest(testFilePath) {
     // This method is an alternative to prepareFileForUpdateScenario
     // TODO: Does not support scoped variables and updates in POM files
     const fileContent = fs_1.default.readFileSync(testFilePath, "utf-8");
@@ -102,13 +106,7 @@ async function replaceTodoWithCreateTest({ testFilePath, testCaseName, testCaseS
     const pageVariable = pageVarName || "page"; // Default to "page" if not specified
     fs_1.default.writeFileSync(testFilePath, fileContent.replace(todoRegex, (_, __, todoText) => `await createTest("${todoText.replace(/"/g, '\\"')}", ${pageVariable});`));
     await addImportForCreateTest(testFilePath);
-    await markTestAsOnly({
-        testCaseName,
-        testCaseSuites,
-        specPath: testFilePath,
-    });
 }
-exports.replaceTodoWithCreateTest = replaceTodoWithCreateTest;
 async function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }) {
     const testFileContent = fs_1.default.readFileSync(specPath, "utf-8");
     const { testBlock, testNode } = (0, web_1.getTypescriptTestBlock)({
@@ -124,7 +122,6 @@ async function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }) {
         fs_1.default.writeFileSync(specPath, updatedTestFileContent);
     }
 }
-exports.markTestAsOnly = markTestAsOnly;
 async function prepareFileForMasterAgent({ testCase, specPath, trace, }) {
     const prepareFileSpan = trace?.span({
         name: "prepare-file-for-master-agent",
@@ -170,7 +167,6 @@ async function prepareFileForMasterAgent({ testCase, specPath, trace, }) {
     });
     return updatePath;
 }
-exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
 function newContentsWithTestOnly(existingContents, originalTestBlock, updatedTestBlock, parentDescribeBlock) {
     if (!parentDescribeBlock) {
         const testMarkedAsOnly = updatedTestBlock.replace("test(", "test.only(");
@@ -290,7 +286,6 @@ async function injectPwLocatorGenerator(page) {
         console.warn("Error injecting script in iframe.");
     }
 }
-exports.injectPwLocatorGenerator = injectPwLocatorGenerator;
 /**
  * function to read playwright config from the source repo
  * @return {*}  {Promise<PlaywrightTestConfig>}
@@ -318,7 +313,6 @@ async function readPlaywrightConfig(repoDir) {
     const playwrightConfig = (await tsxImport.require("./playwright.config.ts", `${repoDir}/${lastDir}`)).default;
     return playwrightConfig;
 }
-exports.readPlaywrightConfig = readPlaywrightConfig;
 function matchAgainstPattern(pattern, filePathToTest) {
     if (isRegExp(pattern)) {
         const regExp = pattern;
@@ -337,7 +331,6 @@ async function getValidProjectNames(playwrightConfig) {
         .filter((p) => !!p);
     return filteredProjectNames;
 }
-exports.getValidProjectNames = getValidProjectNames;
 /**
  * detect the project name for the given file in playwright test repo
  * if project and test file path for running test don't match, then playwright throws error
@@ -387,53 +380,3 @@ async function detectProjectName(testFilePath, playwrightConfig, pwProjectsFilte
     }
     return filteredProjectNames[0];
 }
-exports.detectProjectName = detectProjectName;
-class TeardownManager {
-    directory;
-    constructor(directory) {
-        this.directory = directory;
-    }
-    teardownFiles = [];
-    async getAllTeardownFiles() {
-        const teardownFileRegex = /.*\.teardown\.ts/;
-        const teardownFiles = await (0, fs_2.readFilesInDirectory)(this.directory, (fileName) => teardownFileRegex.test(fileName));
-        return teardownFiles;
-    }
-    async skipTeardownFile(filePath) {
-        const project = new ts_morph_1.Project();
-        const sourceFile = project.addSourceFileAtPath(filePath);
-        sourceFile
-            .getDescendantsOfKind(ts_morph_1.SyntaxKind.CallExpression)
-            .forEach((callExpression) => {
-            const expression = callExpression.getExpression();
-            if (expression.getText() === "teardown") {
-                expression.replaceWithText("teardown.skip");
-            }
-            else if (expression.getText() === "test") {
-                expression.replaceWithText("test.skip");
-            }
-        });
-        // Save the modified file
-        sourceFile.save().catch((error) => {
-            console.error(`Error updating the file: ${filePath}`, error);
-        });
-    }
-    async skipAll() {
-        this.teardownFiles = await this.getAllTeardownFiles();
-        await Promise.all(this.teardownFiles.map(async ({ filePath }) => await this.skipTeardownFile(filePath)));
-        const setupProcessListeners = (cleanup) => {
-            const events = ["beforeExit", "exit", "SIGINT", "SIGTERM"];
-            events.forEach((event) => process.on(event, cleanup));
-            return () => {
-                events.forEach((event) => process.removeListener(event, cleanup));
-            };
-        };
-        return setupProcessListeners(this.unskipAll.bind(this));
-    }
-    unskipAll() {
-        this.teardownFiles.forEach(({ filePath, content }) => {
-            fs_1.default.writeFileSync(filePath, content, "utf-8");
-        });
-    }
-}
-exports.TeardownManager = TeardownManager;

package/dist/agent/chat/agent-loop.d.ts

@@ -1,12 +1,15 @@
 import { TraceClient } from "@empiricalrun/llm";
-import { IChatModel } from "@empiricalrun/llm/chat";
+import { IChatModel, SupportedChatModels } from "@empiricalrun/llm/chat";
 import { ToolCallService } from "../../tool-call-service";
-import { ReporterFunction, SupportedChatModels } from "./types";
-export declare function chatAgentLoop({ chatModel, selectedModel, reporter, trace, toolCallService, }: {
+import { FileInfo } from "../../types";
+import { ReporterFunction } from "./types";
+export declare function chatAgentLoop({ chatModel, selectedModel, reporter, trace, toolCallService, fileInfo, isToolExecutionRemote, }: {
     chatModel: IChatModel<any>;
     selectedModel: SupportedChatModels;
     toolCallService: ToolCallService;
     reporter: ReporterFunction;
     trace?: TraceClient;
+    fileInfo: FileInfo;
+    isToolExecutionRemote: boolean;
 }): Promise<void>;
 //# sourceMappingURL=agent-loop.d.ts.map

package/dist/agent/chat/agent-loop.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAmB,MAAM,wBAAwB,CAAC;AAGrE,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG1D,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAgBhE,wBAAsB,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,QAAQ,EACR,KAAK,EACL,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IAC3B,aAAa,EAAE,mBAAmB,CAAC;IACnC,eAAe,EAAE,eAAe,CAAC;IACjC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,iBAoCA"}
+{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAEL,UAAU,EACV,mBAAmB,EAEpB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AA0B3C,wBAAsB,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,QAAQ,EACR,KAAK,EACL,eAAe,EACf,QAAQ,EACR,qBAAqB,GACtB,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IAC3B,aAAa,EAAE,mBAAmB,CAAC;IACnC,eAAe,EAAE,eAAe,CAAC;IACjC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,qBAAqB,EAAE,OAAO,CAAC;CAChC,iBAuCA"}

package/dist/agent/chat/agent-loop.js

@@ -1,50 +1,66 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.chatAgentLoop = void 0;
+exports.chatAgentLoop = chatAgentLoop;
 const chat_1 = require("@empiricalrun/llm/chat");
 const picocolors_1 = require("picocolors");
 const prompt_1 = require("./prompt");
 const state_1 = require("./state");
 function getModelName(model) {
-    if (model.startsWith("claude"))
+    const provider = (0, chat_1.getProviderForModel)(model);
+    if (provider === "claude")
         return "Claude";
-    if (model.startsWith("gemini"))
+    if (provider === "google")
         return "Gemini";
-    if (model.startsWith("o4"))
+    if (provider === "openai")
         return "o4";
     return "AI";
 }
 const log = (...args) => {
     console.log((0, picocolors_1.gray)(args.join(" ")));
 };
-const isRemote = process.env.TOOL_EXECUTION_IS_REMOTE === "true" || false;
-async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, toolCallService, }) {
-    const systemPrompt = await (0, prompt_1.buildSystemPrompt)();
+function logError(context, error, trace) {
+    console.error((0, picocolors_1.gray)(`[Error in ${context}]:`), error instanceof Error ? error.stack || error.message : error);
+    trace?.update({
+        output: {
+            error: error instanceof Error ? error.message : String(error),
+        },
+    });
+}
+async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, toolCallService, fileInfo, isToolExecutionRemote, }) {
+    const systemPrompt = await (0, prompt_1.buildSystemPrompt)(fileInfo);
     trace?.update({ input: { systemPrompt } });
     const { tools } = await toolCallService.getTools();
     while (!chatModel.askUserForInput) {
-        const toolCalls = chatModel.getPendingToolCalls();
-        if (toolCalls.length) {
-            const toolResults = await toolCallService.execute(toolCalls, isRemote, trace);
-            if (isRemote) {
-                log(`Tool call remote execution in progress`);
-                break;
+        try {
+            const toolCalls = chatModel.getPendingToolCalls();
+            if (toolCalls.length) {
+                if (isToolExecutionRemote) {
+                    await toolCallService.sendToQueue(toolCalls);
+                    log(`Tool call remote execution in progress`);
+                    break;
+                }
+                else {
+                    const toolResults = await toolCallService.execute(toolCalls, trace);
+                    chatModel.pushToolResultsMessage(toolCalls, toolResults);
+                }
             }
-            chatModel.pushToolResultsMessage(toolCalls, toolResults);
+            log(`${getModelName(selectedModel)} is working...`);
+            const response = await chatModel.getLLMResponse({
+                systemPrompt,
+                tools: tools.map((tool) => (0, chat_1.zodToOpenAITool)(tool.schema)),
+                selectedModel,
+                trace,
+            });
+            if (!response) {
+                throw new Error("No response from LLM");
+            }
+            chatModel.pushMessage(response);
+            const latest = chatModel.getHumanReadableLatestMessage();
+            await reporter((0, state_1.chatStateFromModel)(chatModel, selectedModel), latest);
         }
-        log(`${getModelName(selectedModel)} is working...`);
-        const response = await chatModel.getLLMResponse({
-            systemPrompt,
-            tools: tools.map((tool) => (0, chat_1.zodToOpenAITool)(tool.schema)),
-            selectedModel,
-            trace,
-        });
-        if (!response) {
-            throw new Error("No response from LLM");
+        catch (error) {
+            logError("chatAgentLoop", error, trace);
+            throw error;
         }
-        chatModel.pushMessage(response);
-        const latest = chatModel.getHumanReadableLatestMessage();
-        await reporter((0, state_1.chatStateFromModel)(chatModel, selectedModel), latest);
     }
 }
-exports.chatAgentLoop = chatAgentLoop;

package/dist/agent/chat/exports.d.ts

@@ -1,10 +1,9 @@
-import { IChatModel } from "@empiricalrun/llm/chat";
+import { createChatModel, IChatModel } from "@empiricalrun/llm/chat";
+import { FileInfo } from "../../types";
 import { chatAgentLoop } from "./agent-loop";
-import { createChatModel } from "./model";
-import * as State from "./state";
-import { ChatStateOnDisk } from "./state";
-import { ReporterFunction, SupportedChatModels } from "./types";
-declare const createChatState: typeof State.createChatState, createChatStateForMessages: typeof State.createChatStateForMessages, chatStateFromModel: typeof State.chatStateFromModel, loadChatState: typeof State.loadChatState, saveToDisk: typeof State.saveToDisk, CURRENT_CHAT_STATE_VERSION: string;
-export { chatAgentLoop, chatStateFromModel, createChatModel, createChatState, createChatStateForMessages, CURRENT_CHAT_STATE_VERSION, loadChatState, saveToDisk, };
-export type { ChatStateOnDisk, IChatModel, ReporterFunction, SupportedChatModels, };
+import { defaultModel, SUPPORTED_CHAT_MODELS, SupportedChatModels } from "./models";
+import { CHAT_STATE_VERSIONS_MIGRATIONS_MAP, chatStateFromModel, ChatStateOnDisk, createChatState, createChatStateForMessages, getLatestDownloadBuildUrl, LATEST_CHAT_STATE_VERSION, migrateChatState } from "./state";
+import { ReporterFunction } from "./types";
+export { CHAT_STATE_VERSIONS_MIGRATIONS_MAP, chatAgentLoop, chatStateFromModel, createChatModel, createChatState, createChatStateForMessages, defaultModel, getLatestDownloadBuildUrl, LATEST_CHAT_STATE_VERSION, migrateChatState, SUPPORTED_CHAT_MODELS, };
+export type { ChatStateOnDisk, FileInfo, IChatModel, ReporterFunction, SupportedChatModels, };
 //# sourceMappingURL=exports.d.ts.map

package/dist/agent/chat/exports.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exports.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/exports.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEhE,QAAA,MACE,eAAe,gCACf,0BAA0B,2CAC1B,kBAAkB,mCAClB,aAAa,8BACb,UAAU,2BACV,0BAA0B,QACnB,CAAC;AAEV,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,0BAA0B,EAC1B,0BAA0B,EAC1B,aAAa,EACb,UAAU,GACX,CAAC;AAEF,YAAY,EACV,eAAe,EACf,UAAU,EACV,gBAAgB,EAChB,mBAAmB,GACpB,CAAC"}
+{"version":3,"file":"exports.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/exports.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAErE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EACL,YAAY,EACZ,qBAAqB,EACrB,mBAAmB,EACpB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,kCAAkC,EAClC,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EACzB,gBAAgB,EACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EACL,kCAAkC,EAClC,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,0BAA0B,EAC1B,YAAY,EACZ,yBAAyB,EACzB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,GACtB,CAAC;AAEF,YAAY,EACV,eAAe,EACf,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,mBAAmB,GACpB,CAAC"}