@empiricalrun/test-gen 0.60.0 → 0.62.0

package/CHANGELOG.md

@@ -1,5 +1,57 @@
 # @empiricalrun/test-gen
 
+## 0.62.0
+
+### Minor Changes
+
+- f137da5: feat: collect artifact from browser agent, update Artifact UI.
+- e69b9d4: feat: implement artifact collection and enhance tool result handling in chat agent
+
+### Patch Changes
+
+- 4923078: fix: add production=false to npm ls command
+- fe1ea6e: fix: debug logs for npm ls cmd, missing try-catch
+- 333b99f: fix: test run tool should report correct status
+- 6b6742c: refactor: create codegen abstraction for computer-use agent
+- a44e96c: feat: inject working directory and API key instead of deriving them from process context or env
+- 62fa1f2: feat: add support for claude 4 sonnet and opus
+- a0167b3: fix: add more validation in file creation tool
+- 696925d: feat: patch playwright for page.pause codegen approach
+- fc27187: chore: skip some flaky tests to ensure green reviews
+- 28f95d0: fix: ripgrep handles left brackets correctly
+- 1d39277: feat: implement ArtifactDisplay component to showcase artifacts in session details
+- fe3cdcb: Revert "feat: enhance tool execution with working directory and environment v…"
+- a44e96c: refactor: remove getTools method and initiate tool executors in constructor
+- c79ee3d: fix: add more debug logs for pm2/npm ls issue
+- Updated dependencies [333b99f]
+- Updated dependencies [3739bc0]
+- Updated dependencies [a44e96c]
+- Updated dependencies [62fa1f2]
+- Updated dependencies [e69b9d4]
+- Updated dependencies [fe3cdcb]
+  - @empiricalrun/test-run@0.9.3
+  - @empiricalrun/llm@0.17.0
+
+## 0.61.0
+
+### Minor Changes
+
+- fca6729: feat: implement TypeScript compiler execution and error handling in strReplaceEditor
+
+### Patch Changes
+
+- f33bdbf: fix: git changed line should cover untracked also
+- 1b4e3d0: fix: dont wait for fonts when agent needs screenshot
+- bbf735a: chore: remove unused test.only cleaner
+- b7a4821: fix: update pr tool prompt to encourage proactiveness
+- 779dd6e: feat: add email automation recipe to system prompt
+- 05f5184: fix: file create should create parent dir if required
+- Updated dependencies [d71508f]
+- Updated dependencies [79857b3]
+- Updated dependencies [fca6729]
+  - @empiricalrun/test-run@0.9.2
+  - @empiricalrun/llm@0.16.1
+
 ## 0.60.0
 
 ### Minor Changes

package/dist/agent/browsing/run.d.ts

@@ -1,3 +1,4 @@
+import { ArtifactInput } from "@empiricalrun/shared-types";
 type GenerateTestsType = {
     testCaseName: string;
     testCaseSuites: string[];
@@ -18,6 +19,7 @@ export declare function generateTestsUsingMasterAgent({ testCaseName, testCaseSu
     isError: boolean;
     error: string;
     actionsSummary?: string;
+    artifacts?: ArtifactInput[];
 }>;
 export {};
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAcA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,wBAAsB,8BAA8B,CAAC,EACnD,gBAAgB,EAChB,OAAO,EACP,YAAY,GACb,EAAE;IACD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAQlB;AAED,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,OAAO,EACP,OAAO,EACP,yBAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC,CAiFD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAc3D,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,wBAAsB,8BAA8B,CAAC,EACnD,gBAAgB,EAChB,OAAO,EACP,YAAY,GACb,EAAE;IACD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAQlB;AAED,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,OAAO,EACP,OAAO,EACP,yBAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,aAAa,EAAE,CAAC;CAC7B,CAAC,CAmFD"}

package/dist/agent/browsing/run.js

@@ -23,8 +23,7 @@ async function generateTestsUsingMasterAgent({ testCaseName, testCaseSuites, tes
     }
     const pm = new exec_1.ProcessManager();
     const availablePort = await (0, detect_port_1.default)(3030);
-    // start a file service to handle file updates from agent
-    // - also update the file path with updates when agent is done spitting out code
+    // Start a file service for IPC with the agent (which runs in a different process)
     const fileServer = new server_1.FileServiceServer({
         port: availablePort,
         repoDir,
@@ -46,12 +45,14 @@ async function generateTestsUsingMasterAgent({ testCaseName, testCaseSuites, tes
         ],
         projects: [projectName],
         passthroughArgs: "--retries 0 --timeout 0",
-        //@ts-ignore
+        repoDir,
+        // @ts-ignore
         platform: "web",
     });
     let error = undefined;
     try {
-        console.log(`[generateTestsUsingMasterAgent] Running command: ${cmd.command} ${cmd.args.join(" ")}`);
+        const cmdLog = `${cmd.command} ${cmd.args.join(" ")}`;
+        console.log(`[generateTestsUsingMasterAgent] Running command: ${cmdLog}`);
         await pm.execute(cmd.command, cmd.args, {
             env: {
                 IPC_FILE_SERVICE_PORT: availablePort.toString(),
@@ -60,6 +61,10 @@ async function generateTestsUsingMasterAgent({ testCaseName, testCaseSuites, tes
                 PAGE_VAR_NAME: pageVar || "page",
                 DISPLAY: ":99",
                 LANGFUSE_TRACE_ID: traceId || "",
+                // To ensure page.screenshot() doesn't fail when fonts are not loaded
+                PW_TEST_SCREENSHOT_NO_FONTS_READY: "1",
+                // Enable this for the page.pause approach in codegen
+                PW_CODEGEN_NO_INSPECTOR: "1",
             },
         });
     }
@@ -68,7 +73,7 @@ async function generateTestsUsingMasterAgent({ testCaseName, testCaseSuites, tes
         console.error(error);
     }
     if (error) {
-        // clean up the file if there is any error
+        // Clean up the file if there is any error
         try {
             const fileContent = fs_1.default.readFileSync(filePathToUpdate, "utf-8");
             const updatedContent = (0, web_1.replaceCreateTestWithNewCode)(filePathToUpdate, fileContent, "");
@@ -79,12 +84,11 @@ async function generateTestsUsingMasterAgent({ testCaseName, testCaseSuites, tes
             console.error("Failed to remove extra scripts from files post test gen error", e);
         }
     }
-    // remove the test only from the file
-    await (0, web_1.removeTestOnly)(testFilePath);
     await fileServer.stop();
     return {
         isError: !!error,
         error: error || "",
         actionsSummary: fileServer.getActionsSummary(),
+        artifacts: fileServer.getArtifactInputsFromServer(),
     };
 }

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAiFD,wBAAsB,yBAAyB,CAAC,YAAY,EAAE,MAAM,iBAsBnE;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CA2B/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAiFD,wBAAsB,yBAAyB,CAAC,YAAY,EAAE,MAAM,iBAsBnE;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA8HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CA2B/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB"}

package/dist/agent/browsing/utils.js

@@ -239,7 +239,7 @@ async function injectPwLocatorGenerator(page) {
             });
         }
         catch (e) {
-            console.warn("Error during script injection on page load:", e);
+            console.warn(`Error during script injection on page load: ${e.message || String(e)}`);
         }
     });
     try {

package/dist/agent/chat/agent-loop.js

@@ -8,7 +8,6 @@ const utils_1 = require("./utils");
 async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, toolCallService, fileInfo, isToolExecutionRemote, }) {
     const systemPrompt = await (0, prompt_1.buildSystemPrompt)(fileInfo);
     trace?.update({ input: { systemPrompt } });
-    const { tools } = await toolCallService.getTools();
     while (!chatModel.askUserForInput) {
         try {
             const toolCalls = chatModel.getPendingToolCalls();
@@ -19,7 +18,7 @@ async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, toolCa
                     break;
                 }
                 else {
-                    const toolResults = await toolCallService.execute(toolCalls, trace);
+                    const toolResults = await toolCallService.execute(toolCalls);
                     chatModel.pushToolResultsMessage(toolCalls, toolResults);
                 }
             }
@@ -27,7 +26,7 @@ async function chatAgentLoop({ chatModel, selectedModel, reporter, trace, toolCa
             let response = null;
             response = await chatModel.getLLMResponse({
                 systemPrompt,
-                tools: tools.map((tool) => (0, chat_1.zodToOpenAITool)(tool.schema)),
+                tools: toolCallService.tools.map((tool) => (0, chat_1.zodToOpenAITool)(tool.schema)),
                 selectedModel,
                 trace,
             });

package/dist/agent/chat/exports.d.ts

@@ -1,7 +1,7 @@
-import { createChatModel, IChatModel } from "@empiricalrun/llm/chat";
+import { createChatModel, IChatModel, SUPPORTED_CHAT_MODELS, type SupportedChatModels } from "@empiricalrun/llm/chat";
 import { FileInfo } from "../../types";
 import { chatAgentLoop } from "./agent-loop";
-import { defaultModel, SUPPORTED_CHAT_MODELS, SupportedChatModels } from "./models";
+import { defaultModel } from "./models";
 import { CHAT_STATE_VERSIONS_MIGRATIONS_MAP, chatStateFromModel, createChatState, createChatStateForMessages, fetchToolCallAvailability, getLatestDownloadBuildUrl, LATEST_CHAT_STATE_VERSION, migrateChatState } from "./state";
 import { ReporterFunction } from "./types";
 export { CHAT_STATE_VERSIONS_MIGRATIONS_MAP, chatAgentLoop, chatStateFromModel, createChatModel, createChatState, createChatStateForMessages, defaultModel, fetchToolCallAvailability, getLatestDownloadBuildUrl, LATEST_CHAT_STATE_VERSION, migrateChatState, SUPPORTED_CHAT_MODELS, };

package/dist/agent/chat/exports.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exports.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/exports.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAErE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EACL,YAAY,EACZ,qBAAqB,EACrB,mBAAmB,EACpB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,kCAAkC,EAClC,kBAAkB,EAClB,eAAe,EACf,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EACzB,yBAAyB,EACzB,gBAAgB,EACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EACL,kCAAkC,EAClC,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,0BAA0B,EAC1B,YAAY,EACZ,yBAAyB,EACzB,yBAAyB,EACzB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,GACtB,CAAC;AAEF,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,CAAC"}
+{"version":3,"file":"exports.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/exports.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,UAAU,EACV,qBAAqB,EACrB,KAAK,mBAAmB,EACzB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EACL,kCAAkC,EAClC,kBAAkB,EAClB,eAAe,EACf,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EACzB,yBAAyB,EACzB,gBAAgB,EACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EACL,kCAAkC,EAClC,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,0BAA0B,EAC1B,YAAY,EACZ,yBAAyB,EACzB,yBAAyB,EACzB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,GACtB,CAAC;AAEF,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,CAAC"}

package/dist/agent/chat/exports.js

@@ -3,11 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SUPPORTED_CHAT_MODELS = exports.migrateChatState = exports.LATEST_CHAT_STATE_VERSION = exports.getLatestDownloadBuildUrl = exports.fetchToolCallAvailability = exports.defaultModel = exports.createChatStateForMessages = exports.createChatState = exports.createChatModel = exports.chatStateFromModel = exports.chatAgentLoop = exports.CHAT_STATE_VERSIONS_MIGRATIONS_MAP = void 0;
 const chat_1 = require("@empiricalrun/llm/chat");
 Object.defineProperty(exports, "createChatModel", { enumerable: true, get: function () { return chat_1.createChatModel; } });
+Object.defineProperty(exports, "SUPPORTED_CHAT_MODELS", { enumerable: true, get: function () { return chat_1.SUPPORTED_CHAT_MODELS; } });
 const agent_loop_1 = require("./agent-loop");
 Object.defineProperty(exports, "chatAgentLoop", { enumerable: true, get: function () { return agent_loop_1.chatAgentLoop; } });
 const models_1 = require("./models");
 Object.defineProperty(exports, "defaultModel", { enumerable: true, get: function () { return models_1.defaultModel; } });
-Object.defineProperty(exports, "SUPPORTED_CHAT_MODELS", { enumerable: true, get: function () { return models_1.SUPPORTED_CHAT_MODELS; } });
 const state_1 = require("./state");
 Object.defineProperty(exports, "CHAT_STATE_VERSIONS_MIGRATIONS_MAP", { enumerable: true, get: function () { return state_1.CHAT_STATE_VERSIONS_MIGRATIONS_MAP; } });
 Object.defineProperty(exports, "chatStateFromModel", { enumerable: true, get: function () { return state_1.chatStateFromModel; } });

package/dist/agent/chat/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/index.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,mBAAmB,EACpB,MAAM,wBAAwB,CAAC;AAyChC,wBAAsB,kBAAkB,CAAC,EACvC,mBAAmB,EACnB,aAAa,EACb,oBAAoB,EACpB,SAAS,GACV,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,iBAgHA;AAuBD,wBAAsB,wBAAwB,CAAC,EAC7C,aAAa,EACb,aAAa,GACd,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;CACvB,iBAsDA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/index.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,mBAAmB,EACpB,MAAM,wBAAwB,CAAC;AAyChC,wBAAsB,kBAAkB,CAAC,EACvC,mBAAmB,EACnB,aAAa,EACb,oBAAoB,EACpB,SAAS,GACV,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,iBAmHA;AAuBD,wBAAsB,wBAAwB,CAAC,EAC7C,aAAa,EACb,aAAa,GACd,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;CACvB,iBAyDA"}

package/dist/agent/chat/index.js

@@ -100,7 +100,14 @@ async function runChatAgentForCLI({ useDiskForChatState, selectedModel, initialP
         }
         else {
             // TODO: Should we pass a loader function? That would allow us to show a spinner
-            const toolCallService = new tool_call_service_1.ToolCallService(null, selectedModel, branchName);
+            const toolCallService = new tool_call_service_1.ToolCallService({
+                chatSessionId: null,
+                selectedModel,
+                branchName,
+                repoPath: process.cwd(),
+                apiKey: process.env.EMPIRICALRUN_API_KEY,
+                trace,
+            });
             const fileInfo = await (0, file_tree_1.getFileInfoFromFS)(process.cwd());
             await (0, agent_loop_1.chatAgentLoop)({
                 chatModel,
@@ -152,7 +159,14 @@ async function runChatAgentForDashboard({ chatSessionId, selectedModel, }) {
             chatSessionId,
         },
     });
-    const toolCallService = new tool_call_service_1.ToolCallService(chatSessionId, selectedModel, branchName);
+    const toolCallService = new tool_call_service_1.ToolCallService({
+        chatSessionId,
+        selectedModel,
+        branchName,
+        repoPath: process.cwd(),
+        apiKey: process.env.EMPIRICALRUN_API_KEY,
+        trace,
+    });
     await (0, git_1.checkoutBranch)(branchName);
     let chatModel = (0, chat_1.createChatModel)(chatState.messages, selectedModel);
     let reporterFunc = async (chatState, latest) => {

package/dist/agent/chat/models.d.ts

@@ -1,6 +1,4 @@
-import type { ModelInfo } from "@empiricalrun/shared-types";
-export declare const SUPPORTED_CHAT_MODELS: readonly ModelInfo[];
-export type SupportedChatModels = (typeof SUPPORTED_CHAT_MODELS)[number]["id"];
+import { type SupportedChatModels } from "@empiricalrun/llm/chat";
 export declare const defaultModel: SupportedChatModels;
 export declare const modelLabels: Record<SupportedChatModels, string>;
 //# sourceMappingURL=models.d.ts.map

package/dist/agent/chat/models.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/models.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAE5D,eAAO,MAAM,qBAAqB,EAAE,SAAS,SAAS,EAqB5C,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;AAW/E,eAAO,MAAM,YAAY,EAAE,mBAA6C,CAAC;AAEzE,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAOzD,CAAC"}
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,wBAAwB,CAAC;AAWhC,eAAO,MAAM,YAAY,EAAE,mBAA6C,CAAC;AAEzE,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAOzD,CAAC"}

package/dist/agent/chat/models.js

@@ -1,37 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.modelLabels = exports.defaultModel = exports.SUPPORTED_CHAT_MODELS = void 0;
-exports.SUPPORTED_CHAT_MODELS = [
-    {
-        id: "gemini-2.5-pro-preview-03-25",
-        label: "Gemini 2.5 Pro",
-        provider: "google",
-    },
-    {
-        id: "o4-mini-2025-04-16",
-        label: "OpenAI O4 Mini",
-        provider: "openai",
-    },
-    {
-        id: "claude-3-7-sonnet-20250219",
-        label: "Claude 3.7 Sonnet",
-        provider: "claude",
-    },
-    {
-        id: "claude-3-5-sonnet-20241022",
-        label: "Claude 3.5 Sonnet",
-        provider: "claude",
-    },
-];
+exports.modelLabels = exports.defaultModel = void 0;
+const chat_1 = require("@empiricalrun/llm/chat");
 const DEFAULT_CHAT_MODEL_ID = "gemini-2.5-pro-preview-03-25";
 function getDefaultChatModelId() {
-    if (!exports.SUPPORTED_CHAT_MODELS.some((m) => m.id === DEFAULT_CHAT_MODEL_ID)) {
+    if (!chat_1.SUPPORTED_CHAT_MODELS.some((m) => m.id === DEFAULT_CHAT_MODEL_ID)) {
         throw new Error("Default chat model is not in SUPPORTED_CHAT_MODELS");
     }
     return DEFAULT_CHAT_MODEL_ID;
 }
 exports.defaultModel = getDefaultChatModelId();
-exports.modelLabels = exports.SUPPORTED_CHAT_MODELS.reduce((acc, model) => ({
+exports.modelLabels = chat_1.SUPPORTED_CHAT_MODELS.reduce((acc, model) => ({
     ...acc,
     [model.id]: model.label,
 }), {});

package/dist/agent/chat/prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,mBAiGzD"}
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAuDvC,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,mBAwGzD"}

package/dist/agent/chat/prompt.js

@@ -2,6 +2,57 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildSystemPrompt = buildSystemPrompt;
 const repo_1 = require("./repo");
+const emailRecipe = `
+# Email automation
+
+## Example usage
+
+### Dynamic email
+
+This dynamically generates a random email address that can 
+be used for the test (e.g. invite a new user).
+
+\`\`\`ts
+import { EmailClient } from "@empiricalrun/playwright-utils";
+import { expect } from "@playwright/test";
+
+const client = new EmailClient();
+const address = client.getAddress();
+
+// Input the \`address\` in the application
+// that sends the email.
+
+// Get email received on the \`address\`
+const email = await client.waitForEmail();
+expect(
+  email.links.find((l) => l.text === "Join your team")
+).toBeTruthy();
+\`\`\`
+
+### Static email
+
+This uses a known (static) email that can be used to login
+into an application.
+
+This needs an email id (e.g. \`test-login-user\`). The email id
+is appended with the domain (managed internally) to get the full
+email address.
+
+\`\`\`ts
+import { EmailClient } from "@empiricalrun/playwright-utils";
+
+const emailId = \`test-login-user\`;
+
+const client = new EmailClient({ emailId });
+const address = client.getAddress(); // Returns full address with domain
+
+// Get email received on the \`address\`
+const email = await client.waitForEmail();
+
+// Get login OTP
+const loginCode = email.codes[0];
+\`\`\`
+`;
 async function buildSystemPrompt(fileInfo) {
     const repoContext = await (0, repo_1.getRepoInfoPrompt)(fileInfo);
     return `
@@ -96,6 +147,13 @@ if (await saveButton.isVisible()) {
 }
 \`\`\`
 
+# Recipes
+You can refer to the following recipes to learn how to write tests for different scenarios.
+
+<email-automation>
+${emailRecipe}
+</email-automation>
+
 # Repo context
 ${repoContext}
 `;

package/dist/agent/cua/computer.d.ts

@@ -1,12 +1,12 @@
 import { ResponseComputerToolCall } from "openai/resources/responses/responses.mjs";
 import type { Page } from "playwright";
-type ComputerAction = ResponseComputerToolCall.Click | ResponseComputerToolCall.DoubleClick | ResponseComputerToolCall.Drag | ResponseComputerToolCall.Keypress | ResponseComputerToolCall.Move | ResponseComputerToolCall.Screenshot | ResponseComputerToolCall.Scroll | ResponseComputerToolCall.Type | ResponseComputerToolCall.Wait;
+import { BasePlaywrightCodegen } from "./pw-codegen/types";
+type OpenAIComputerAction = ResponseComputerToolCall.Click | ResponseComputerToolCall.DoubleClick | ResponseComputerToolCall.Drag | ResponseComputerToolCall.Keypress | ResponseComputerToolCall.Move | ResponseComputerToolCall.Screenshot | ResponseComputerToolCall.Scroll | ResponseComputerToolCall.Type | ResponseComputerToolCall.Wait;
 export declare function getScreenshot(page: Page): Promise<string>;
-export declare function handlePageGoto(page: Page, url: string): Promise<{
-    actionSummary: string;
-    actionCode: string;
-}>;
-export declare function handleModelAction(page: Page, action: ComputerAction): Promise<{
+export declare function executeModelAction(page: Page, action: OpenAIComputerAction | {
+    type: "goto";
+    url: string;
+}, codegen: BasePlaywrightCodegen): Promise<{
     actionSummary: string;
     actionCode: string;
 }>;

package/dist/agent/cua/computer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIvC,KAAK,cAAc,GACf,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAG7C;AAgCD,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAMD;AA4DD,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAiID"}
+{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,KAAK,oBAAoB,GACrB,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAG7C;AAgCD,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,oBAAoB,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAC5D,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAkJD"}

package/dist/agent/cua/computer.js

@@ -1,9 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getScreenshot = getScreenshot;
-exports.handlePageGoto = handlePageGoto;
-exports.handleModelAction = handleModelAction;
-const utils_1 = require("../browsing/utils");
+exports.executeModelAction = executeModelAction;
 async function getScreenshot(page) {
     const screenshotBytes = await page.screenshot();
     return Buffer.from(screenshotBytes).toString("base64");
@@ -37,70 +35,24 @@ const CUA_KEY_TO_PLAYWRIGHT_KEY = {
     tab: "Tab",
     win: "Meta",
 };
-async function handlePageGoto(page, url) {
-    await page.goto(url);
-    return {
-        actionSummary: `Navigated page to ${url}`,
-        actionCode: `await page.goto("${url}");\n`,
-    };
-}
-async function getLocatorForClick(page, { x, y }) {
-    const locator = await page.evaluate(([x, y]) => {
-        const element = document.elementFromPoint(x, y);
-        const bbox = element?.getBoundingClientRect();
-        return {
-            locator: window.playwright.generateLocator(element),
-            isIframe: element?.tagName === "IFRAME",
-            x0: bbox?.x,
-            y0: bbox?.y,
-            src: element?.getAttribute("src"),
-            id: element?.getAttribute("id"),
-        };
-    }, [x, y]);
-    if (!locator.isIframe) {
-        return locator.locator;
-    }
-    await (0, utils_1.injectPwLocatorGenerator)(page);
-    const relativeX = x - locator.x0;
-    const relativeY = y - locator.y0;
-    // TODO: Reuse locator.locator for the frameLocator.
-    let frameEl = locator.src
-        ? page.locator(`[src="${locator.src}"]`)
-        : page.locator(`#${locator.id}`);
-    const frameLocator = locator.src
-        ? `locator('[src="${locator.src}"]')`
-        : `locator('#${locator.id}')`;
-    const elementLocatorInsideFrame = await frameEl
-        .contentFrame()
-        .locator(":root")
-        .evaluate((rootElement, coords) => {
-        const xPos = coords[0];
-        const yPos = coords[1];
-        if (xPos === undefined || yPos === undefined) {
-            throw new Error("Coordinates are undefined in evaluate call");
-        }
-        const element = document.elementFromPoint(xPos, yPos);
-        return window.playwright.generateLocator(element);
-    }, [relativeX, relativeY]);
-    return `${frameLocator}.contentFrame().${elementLocatorInsideFrame}`;
-}
-async function getLocatorForFill(page) {
-    const locator = await page.evaluate(() => {
-        const element = document.activeElement;
-        return window.playwright.generateLocator(element);
-    });
-    return locator;
-}
-async function handleModelAction(page, action) {
+async function executeModelAction(page, action, codegen) {
     const actionType = action.type;
     let actionCode = "";
     let actionSummary = "";
     try {
         switch (actionType) {
+            // For every action, this method 1. executes the action,
+            // 2. adds to action summary, and 3. adds to action code
+            case "goto": {
+                const { url } = action;
+                await page.goto(url);
+                actionSummary = `Navigated page to ${url}`;
+                actionCode = await codegen.codeForAction({ type: "goto", url });
+                break;
+            }
             case "click": {
                 const { x, y, button = "left" } = action;
                 console.log(`Action: click at (${x}, ${y}) with button '${button}'`);
-                actionSummary = `Click at (${x}, ${y}) with button '${button}'`;
                 let pwButton = undefined;
                 if (button === "left" || button === "right") {
                     pwButton = button;
@@ -109,19 +61,20 @@ async function handleModelAction(page, action) {
                     pwButton = "middle";
                 }
                 if (pwButton) {
-                    const locator = await getLocatorForClick(page, { x, y });
-                    actionCode = `await page.${locator}.click();\n`;
                     await page.mouse.click(x, y, { button: pwButton });
+                    actionCode = await codegen.codeForAction({ type: "click", x, y });
+                    actionSummary = `Click at (${x}, ${y}) with button '${button}'`;
                 }
                 if (button === "back" || button === "forward") {
-                    // Do page navigations, since there is no way to click on the back/forward buttons
                     if (button === "back") {
-                        actionCode = `await page.goBack();\n`;
                         await page.goBack();
+                        actionCode = await codegen.codeForAction({ type: "back" });
+                        actionSummary = `Navigated page back`;
                     }
                     else if (button === "forward") {
-                        actionCode = `await page.goForward();\n`;
                         await page.goForward();
+                        actionCode = await codegen.codeForAction({ type: "forward" });
+                        actionSummary = `Navigated page forward`;
                     }
                 }
                 break;
@@ -129,23 +82,22 @@ async function handleModelAction(page, action) {
             case "double_click": {
                 const { x, y } = action;
                 console.log(`Action: doubleclick at (${x}, ${y})`);
-                actionSummary = `Double click at (${x}, ${y})`;
-                const locator = await getLocatorForClick(page, { x, y });
-                actionCode = `await page.${locator}.dblclick();\n`;
                 await page.mouse.dblclick(x, y, { button: "left" });
+                actionSummary = `Double click at (${x}, ${y})`;
+                actionCode = await codegen.codeForAction({ type: "doubleclick", x, y });
                 break;
             }
             case "move": {
                 const { x, y } = action;
                 console.log(`Action: mouse move to (${x}, ${y})`);
-                actionSummary = `Mouse move to (${x}, ${y})`;
                 await page.mouse.move(x, y);
+                actionSummary = `Mouse move to (${x}, ${y})`;
+                actionCode = await codegen.codeForAction({ type: "move", x, y });
                 break;
             }
             case "drag": {
                 const { path } = action;
                 console.log(`Action: drag along path ${path}`);
-                // TODO: actionCode is not implemented
                 actionSummary = `Drag along path ${path}`;
                 if (!path || path.length === 0) {
                     break;
@@ -156,14 +108,22 @@ async function handleModelAction(page, action) {
                     await page.mouse.move(path[i].x, path[i].y);
                 }
                 await page.mouse.up();
+                actionCode = await codegen.codeForAction({ type: "drag", path });
                 break;
             }
             case "scroll": {
                 const { x, y, scroll_x, scroll_y } = action;
                 console.log(`Action: scroll at (${x}, ${y}) with offsets (scroll_x=${scroll_x}, scroll_y=${scroll_y})`);
-                actionSummary = `Scroll at (${x}, ${y}) with offsets (scroll_x=${scroll_x}, scroll_y=${scroll_y})`;
                 await page.mouse.move(x, y);
                 await page.evaluate(`window.scrollBy(${scroll_x}, ${scroll_y})`);
+                actionSummary = `Scroll at (${x}, ${y}) with offsets (scroll_x=${scroll_x}, scroll_y=${scroll_y})`;
+                actionCode = await codegen.codeForAction({
+                    type: "scroll",
+                    x,
+                    y,
+                    scroll_x,
+                    scroll_y,
+                });
                 break;
             }
             case "keypress": {
@@ -173,29 +133,24 @@ async function handleModelAction(page, action) {
                 });
                 const mappedKey = mappedKeys.join("+"); // ["CTRL", "A"] becomes ControlOrMeta+A
                 console.log(`Action: keypress for keys ${keys} -> '${mappedKey}'`);
+                await page.keyboard.press(mappedKey);
                 actionSummary = `Keypress for keys ${keys} (mapped to '${mappedKey}' for Playwright)`;
-                try {
-                    await page.keyboard.press(mappedKey);
-                    actionCode = `await page.keyboard.press('${mappedKey}');\n`;
-                }
-                catch (e) {
-                    console.error("Error pressing key", mappedKey, ":", e);
-                }
+                actionCode = await codegen.codeForAction({ type: "keypress", keys });
                 break;
             }
             case "type": {
                 const { text } = action;
                 console.log(`Action: type text '${text}'`);
-                actionSummary = `Type text '${text}'`;
                 await page.keyboard.type(text);
-                const locator = await getLocatorForFill(page);
-                actionCode = `await page.${locator}.fill("${text}");\n`;
+                actionSummary = `Type text '${text}'`;
+                actionCode = await codegen.codeForAction({ type: "type", text });
                 break;
             }
             case "wait": {
                 console.log(`Action: wait`);
-                actionSummary = `Wait for 2 seconds`;
                 await page.waitForTimeout(2000);
+                actionSummary = `Wait for 2 seconds`;
+                actionCode = await codegen.codeForAction({ type: "wait" });
                 break;
             }
             case "screenshot": {
@@ -208,8 +163,8 @@ async function handleModelAction(page, action) {
                 console.log("Unrecognized action:", action);
         }
     }
-    catch (e) {
-        console.error("Error handling action", action, ":", e);
+    catch (err) {
+        console.error(`Error handling action ${action.type}: ${err.message || err.toString()}`);
     }
     return { actionSummary, actionCode };
 }

package/dist/agent/cua/index.d.ts

@@ -1,6 +1,6 @@
 import { TraceClient } from "@empiricalrun/llm";
+import { ArtifactInput } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
-export declare function startPlaywrightCodegen(page: Page): Promise<void>;
 export declare function createTestUsingComputerUseAgent({ page, task, trace, }: {
     page: Page;
     task: string;
@@ -9,5 +9,6 @@ export declare function createTestUsingComputerUseAgent({ page, task, trace, }:
     code: string;
     importPaths: string[];
     actionsSummary: string;
+    artifacts: ArtifactInput[];
 }>;
 //# sourceMappingURL=index.d.ts.map