@empiricalrun/test-gen 0.56.4 → 0.57.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @empiricalrun/test-gen
 
+## 0.57.0
+
+### Minor Changes
+
+- df30a88: feat: commit context to git on every tool file change
+
+### Patch Changes
+
+- 8a847fd: feat: tracing for CUA when it runs in a tool
+- 597072c: feat: suggest unique occurences for old_str in text editor tools
+- 813995a: fix: grep tool should handle whitespaces in search string
+- Updated dependencies [629bbd2]
+  - @empiricalrun/llm@0.15.3
+
 ## 0.56.4
 
 ### Patch Changes

package/dist/agent/browsing/run.d.ts

@@ -4,9 +4,10 @@ type GenerateTestsType = {
     pwProjectsFilter: string[];
     testGenToken: string;
     repoDir: string;
+    traceId?: string;
     editFileWithGeneratedCode: boolean;
 };
-export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, editFileWithGeneratedCode, }: GenerateTestsType): Promise<{
+export declare function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, traceId, editFileWithGeneratedCode, }: GenerateTestsType): Promise<{
     isError: boolean;
     error: string;
     actionsSummary?: string;

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,EACP,yBAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC,CAsFD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAiBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,EACP,OAAO,EACP,yBAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC,CAuFD"}

package/dist/agent/browsing/run.js

@@ -10,7 +10,7 @@ const web_1 = require("../../bin/utils/platform/web");
 const server_1 = require("../../file/server");
 const exec_1 = require("../../utils/exec");
 const utils_1 = require("./utils");
-async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, editFileWithGeneratedCode, }) {
+async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, pwProjectsFilter, testGenToken, repoDir, traceId, editFileWithGeneratedCode, }) {
     if (!fs_1.default.existsSync(testFilePath)) {
         throw new Error(`File for master agent to run not found: ${testFilePath}`);
     }
@@ -53,6 +53,7 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
                 TEST_GEN_TOKEN: testGenToken,
                 PAGE_VAR_NAME: pageVar || "page",
                 DISPLAY: ":99",
+                LANGFUSE_TRACE_ID: traceId || "",
             },
         });
     }

package/dist/agent/chat/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAAoB,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAiBhE,wBAAsB,kBAAkB,CAAC,EACvC,mBAAmB,EACnB,aAAa,EACb,oBAAoB,GACrB,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,iBAsFA;AA0BD,wBAAsB,wBAAwB,CAAC,EAC7C,aAAa,EACb,aAAa,GACd,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;CACvB,iBA2CA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAAoB,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAiBhE,wBAAsB,kBAAkB,CAAC,EACvC,mBAAmB,EACnB,aAAa,EACb,oBAAoB,GACrB,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,iBA8FA;AA0BD,wBAAsB,wBAAwB,CAAC,EAC7C,aAAa,EACb,aAAa,GACd,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;CACvB,iBA+CA"}

package/dist/agent/chat/index.js

@@ -23,6 +23,10 @@ async function runChatAgentForCLI({ useDiskForChatState, selectedModel, initialP
     if (useDiskForChatState) {
         chatState = (0, state_1.loadChatState)();
     }
+    // TODO: Store branch name in chat state so that we don't recreate it every time
+    const randomId = crypto.randomUUID().substring(0, 8);
+    const branchName = `branch-${randomId}`;
+    await (0, git_1.checkoutBranch)(branchName);
     let messagesLoadedFromDisk = chatState?.messages || [];
     let chatModel = (0, model_1.createChatModel)(messagesLoadedFromDisk, selectedModel);
     if (initialPromptContent && chatModel.messages.length === 0) {
@@ -84,7 +88,7 @@ async function runChatAgentForCLI({ useDiskForChatState, selectedModel, initialP
         }
         else {
             // TODO: Should we pass a loader function? That would allow us to show a spinner
-            const toolCallService = new tool_call_service_1.ToolCallService(null, selectedModel);
+            const toolCallService = new tool_call_service_1.ToolCallService(null, selectedModel, branchName);
             await (0, agent_loop_1.chatAgentLoop)({
                 chatModel,
                 selectedModel,
@@ -131,7 +135,7 @@ async function runChatAgentForDashboard({ chatSessionId, selectedModel, }) {
             chatSessionId,
         },
     });
-    const toolCallService = new tool_call_service_1.ToolCallService(chatSessionId, selectedModel);
+    const toolCallService = new tool_call_service_1.ToolCallService(chatSessionId, selectedModel, branchName);
     await (0, git_1.checkoutBranch)(branchName);
     let chatModel = (0, model_1.createChatModel)(chatState.messages, selectedModel);
     let reporterFunc = async (chatState, latest) => {

package/dist/agent/chat/state.d.ts

@@ -7,7 +7,7 @@ export type ChatStateOnDisk<T> = {
     model: SupportedChatModels;
     messages: T[];
 };
-export declare function createChatState(userPrompt: string, existingState: ChatStateOnDisk<any>, selectedModel: SupportedChatModels): ChatStateOnDisk<unknown>;
+export declare function createChatState(userPrompt: string, existingState: ChatStateOnDisk<any> | undefined, selectedModel: SupportedChatModels): ChatStateOnDisk<unknown>;
 export declare function createChatStateForMessages<T>(messages: any, selectedModel: SupportedChatModels): ChatStateOnDisk<T>;
 export declare function chatStateFromModel<T>(chatModel: IChatModel<T>, selectedModel: SupportedChatModels): ChatStateOnDisk<unknown>;
 export declare function loadChatState<T>(): ChatStateOnDisk<T> | undefined;

package/dist/agent/chat/state.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAKpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE9C,eAAO,MAAM,0BAA0B,eAAe,CAAC;AAEvD,eAAO,MAAM,eAAe,QAI3B,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,CAAC,IAAI;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,QAAQ,EAAE,CAAC,EAAE,CAAC;CACf,CAAC;AAEF,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,eAAe,CAAC,GAAG,CAAC,EACnC,aAAa,EAAE,mBAAmB,4BAMnC;AAED,wBAAgB,0BAA0B,CAAC,CAAC,EAC1C,QAAQ,EAAE,GAAG,EACb,aAAa,EAAE,mBAAmB,GACjC,eAAe,CAAC,CAAC,CAAC,CAOpB;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,EACxB,aAAa,EAAE,mBAAmB,4BAGnC;AAED,wBAAgB,aAAa,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC,CAAC,GAAG,SAAS,CAajE;AAED,wBAAgB,UAAU,CAAC,CAAC,EAC1B,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAClB,aAAa,EAAE,mBAAmB,QAsBnC"}
+{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAKpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE9C,eAAO,MAAM,0BAA0B,eAAe,CAAC;AAEvD,eAAO,MAAM,eAAe,QAI3B,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,CAAC,IAAI;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,QAAQ,EAAE,CAAC,EAAE,CAAC;CACf,CAAC;AAEF,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,eAAe,CAAC,GAAG,CAAC,GAAG,SAAS,EAC/C,aAAa,EAAE,mBAAmB,4BAMnC;AAED,wBAAgB,0BAA0B,CAAC,CAAC,EAC1C,QAAQ,EAAE,GAAG,EACb,aAAa,EAAE,mBAAmB,GACjC,eAAe,CAAC,CAAC,CAAC,CAOpB;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,EACxB,aAAa,EAAE,mBAAmB,4BAGnC;AAED,wBAAgB,aAAa,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC,CAAC,GAAG,SAAS,CAajE;AAED,wBAAgB,UAAU,CAAC,CAAC,EAC1B,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAClB,aAAa,EAAE,mBAAmB,QAsBnC"}

package/dist/agent/chat/state.js

@@ -10,7 +10,7 @@ const model_1 = require("./model");
 exports.CURRENT_CHAT_STATE_VERSION = "20250327.1";
 exports.CHAT_STATE_PATH = path_1.default.join(process.cwd(), ".empiricalrun", "last-chat.json");
 function createChatState(userPrompt, existingState, selectedModel) {
-    const messages = existingState.messages || [];
+    const messages = existingState?.messages || [];
     const chatModel = (0, model_1.createChatModel)(messages, selectedModel);
     chatModel.pushUserMessage(userPrompt);
     return createChatStateForMessages(chatModel.messages, selectedModel);

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQhD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAoBpC,wBAAsB,UAAU,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,IAAI,GAAG,YAAY,EAC5B,KAAK,CAAC,EAAE,SAAS,iBAwDlB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQhD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAoBpC,wBAAsB,UAAU,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,IAAI,GAAG,YAAY,EAC5B,KAAK,CAAC,EAAE,SAAS,iBA0ElB"}

package/dist/index.js

@@ -27,6 +27,18 @@ async function createTest(task, pageRef, scope) {
         const testConfigArg = process.env.TEST_GEN_TOKEN;
         const testGenConfig = (0, scenarios_1.loadTestConfigs)(testConfigArg);
         const useComputerUseAgent = testGenConfig.options?.useComputerUseAgent;
+        let trace, span;
+        if (process.env.LANGFUSE_TRACE_ID) {
+            trace = llm_1.langfuseInstance?.trace({
+                id: process.env.LANGFUSE_TRACE_ID,
+            });
+            span = trace?.span({
+                name: "computer-use-agent-span",
+                input: {
+                    task,
+                },
+            });
+        }
         if (testGenConfig.options && testGenConfig.options.metadata) {
             (0, reporter_1.setReporterConfig)({
                 projectRepoName: testGenConfig.options?.metadata.projectRepoName,
@@ -48,6 +60,7 @@ async function createTest(task, pageRef, scope) {
             agentResult = await (0, cua_1.createTestUsingComputerUseAgent)({
                 task,
                 page,
+                trace: span,
             });
         }
         else {
@@ -69,6 +82,11 @@ async function createTest(task, pageRef, scope) {
             importPaths,
             actionsSummary,
         });
+        span?.end({
+            output: {
+                actionsSummary,
+            },
+        });
     }
     finally {
         // Ensure listeners are removed even if an error occurs

package/dist/tool-call-service/index.d.ts

@@ -3,14 +3,15 @@ import { PendingToolCall, Tool, ToolResult } from "@empiricalrun/llm/chat";
 import { SupportedChatModels } from "../agent/chat/types";
 export type { SupportedChatModels };
 type ToolExecutors = {
-    [key: string]: (input: any) => Promise<ToolResult>;
+    [key: string]: (input: any, trace?: TraceClient) => Promise<ToolResult>;
 };
 export declare class ToolCallService {
     tools: Tool[];
     toolExecutors: ToolExecutors;
     chatSessionId: number | null;
     selectedModel: SupportedChatModels;
-    constructor(chatSessionId: number | null, selectedModel: SupportedChatModels);
+    branchName: string;
+    constructor(chatSessionId: number | null, selectedModel: SupportedChatModels, branchName: string);
     getTools(): Promise<{
         tools: Tool[];
     }>;

package/dist/tool-call-service/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tool-call-service/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAE3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAa1D,YAAY,EAAE,mBAAmB,EAAE,CAAC;AAEpC,KAAK,aAAa,GAAG;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CACpD,CAAC;AA4BF,qBAAa,eAAe;IAC1B,KAAK,EAAE,IAAI,EAAE,CAAM;IACnB,aAAa,EAAE,aAAa,CAAM;IAClC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,mBAAmB,CAAC;gBAGjC,aAAa,EAAE,MAAM,GAAG,IAAI,EAC5B,aAAa,EAAE,mBAAmB;IAgB9B,QAAQ;;;IAaR,OAAO,CACX,SAAS,EAAE,eAAe,EAAE,EAC5B,QAAQ,EAAE,OAAO,EACjB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,UAAU,EAAE,CAAC;CAuDzB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tool-call-service/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAE3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAe1D,YAAY,EAAE,mBAAmB,EAAE,CAAC;AAEpC,KAAK,aAAa,GAAG;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CACzE,CAAC;AA6BF,qBAAa,eAAe;IAC1B,KAAK,EAAE,IAAI,EAAE,CAAM;IACnB,aAAa,EAAE,aAAa,CAAM;IAClC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,mBAAmB,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;gBAEjB,aAAa,EAAE,MAAM,GAAG,IAAI,EAC5B,aAAa,EAAE,mBAAmB,EAClC,UAAU,EAAE,MAAM;IAiBd,QAAQ;;;IAaR,OAAO,CACX,SAAS,EAAE,eAAe,EAAE,EAC5B,QAAQ,EAAE,OAAO,EACjB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,UAAU,EAAE,CAAC;CAyDzB"}

package/dist/tool-call-service/index.js

@@ -11,6 +11,7 @@ const str_replace_editor_1 = require("../tools/str_replace_editor");
 const test_gen_browser_1 = require("../tools/test-gen-browser");
 const test_run_1 = require("../tools/test-run");
 const test_run_fetcher_1 = require("../tools/test-run-fetcher");
+const checkpoint_1 = require("../utils/checkpoint");
 async function sendToolRequestToRemoteQueue(payload) {
     const sqs = new client_sqs_1.SQSClient({
         region: process.env.AWS_REGION,
@@ -35,16 +36,18 @@ class ToolCallService {
     toolExecutors = {};
     chatSessionId;
     selectedModel;
-    constructor(chatSessionId, selectedModel) {
+    branchName;
+    constructor(chatSessionId, selectedModel, branchName) {
         this.chatSessionId = chatSessionId;
         this.selectedModel = selectedModel;
+        this.branchName = branchName;
         this.tools = [
             grep_1.grepTool,
             test_run_1.runTestTool,
             test_run_fetcher_1.fetchTestRunReportTool,
             diagnosis_fetcher_1.fetchDiagnosisReportTool,
             test_gen_browser_1.generateTestWithBrowserAgent,
-            commit_and_create_pr_1.commitAndPushChangesTool,
+            commit_and_create_pr_1.createPullRequestTool,
             environment_crud_1.getEnvironmentTool,
             download_build_1.downloadBuildTool,
         ];
@@ -68,6 +71,7 @@ class ToolCallService {
                 requestId: crypto.randomUUID(),
                 chatSessionId: this.chatSessionId,
                 selectedModel: this.selectedModel,
+                branchName: this.branchName,
             });
             return toolCalls.map(() => ({
                 isError: false,
@@ -96,7 +100,7 @@ class ToolCallService {
                     continue;
                 }
                 try {
-                    const result = await toolExecutor(toolCall.input);
+                    const result = await toolExecutor(toolCall.input, trace);
                     toolResults.push(result);
                     span?.end({ output: result });
                 }
@@ -109,6 +113,7 @@ class ToolCallService {
                     span?.end({ output: errorResult });
                 }
             }
+            await (0, checkpoint_1.createCheckpoint)(toolCalls, this.branchName);
             executeSpan?.end({ output: { toolResults } });
             return toolResults;
         }

package/dist/tools/commit-and-create-pr.d.ts

@@ -4,6 +4,6 @@ interface GitHubRepoInfo {
     repo: string;
 }
 export declare function parseGitHubUrl(url: string): GitHubRepoInfo;
-export declare const commitAndPushChangesTool: Tool;
+export declare const createPullRequestTool: Tool;
 export {};
 //# sourceMappingURL=commit-and-create-pr.d.ts.map

package/dist/tools/commit-and-create-pr.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commit-and-create-pr.d.ts","sourceRoot":"","sources":["../../src/tools/commit-and-create-pr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAQnD,UAAU,cAAc;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAiB1D;AA2CD,eAAO,MAAM,wBAAwB,EAAE,IAwFtC,CAAC"}
+{"version":3,"file":"commit-and-create-pr.d.ts","sourceRoot":"","sources":["../../src/tools/commit-and-create-pr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAOnD,UAAU,cAAc;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAiB1D;AA2CD,eAAO,MAAM,qBAAqB,EAAE,IAuEnC,CAAC"}

package/dist/tools/commit-and-create-pr.js

@@ -1,11 +1,7 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.commitAndPushChangesTool = exports.parseGitHubUrl = void 0;
+exports.createPullRequestTool = exports.parseGitHubUrl = void 0;
 const child_process_1 = require("child_process");
-const crypto_1 = __importDefault(require("crypto"));
 const zod_1 = require("zod");
 const git_1 = require("../utils/git");
 const utils_1 = require("./utils");
@@ -24,11 +20,11 @@ function parseGitHubUrl(url) {
     return { owner, repo };
 }
 exports.parseGitHubUrl = parseGitHubUrl;
-const CommitAndPushChangesSchema = zod_1.z.object({
-    commitMessage: zod_1.z
+const createPullRequestSchema = zod_1.z.object({
+    pullRequestTitle: zod_1.z
         .string()
         .describe("A short message to use for the commit. Should not be more than 8 words. Should follow conventional commit format."),
-    description: zod_1.z.string().describe(`A longer description of the changes you made. This will be used as the description of a pull request on GitHub, and so you should follow markdown formatting.
+    pullRequestDescription: zod_1.z.string().describe(`A longer description of the changes you made. This will be used as the description of a pull request on GitHub, and so you should follow markdown formatting.
 Your code will be reviewed by a human, and you should include everything that will provide context and improve the reviewer's confidence in the changes.
 
 For example, if you used the test run tool, you should include the results (and the report URL if available). Report URL is especially important, because it contains
@@ -46,38 +42,19 @@ function formatDescriptionWithTimestamp(description, existingBody, type = "creat
     }
     return `${description}\n\n${timestampText}`;
 }
-exports.commitAndPushChangesTool = {
+exports.createPullRequestTool = {
     schema: {
-        name: "commitAndPushChanges",
-        description: `Creates a commit with all modified files and pushes them to the current branch.
-If currently on main branch, creates a new branch with a random name.
+        name: "createPullRequest",
+        description: `Creates a new Pull Request on GitHub.
 If the current branch already has an open PR, commits and pushes changes to that PR.
 Uses the empiricalrun[bot] credentials for git operations.
 Returns the URL of the created or updated pull request.`,
-        parameters: CommitAndPushChangesSchema,
+        parameters: createPullRequestSchema,
     },
     execute: async (input) => {
         try {
-            const { commitMessage, description } = input;
-            const currentBranch = (0, child_process_1.execSync)("git rev-parse --abbrev-ref HEAD")
-                .toString()
-                .trim();
-            let branchName = currentBranch;
-            if (currentBranch === "main") {
-                // If on main, create a new branch
-                const randomId = crypto_1.default.randomUUID().substring(0, 8);
-                branchName = `branch-${randomId}`;
-                (0, child_process_1.execSync)(`git checkout -b ${branchName}`);
-            }
-            const modifiedFiles = (0, child_process_1.execSync)("git status --porcelain")
-                .toString()
-                .split("\n")
-                .filter((line) => line && !line.includes(".bak"))
-                .map((line) => line.substring(3)); // Remove status prefix
-            for (const file of modifiedFiles) {
-                (0, child_process_1.execSync)(`git add "${file}"`);
-            }
-            await (0, git_1.commitAsBotUser)(commitMessage);
+            const { pullRequestTitle, pullRequestDescription } = input;
+            const branchName = await (0, git_1.getCurrentBranchName)();
             const repoUrl = (0, child_process_1.execSync)("git config --get remote.origin.url")
                 .toString()
                 .trim();
@@ -94,7 +71,7 @@ Returns the URL of the created or updated pull request.`,
             const existingPR = existingPRs?.find((pr) => pr.head.ref === branchName);
             if (existingPR) {
                 // Append the new description to the existing PR description
-                const updatedDescription = formatDescriptionWithTimestamp(description, existingPR.body, "update");
+                const updatedDescription = formatDescriptionWithTimestamp(pullRequestDescription, existingPR.body, "update");
                 await (0, utils_1.callGitHubProxy)({
                     method: "PATCH",
                     url: `https://api.github.com/repos/${owner}/${repo}/pulls/${existingPR.number}`,
@@ -107,12 +84,12 @@ Returns the URL of the created or updated pull request.`,
                     result: `Committed and pushed changes to existing PR: ${existingPR.html_url}`,
                 };
             }
-            const initialDescription = formatDescriptionWithTimestamp(description);
+            const initialDescription = formatDescriptionWithTimestamp(pullRequestDescription);
             const pr = (await (0, utils_1.callGitHubProxy)({
                 method: "POST",
                 url: `https://api.github.com/repos/${owner}/${repo}/pulls`,
                 body: {
-                    title: commitMessage,
+                    title: pullRequestTitle,
                     head: branchName,
                     base: "main",
                     body: initialDescription,

package/dist/tools/grep/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/grep/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAc,MAAM,wBAAwB,CAAC;AAoH/D,eAAO,MAAM,QAAQ,EAAE,IAkBtB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/grep/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAc,MAAM,wBAAwB,CAAC;AAqH/D,eAAO,MAAM,QAAQ,EAAE,IAkBtB,CAAC"}

package/dist/tools/grep/index.js

@@ -72,8 +72,9 @@ async function usingSystemGrep(input) {
 async function usingRipgrep(input) {
     try {
         const dir = path_1.default.join(process.cwd(), input.directory || "");
+        const escapedPattern = input.pattern.replace(/\s+/g, "\\ ");
         const results = await (0, ripgrep_1.ripgrep)(dir, {
-            string: input.pattern,
+            string: escapedPattern,
             globs: input.filePattern ? [input.filePattern] : undefined,
         });
         const resultsSummary = results
@@ -105,7 +106,7 @@ All paths are relative to the current working directory.`;
         console.error("Error executing ripgrep", error);
         return {
             isError: true,
-            result: error instanceof Error ? error.message : String(error),
+            result: error instanceof ripgrep_1.RipGrepError ? error.message : String(error),
         };
     }
 }

package/dist/tools/str_replace_editor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"str_replace_editor.d.ts","sourceRoot":"","sources":["../../src/tools/str_replace_editor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AA2B1D,UAAU,eAAe;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwC1D;AAMD;;;GAGG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,UAAU,CAAC,CA6JrB;AA+FD,eAAO,MAAM,eAAe,EAAE,IAAI,EAKjC,CAAC"}
+{"version":3,"file":"str_replace_editor.d.ts","sourceRoot":"","sources":["../../src/tools/str_replace_editor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AA2B1D,UAAU,eAAe;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAqED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwC1D;AAMD;;;GAGG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,UAAU,CAAC,CA0KrB;AA+FD,eAAO,MAAM,eAAe,EAAE,IAAI,EAKjC,CAAC"}

package/dist/tools/str_replace_editor.js

@@ -25,6 +25,64 @@ function restoreBackup(filePath) {
         fs_1.default.unlinkSync(backupPath);
     }
 }
+/**
+ * While running str_replace command, we've seen LLM can struggle to send unique old_str.
+ * This function tries to find unique contexts for each occurrence of old_str, so that the error
+ * message to the LLM is more informative, enabling it to pick a unique old_str in the next attempt.
+ */
+function getUniqueOccurences(contents, old_str) {
+    const lines = contents.split("\n");
+    const lineIndices = [];
+    lines.forEach((line, index) => {
+        if (line.includes(old_str)) {
+            lineIndices.push(index);
+        }
+    });
+    if (lineIndices.length === 0) {
+        return [];
+    }
+    if (lineIndices.length == 1) {
+        // line number should be 1-indexed, hence we add 1 to the line index
+        return [{ uniqueContext: old_str, lineNumber: lineIndices[0] + 1 }];
+    }
+    const uniqueContexts = [];
+    const maxContextSize = Math.min(10, lines.length);
+    for (const lineIndex of lineIndices) {
+        let contextSize = 0; // Start with just the line itself
+        let isUnique = false;
+        let context = "";
+        while (!isUnique && contextSize < maxContextSize) {
+            // Expand context to include more lines (both above and below)
+            const startLine = Math.max(0, lineIndex - contextSize);
+            const endLine = Math.min(lines.length - 1, lineIndex + contextSize);
+            // Extract the context as a group of lines
+            context = lines.slice(startLine, endLine + 1).join("\n");
+            // Check if this context is unique among all occurrences
+            isUnique = lineIndices.every((idx) => {
+                if (idx === lineIndex)
+                    return true; // Skip self
+                const otherStartLine = Math.max(0, idx - contextSize);
+                const otherEndLine = Math.min(lines.length - 1, idx + contextSize);
+                const otherContext = lines
+                    .slice(otherStartLine, otherEndLine + 1)
+                    .join("\n");
+                return context !== otherContext;
+            });
+            // If not unique, expand context by one more line in each direction
+            if (!isUnique) {
+                contextSize++;
+            }
+        }
+        // Only add context if we actually found a unique one
+        if (isUnique) {
+            uniqueContexts.push({
+                uniqueContext: context,
+                lineNumber: lineIndex + 1,
+            });
+        }
+    }
+    return uniqueContexts;
+}
 /**
  * Cleans up any backup files that were created during the editing process
  * @returns The number of backup files that were cleaned up
@@ -127,6 +185,7 @@ async function strReplaceEditorExecutor(input) {
                     throw new Error("file_text is required for create command");
                 }
                 fs_1.default.writeFileSync(filePath, input.file_text);
+                // TODO: Add type checking
                 return {
                     result: `Successfully created file ${filePath}`,
                     isError: false,
@@ -156,9 +215,18 @@ async function strReplaceEditorExecutor(input) {
                     const escapedOldStr = escapeRegExp(normalizedOldStr);
                     const occurences = normalizedContent.match(new RegExp(escapedOldStr, "g"));
                     if (occurences && occurences.length > 1) {
-                        // TODO: Help find unique matches
+                        const uniqueContexts = getUniqueOccurences(content, input.old_str);
+                        if (uniqueContexts.length === 0) {
+                            return {
+                                result: `Error: old_str found ${occurences.length} times in file: ${filePath}, but no unique contexts could be identified. Try using a more specific string.`,
+                                isError: true,
+                            };
+                        }
+                        const uniqueContextsString = uniqueContexts
+                            .map(({ uniqueContext, lineNumber }, idx) => `${idx + 1}. For occurence at line number ${lineNumber}, unique context is:\n\`\`\`\n${uniqueContext}\n\`\`\`\n`)
+                            .join("\n");
                         return {
-                            result: `Error: old_str found ${occurences.length} times in file: ${filePath}. Please provide more context to make a unique match.`,
+                            result: `Error: old_str found ${occurences.length} times in file: ${filePath}. Please use one of these unique contexts instead:\n\n${uniqueContextsString}`,
                             isError: true,
                         };
                     }

package/dist/tools/test-gen-browser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAyFnD,eAAO,MAAM,4BAA4B,EAAE,IA0E1C,CAAC"}
+{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAyFnD,eAAO,MAAM,4BAA4B,EAAE,IA8E1C,CAAC"}

package/dist/tools/test-gen-browser.js

@@ -86,7 +86,7 @@ exports.generateTestWithBrowserAgent = {
         description: BROWSER_AGENT_DESCRIPTION,
         parameters: BrowserAgentSchema,
     },
-    execute: async (input) => {
+    execute: async (input, trace) => {
         const { testName, testSuites, fileName, changeToMake, project } = input;
         const playwrightConfig = await (0, utils_1.readPlaywrightConfig)(process.cwd());
         const validProjectNames = await (0, utils_1.getValidProjectNames)(playwrightConfig);
@@ -120,6 +120,7 @@ exports.generateTestWithBrowserAgent = {
             testFilePath: fileName,
             filePathToUpdate: fileName,
             pwProjectsFilter: [project],
+            traceId: trace?.id,
             testGenToken: (0, scenarios_1.buildTokenFromOptions)({
                 name: testName,
                 file: fileName,

package/dist/utils/checkpoint.d.ts

@@ -0,0 +1,3 @@
+import { PendingToolCall } from "@empiricalrun/llm/chat";
+export declare function createCheckpoint(toolCalls: PendingToolCall[], branchName: string): Promise<void>;
+//# sourceMappingURL=checkpoint.d.ts.map

package/dist/utils/checkpoint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint.d.ts","sourceRoot":"","sources":["../../src/utils/checkpoint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAIzD,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,eAAe,EAAE,EAC5B,UAAU,EAAE,MAAM,iBAcnB"}

package/dist/utils/checkpoint.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createCheckpoint = void 0;
+const git_1 = require("./git");
+async function createCheckpoint(toolCalls, branchName) {
+    const filesChanged = await (0, git_1.getFilesChanged)();
+    const toolsWithUpdatedFiles = toolCalls
+        .filter((tc) => tc.input.path && filesChanged.includes(tc.input.path))
+        .map((toolCall) => ({
+        name: toolCall.name,
+        path: toolCall.input.path,
+    }));
+    const filesToCommit = toolsWithUpdatedFiles.map((tool) => tool.path);
+    if (toolsWithUpdatedFiles.length > 0) {
+        let commitMessage = `Toolcall Checkpoint: ${toolsWithUpdatedFiles.map((tool) => `${tool.name} on ${tool.path}`).join(", ")}`;
+        await (0, git_1.commitFilesAndPushBranch)(commitMessage, branchName, filesToCommit);
+    }
+}
+exports.createCheckpoint = createCheckpoint;

package/dist/utils/git.d.ts

@@ -2,4 +2,8 @@ export declare function getGitDiff(filepath: string): string;
 export declare function checkoutBranch(branchName: string): Promise<void>;
 export declare function commitAsBotUser(commitMessage: string): Promise<void>;
 export declare function commitLocalAndPushBranchToRemote(branchName: string): Promise<void>;
+export declare function getCurrentBranchName(): Promise<string>;
+export declare function pullBranch(branchName: string): Promise<void>;
+export declare function commitFilesAndPushBranch(commitMessage: string, branchName: string, files: string[]): Promise<void>;
+export declare function getFilesChanged(): Promise<string[]>;
 //# sourceMappingURL=git.d.ts.map

package/dist/utils/git.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/utils/git.ts"],"names":[],"mappings":"AAKA,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED,wBAAsB,cAAc,CAAC,UAAU,EAAE,MAAM,iBAQtD;AAED,wBAAsB,eAAe,CAAC,aAAa,EAAE,MAAM,iBAS1D;AAED,wBAAsB,gCAAgC,CAAC,UAAU,EAAE,MAAM,iBAIxE"}
+{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/utils/git.ts"],"names":[],"mappings":"AAKA,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED,wBAAsB,cAAc,CAAC,UAAU,EAAE,MAAM,iBAQtD;AAED,wBAAsB,eAAe,CAAC,aAAa,EAAE,MAAM,iBAS1D;AAED,wBAAsB,gCAAgC,CAAC,UAAU,EAAE,MAAM,iBAIxE;AAED,wBAAsB,oBAAoB,oBAGzC;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,iBAElD;AAED,wBAAsB,wBAAwB,CAC5C,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EAAE,iBAOhB;AAED,wBAAsB,eAAe,sBAQpC"}

package/dist/utils/git.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.commitLocalAndPushBranchToRemote = exports.commitAsBotUser = exports.checkoutBranch = exports.getGitDiff = void 0;
+exports.getFilesChanged = exports.commitFilesAndPushBranch = exports.pullBranch = exports.getCurrentBranchName = exports.commitLocalAndPushBranchToRemote = exports.commitAsBotUser = exports.checkoutBranch = exports.getGitDiff = void 0;
 const child_process_1 = require("child_process");
 const GIT_USER_NAME = "empiricalrun[bot]";
 const GIT_USER_EMAIL = "180257021+empiricalrun[bot]@users.noreply.github.com";
@@ -37,3 +37,28 @@ async function commitLocalAndPushBranchToRemote(branchName) {
     (0, child_process_1.execSync)(`git push origin ${branchName}`);
 }
 exports.commitLocalAndPushBranchToRemote = commitLocalAndPushBranchToRemote;
+async function getCurrentBranchName() {
+    const branchName = (0, child_process_1.execSync)("git branch --show-current").toString().trim();
+    return branchName;
+}
+exports.getCurrentBranchName = getCurrentBranchName;
+async function pullBranch(branchName) {
+    (0, child_process_1.execSync)(`git pull origin ${branchName}`);
+}
+exports.pullBranch = pullBranch;
+async function commitFilesAndPushBranch(commitMessage, branchName, files) {
+    (0, child_process_1.execSync)(`git add ${files.join(" ")}`);
+    await commitAsBotUser(commitMessage || "Intermediate commit from @empiricalrun/test-gen");
+    (0, child_process_1.execSync)(`git push origin ${branchName}`);
+}
+exports.commitFilesAndPushBranch = commitFilesAndPushBranch;
+async function getFilesChanged() {
+    const output = (0, child_process_1.execSync)("git status --porcelain").toString();
+    let filesChanged = output
+        .split("\n")
+        .map((line) => line.trim().split(" ").pop())
+        .filter((fileName) => !!fileName)
+        .filter((fileName) => fileName != "");
+    return filesChanged;
+}
+exports.getFilesChanged = getFilesChanged;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.56.4",
+  "version": "0.57.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -60,7 +60,7 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "zod": "^3.23.8",
-    "@empiricalrun/llm": "^0.15.2",
+    "@empiricalrun/llm": "^0.15.3",
     "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/test-run": "^0.8.4"
   },