@empiricalrun/test-gen 0.75.0 → 0.77.0

package/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @empiricalrun/test-gen
 
+## 0.77.0
+
+### Minor Changes
+
+- bc38c65: feat: diff tab ui update with code review comments
+- e5e45e0: feat: code review agent output format update to v2
+- 44373a4: feat: code review agent system prompt and ui update
+- 8a206cf: feat: optional line annotated diff on /diff endpoint for code review agent
+- 151dc8f: feat: fetchVideoAnalysis uses VideoAnalysisAgent under the hood
+- 49b44a6: feat: cr agent auto refresh ui with fresh code review data
+- 82a6fe7: refactor: chat model update in base agent v2
+- f9a7023: feat: tool exec v2 integration with base agent v2
+- a25735a: feat: added trace in code review agent
+
+### Patch Changes
+
+- a45714d: fix: updated prompt to ask agent to format code description in markdown
+- ba65440: feat: convert image url to base64 uri before sending to llm
+- cf991ee: chore: file view tool for code review agent
+- eda0ccd: feat: support tool response image parts with urls
+- 1f7e8b0: chore: diff endpoint now also returns pr metadata, code review agent receives pr metadata
+- 42e19ee: fix: removed duplicate trigger on create pr tool for code review
+- 02c3532: fix: graceful handling of missing verdict in xml - infer verdict from cr agent line comments
+- 8f3a25f: feat: versioned code review responses
+- bd61b13: feat: change default model to gpt-5
+- Updated dependencies [8ff49e5]
+- Updated dependencies [ba65440]
+- Updated dependencies [eda0ccd]
+- Updated dependencies [25bd9b6]
+- Updated dependencies [d56954c]
+  - @empiricalrun/llm@0.23.0
+  - @empiricalrun/test-run@0.11.1
+
+## 0.76.0
+
+### Minor Changes
+
+- 9c9d6e5: feat: code review basic agent
+- ee8ce00: feat: fetchVideoAnalysis multi modal tool response
+- 86a1549: feat: code review agent v1 with workflow as an ad hoc tool call
+- 5339d2c: feat: VideoAnalysisAgent working in CLI with extractFrames tool
+
+### Patch Changes
+
+- a6aa6d8: fix: bug where video analysis was set to empty string
+  - @empiricalrun/llm@0.22.0
+  - @empiricalrun/test-run@0.11.1
+
 ## 0.75.0
 
 ### Minor Changes

package/dist/agent/base/index.d.ts

@@ -1,36 +1,47 @@
-import { IChatModel } from "@empiricalrun/llm/chat";
-import { FileInfo, PendingToolCall, ReporterFunction, StreamingMessageReporterFunc, SupportedChatModels, ToolDefinition, TraceClient } from "@empiricalrun/shared-types";
+import { type IChatCanonicalModel } from "@empiricalrun/llm/chat";
+import type { Attachment, CanonicalMessage, ChatState, FileInfo, MessageCheckpoint, PendingToolCall, ReporterFunction, StreamingMessageReporterFunc, SupportedChatModels, ToolResult, ToolsForLLM, TraceClient } from "@empiricalrun/shared-types";
+import { BaseToolExecutor } from "../../tools/executor/base";
 type RunLoopParams<T> = {
-    chatModel: IChatModel<T>;
     reporter: ReporterFunction;
     streamingMessageReporter?: StreamingMessageReporterFunc;
     trace?: TraceClient;
     signal?: AbortSignal;
-    onPendingToolCall: (toolCalls: PendingToolCall[]) => Promise<void>;
+    onPendingToolCall?: (toolCalls: PendingToolCall[]) => Promise<void>;
     onLLMResponse?: (response: T, selectedModel: SupportedChatModels) => Promise<void>;
+    repoInfoBuilder?: () => Promise<FileInfo>;
 };
-export declare const getChatAgentMode: (source: string | undefined) => "triage" | "test_generation";
-type AgentParams = {
-    featureFlags: string[];
+export type AgentParams = {
     selectedModel: SupportedChatModels;
-    repoInfo: FileInfo;
+    chatState: ChatState;
+    featureFlags?: string[];
+    workerEnv?: Record<string, string>;
+    toolExecutor?: BaseToolExecutor;
+    trace?: TraceClient;
 };
-export declare function getAgentForMode(isTriage: boolean, params: AgentParams): TriageAgent | ChatAgent;
-export declare class ChatAgent {
+export declare abstract class BaseAgent {
     featureFlags: string[];
-    tools: ToolDefinition[];
+    tools: ToolsForLLM;
     selectedModel: SupportedChatModels;
-    repoInfo: FileInfo;
-    constructor({ featureFlags, selectedModel, repoInfo }: AgentParams);
-    runLoop(params: RunLoopParams<any>): Promise<void>;
-}
-export declare class TriageAgent {
-    featureFlags: string[];
-    tools: ToolDefinition[];
-    selectedModel: SupportedChatModels;
-    repoInfo: FileInfo;
-    constructor({ featureFlags, selectedModel, repoInfo }: AgentParams);
+    chatModel: IChatCanonicalModel;
+    workerEnv?: Record<string, string>;
+    toolExecutor?: BaseToolExecutor;
+    trace?: TraceClient;
+    constructor({ featureFlags, selectedModel, workerEnv, chatState, toolExecutor, trace, }: AgentParams);
+    protected abstract getTools(): ToolsForLLM;
+    protected abstract buildSystemPrompt(repoContext?: string): Promise<string>;
+    get messages(): CanonicalMessage[];
+    get askUserForInput(): boolean;
+    get chatState(): ChatState;
+    pushUserMessage(text: string, attachments: Attachment[]): void;
+    setChatState(chatState: ChatState): void;
+    stop(): ChatState;
+    processToolResults(toolCalls: PendingToolCall[], toolResults: ToolResult[], checkpoint: MessageCheckpoint | null): ChatState | undefined;
+    toolCallHandler(toolCalls: PendingToolCall[]): Promise<void>;
     runLoop(params: RunLoopParams<any>): Promise<void>;
+    getHumanReadableLatestMessage(): {
+        role: string;
+        textMessage: string;
+    } | undefined;
 }
 export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/base/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/base/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EACL,QAAQ,EACR,eAAe,EACf,gBAAgB,EAChB,4BAA4B,EAC5B,mBAAmB,EACnB,cAAc,EACd,WAAW,EACZ,MAAM,4BAA4B,CAAC;AAMpC,KAAK,aAAa,CAAC,CAAC,IAAI;IACtB,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACzB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,wBAAwB,CAAC,EAAE,4BAA4B,CAAC;IACxD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,iBAAiB,EAAE,CAAC,SAAS,EAAE,eAAe,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,aAAa,CAAC,EAAE,CACd,QAAQ,EAAE,CAAC,EACX,aAAa,EAAE,mBAAmB,KAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;CACpB,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,QAAQ,MAAM,GAAG,SAAS,iCAW1D,CAAC;AAEF,KAAK,WAAW,GAAG;IACjB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,QAAQ,CAAC;CACpB,CAAC;AAEF,wBAAgB,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,2BAMrE;AAED,qBAAa,SAAS;IACpB,YAAY,EAAE,MAAM,EAAE,CAAM;IAC5B,KAAK,EAAE,cAAc,EAAE,CAAM;IAC7B,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,QAAQ,CAAC;gBAEP,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,WAAW;IAO5D,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC;CAUzC;AAED,qBAAa,WAAW;IACtB,YAAY,EAAE,MAAM,EAAE,CAAM;IAC5B,KAAK,EAAE,cAAc,EAAE,CAAM;IAC7B,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,QAAQ,CAAC;gBAEP,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,WAAW;IAO5D,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC;CAUzC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/base/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,4BAA4B,EAC5B,mBAAmB,EACnB,UAAU,EACV,WAAW,EACX,WAAW,EACZ,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAO7D,KAAK,aAAa,CAAC,CAAC,IAAI;IACtB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,wBAAwB,CAAC,EAAE,4BAA4B,CAAC;IACxD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,iBAAiB,CAAC,EAAE,CAAC,SAAS,EAAE,eAAe,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE,aAAa,CAAC,EAAE,CACd,QAAQ,EAAE,CAAC,EACX,aAAa,EAAE,mBAAmB,KAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;CAC3C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,aAAa,EAAE,mBAAmB,CAAC;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,CAAC;AAEF,8BAAsB,SAAS;IAC7B,YAAY,EAAE,MAAM,EAAE,CAAM;IAC5B,KAAK,EAAE,WAAW,CAAC;IACnB,aAAa,EAAE,mBAAmB,CAAC;IACnC,SAAS,EAAE,mBAAmB,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;gBAER,EACV,YAAY,EACZ,aAAa,EACb,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,GACN,EAAE,WAAW;IAed,SAAS,CAAC,QAAQ,CAAC,QAAQ,IAAI,WAAW;IAC1C,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAE3E,IAAI,QAAQ,IAAI,gBAAgB,EAAE,CAEjC;IAED,IAAI,eAAe,IAAI,OAAO,CAE7B;IAED,IAAI,SAAS,IAAI,SAAS,CAQzB;IAED,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,IAAI;IAI9D,YAAY,CAAC,SAAS,EAAE,SAAS;IAQjC,IAAI,IAAI,SAAS;IA6BjB,kBAAkB,CAChB,SAAS,EAAE,eAAe,EAAE,EAC5B,WAAW,EAAE,UAAU,EAAE,EACzB,UAAU,EAAE,iBAAiB,GAAG,IAAI,GACnC,SAAS,GAAG,SAAS;IA4BlB,eAAe,CAAC,SAAS,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5D,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC;IAmBxC,6BAA6B,IACzB;QACE,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;KACrB,GACD,SAAS;CAGd"}

package/dist/agent/base/index.js

@@ -1,74 +1,117 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TriageAgent = exports.ChatAgent = exports.getChatAgentMode = void 0;
-exports.getAgentForMode = getAgentForMode;
-const tools_1 = require("../../tools");
+exports.BaseAgent = void 0;
+const chat_1 = require("@empiricalrun/llm/chat");
 const agent_loop_1 = require("../chat/agent-loop");
-const prompt_1 = require("../chat/prompt");
-const getChatAgentMode = (source) => {
-    switch (source) {
-        case "triage":
-            return "triage";
-        case "dashboard":
-            return "test_generation";
-        case "manual":
-            return "test_generation";
-        default:
-            return "test_generation";
-    }
-};
-exports.getChatAgentMode = getChatAgentMode;
-function getAgentForMode(isTriage, params) {
-    if (isTriage) {
-        return new TriageAgent(params);
-    }
-    else {
-        return new ChatAgent(params);
-    }
-}
-class ChatAgent {
+const repo_1 = require("../chat/prompt/repo");
+const state_1 = require("../chat/state");
+const REJECTION_RESULT = "This tool call was rejected by the user.";
+class BaseAgent {
     featureFlags = [];
-    tools = [];
+    tools;
     selectedModel;
-    repoInfo;
-    constructor({ featureFlags, selectedModel, repoInfo }) {
-        this.featureFlags = featureFlags;
-        this.tools = (0, tools_1.availableTools)(selectedModel, false, []);
+    chatModel;
+    workerEnv;
+    toolExecutor;
+    trace;
+    constructor({ featureFlags, selectedModel, workerEnv, chatState, toolExecutor, trace, }) {
+        this.featureFlags = featureFlags || [];
         this.selectedModel = selectedModel;
-        this.repoInfo = repoInfo;
+        this.workerEnv = workerEnv;
+        this.tools = this.getTools();
+        this.toolExecutor = toolExecutor;
+        this.chatModel = (0, chat_1.createChatModel)(chatState.messages, this.selectedModel, this.workerEnv);
+        this.chatModel.validateEnvVarsForAuth();
+        this.trace = trace;
     }
-    async runLoop(params) {
-        const mode = (0, exports.getChatAgentMode)("test_generation");
-        return (0, agent_loop_1.chatAgentLoop)({
-            ...params,
-            tools: this.tools,
+    get messages() {
+        return this.chatModel.messages;
+    }
+    get askUserForInput() {
+        return this.chatModel.askUserForInput || false;
+    }
+    get chatState() {
+        return (0, state_1.createChatStateForMessages)({
+            messages: this.chatModel.messages,
             selectedModel: this.selectedModel,
-            systemPrompt: await (0, prompt_1.getSystemPrompt)(this.repoInfo, mode),
-            featureFlags: this.featureFlags,
+            askUserForInput: this.chatModel.askUserForInput,
+            // TODO: Test out the error scenario
+            error: null,
         });
     }
-}
-exports.ChatAgent = ChatAgent;
-class TriageAgent {
-    featureFlags = [];
-    tools = [];
-    selectedModel;
-    repoInfo;
-    constructor({ featureFlags, selectedModel, repoInfo }) {
-        this.featureFlags = featureFlags;
-        this.tools = (0, tools_1.availableTools)(selectedModel, true, []);
-        this.selectedModel = selectedModel;
-        this.repoInfo = repoInfo;
+    pushUserMessage(text, attachments) {
+        this.chatModel.pushUserMessage(text, attachments);
+    }
+    setChatState(chatState) {
+        this.chatModel = (0, chat_1.createChatModel)(chatState.messages, this.selectedModel, this.workerEnv);
+    }
+    stop() {
+        const pendingToolCalls = this.chatModel.getPendingToolCalls();
+        let updatedChatState = {
+            ...this.chatState,
+            messages: this.chatModel.messages,
+            askUserForInput: true,
+        };
+        if (pendingToolCalls.length > 0) {
+            const toolResults = pendingToolCalls.map(() => ({
+                isError: true,
+                result: REJECTION_RESULT,
+                artifacts: null,
+            }));
+            this.chatModel.pushToolResultsMessage(pendingToolCalls, toolResults, null);
+            updatedChatState = {
+                ...updatedChatState,
+                messages: this.chatModel.messages,
+            };
+        }
+        return updatedChatState;
+    }
+    processToolResults(toolCalls, toolResults, checkpoint) {
+        const pendingToolCalls = this.chatModel.getPendingToolCalls() || [];
+        // Filter tool calls to only include pending ones
+        const filteredToolResults = [];
+        const filteredToolCalls = toolCalls.filter((call, index) => {
+            if (pendingToolCalls.some((p) => p.id === call.id)) {
+                filteredToolResults.push(toolResults[index]);
+                return true;
+            }
+            return false;
+        });
+        if (filteredToolCalls.length > 0) {
+            this.chatModel.pushToolResultsMessage(filteredToolCalls, filteredToolResults, checkpoint);
+            return {
+                ...this.chatState,
+                messages: this.chatModel.messages,
+                askUserForInput: this.chatModel.askUserForInput,
+            };
+        }
+        return undefined;
+    }
+    async toolCallHandler(toolCalls) {
+        if (!this.toolExecutor) {
+            throw new Error("No toolExecutor available and no onPendingToolCall provided");
+        }
+        const { toolResults, checkpoint } = await this.toolExecutor.execute(toolCalls);
+        this.processToolResults(toolCalls, toolResults, checkpoint);
     }
     async runLoop(params) {
-        const mode = (0, exports.getChatAgentMode)("triage");
-        return (0, agent_loop_1.chatAgentLoop)({
+        let repoContext;
+        if (params.repoInfoBuilder) {
+            const repoInfo = await params.repoInfoBuilder();
+            repoContext = await (0, repo_1.getRepoInfoPrompt)(repoInfo);
+        }
+        const systemPrompt = await this.buildSystemPrompt(repoContext);
+        await (0, agent_loop_1.chatAgentLoop)({
             ...params,
-            tools: this.tools,
-            selectedModel: this.selectedModel,
-            systemPrompt: await (0, prompt_1.getSystemPrompt)(this.repoInfo, mode),
+            onPendingToolCall: params.onPendingToolCall || this.toolCallHandler.bind(this),
+            chatModel: this.chatModel,
+            tools: this.getTools(),
+            systemPrompt,
             featureFlags: this.featureFlags,
         });
     }
+    getHumanReadableLatestMessage() {
+        return this.chatModel.getHumanReadableLatestMessage();
+    }
 }
-exports.TriageAgent = TriageAgent;
+exports.BaseAgent = BaseAgent;

package/dist/agent/browsing/run.d.ts

@@ -1,4 +1,4 @@
-import { ArtifactInput, Usage } from "@empiricalrun/shared-types";
+import { Usage } from "@empiricalrun/shared-types";
 import { BrowserAgentResult } from "../cua";
 type GenerateTestsType = {
     testCaseName: string;
@@ -22,7 +22,6 @@ export declare function runBrowsingAgent({ testCaseName, testCaseSuites, testFil
     error: string;
     result: BrowserAgentResult | undefined;
     usage: Usage | undefined;
-    artifacts: ArtifactInput[];
 }>;
 export {};
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAclE,OAAO,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AAG5C,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,OAAO,CAAC;IACnC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC,CAAC;AAEF,wBAAsB,8BAA8B,CAAC,EACnD,gBAAgB,EAChB,OAAO,EACP,YAAY,GACb,EAAE;IACD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAQlB;AAED,wBAAsB,gBAAgB,CAAC,EACrC,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,OAAO,EACP,OAAO,EACP,yBAAyB,EACzB,YAAY,GACb,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,kBAAkB,GAAG,SAAS,CAAC;IACvC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC;IACzB,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B,CAAC,CAmGD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAanD,OAAO,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AAG5C,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,OAAO,CAAC;IACnC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC,CAAC;AAEF,wBAAsB,8BAA8B,CAAC,EACnD,gBAAgB,EAChB,OAAO,EACP,YAAY,GACb,EAAE;IACD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAQlB;AAED,wBAAsB,gBAAgB,CAAC,EACrC,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,OAAO,EACP,OAAO,EACP,yBAAyB,EACzB,YAAY,GACb,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,kBAAkB,GAAG,SAAS,CAAC;IACvC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC;CAC1B,CAAC,CA+FD"}

package/dist/agent/browsing/run.js

@@ -9,14 +9,13 @@ const test_run_1 = require("@empiricalrun/test-run");
 const detect_port_1 = __importDefault(require("detect-port"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
-const utils_1 = require("../../artifacts/utils");
 const web_1 = require("../../bin/utils/platform/web");
 const server_1 = require("../../file/server");
 const exec_1 = require("../../utils/exec");
-const utils_2 = require("./utils");
+const utils_1 = require("./utils");
 async function convertProjectsFilterToProject({ pwProjectsFilter, repoDir, testFilePath, }) {
-    const playwrightConfig = await (0, utils_2.readPlaywrightConfig)(repoDir);
-    const project = await (0, utils_2.detectProjectName)(testFilePath, playwrightConfig, pwProjectsFilter);
+    const playwrightConfig = await (0, utils_1.readPlaywrightConfig)(repoDir);
+    const project = await (0, utils_1.detectProjectName)(testFilePath, playwrightConfig, pwProjectsFilter);
     return project;
 }
 async function runBrowsingAgent({ testCaseName, testCaseSuites, testFilePath, filePathToUpdate, projectName, testGenToken, repoDir, traceId, editFileWithGeneratedCode, envOverrides, }) {
@@ -92,17 +91,12 @@ async function runBrowsingAgent({ testCaseName, testCaseSuites, testFilePath, fi
             console.error(`[generateTestsUsingMasterAgent] Failed to remove extra scripts from files post test gen error:`, e);
         }
     }
-    const artifacts = [
-        ...((0, utils_1.findPlaywrightArtifacts)(repoDir) || []),
-        ...(fileServer.getArtifactInputsFromServer() || []),
-    ];
     await fileServer.stop();
     const { result, usage } = fileServer.getResultAndUsage();
     return {
         isError: !!error,
         error: error || "",
         result,
-        artifacts,
         usage,
     };
 }

package/dist/agent/browsing/utils.d.ts

@@ -1,21 +1,14 @@
-import { TraceClient } from "@empiricalrun/llm";
-import type { TestCase } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
 import { PlaywrightTestConfig } from "playwright/test";
 export declare function isRegExp(obj: any): obj is RegExp;
 export declare function prepareBrowsingAgentTask(steps: string[]): string;
-export declare function addImportForMethod(testFilePath: string, methodName: string): Promise<void>;
-export declare function replaceTodoWithCreateTest(testFilePath: string, repoDir: string): Promise<void>;
+export declare function addImportForMethod(testFilePath: string, methodName: string): void;
+export declare function replaceTodoWithCreateTest(testFilePath: string, repoDir: string): string | undefined;
 export declare function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }: {
     testCaseName: string;
     testCaseSuites: string[];
     specPath: string;
 }): Promise<void>;
-export declare function prepareFileForMasterAgent({ testCase, specPath, trace, }: {
-    testCase: TestCase;
-    specPath: string;
-    trace?: TraceClient;
-}): Promise<string>;
 export declare function injectPwLocatorGenerator(page: Page): Promise<void>;
 /**
  * function to read playwright config from the source repo

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,kBAAkB,CACtC,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,iBAgBnB;AAgED,wBAAsB,yBAAyB,CAC7C,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,iBAwBhB;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA6HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CA2B/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAYvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAe1E;AAED,wBAAgB,yBAAyB,CACvC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAsBpB;AAED,wBAAsB,cAAc,CAAC,EACnC,YAAY,EACZ,cAAc,EACd,QAAQ,GACT,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,iBAoBA;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA6HxD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CA2B/B;AAWD,wBAAsB,oBAAoB,CACxC,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,EAAE,CAAC,CAQnB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB"}

package/dist/agent/browsing/utils.js

@@ -8,7 +8,6 @@ exports.prepareBrowsingAgentTask = prepareBrowsingAgentTask;
 exports.addImportForMethod = addImportForMethod;
 exports.replaceTodoWithCreateTest = replaceTodoWithCreateTest;
 exports.markTestAsOnly = markTestAsOnly;
-exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
 exports.injectPwLocatorGenerator = injectPwLocatorGenerator;
 exports.readPlaywrightConfig = readPlaywrightConfig;
 exports.getValidProjectNames = getValidProjectNames;
@@ -18,13 +17,7 @@ const minimatch_1 = require("minimatch");
 const path_1 = __importDefault(require("path"));
 // For TypeScript type safety
 let tsxImport = null;
-const logger_1 = require("../../bin/logger");
-const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
-const create_test_block_1 = require("../codegen/create-test-block");
-const fix_ts_errors_1 = require("../codegen/fix-ts-errors");
-const lexical_scoped_vars_1 = require("../codegen/lexical-scoped-vars");
-const update_flow_1 = require("../codegen/update-flow");
 function isRegExp(obj) {
     return (obj instanceof RegExp ||
         Object.prototype.toString.call(obj) === "[object RegExp]");
@@ -34,7 +27,7 @@ function prepareBrowsingAgentTask(steps) {
     const task = `${sanitizedSteps.join("\n")}\n`;
     return task;
 }
-async function addImportForMethod(testFilePath, methodName) {
+function addImportForMethod(testFilePath, methodName) {
     // Instead of using "@empiricalrun/test-gen", we use the local dist file
     // This is to avoid assuming that the test-gen package is installed in the project
     const importSource = path_1.default.join(__dirname, "../../../dist/index.js");
@@ -43,60 +36,7 @@ async function addImportForMethod(testFilePath, methodName) {
     }
     fs_1.default.writeFileSync(testFilePath, (0, web_1.addNewImport)(fs_1.default.readFileSync(testFilePath, "utf-8"), [methodName], importSource));
 }
-async function prepareFileForUpdateScenario({ testCase, specPath, trace, }) {
-    const { name, suites } = testCase;
-    await (0, web_1.addUserContextFixture)({
-        scenarioName: name,
-        filePath: specPath,
-        suites,
-    });
-    // update the test case with appropriate location for createTest
-    const [suggestion] = await (0, update_flow_1.appendCreateTestBlock)({
-        trace,
-        testCase,
-        file: specPath,
-        validateTypes: false,
-    });
-    const createTestFilePath = suggestion?.updatedFiles[0] || "";
-    console.log("appending to existing test block");
-    console.log("updated test file path", createTestFilePath);
-    const fetchScopeVariablesSpan = trace?.span({
-        name: "fetch-scope-variables",
-        input: {
-            createTestFilePath,
-        },
-    });
-    const scopeVariables = await (0, lexical_scoped_vars_1.getLexicalScopedVars)({
-        file: fs_1.default.readFileSync(createTestFilePath, "utf-8"),
-        referencePoint: "await createTest",
-        trace: fetchScopeVariablesSpan,
-    });
-    fetchScopeVariablesSpan?.end({
-        name: "fetch-scope-variables",
-        output: {
-            variables: scopeVariables,
-        },
-    });
-    await (0, web_1.appendScopeToCreateTest)(createTestFilePath, scopeVariables);
-    await addImportForMethod(createTestFilePath, "createTest");
-    const { pomPrompt, nonSpecFilePrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
-    await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
-        trace,
-        file: createTestFilePath,
-        pomCode: pomPrompt,
-        nonSpecFileCode: nonSpecFilePrompt,
-        testCase: testCase,
-    });
-    await markTestAsOnly({
-        testCaseName: name,
-        testCaseSuites: suites,
-        specPath,
-    });
-    return createTestFilePath;
-}
-async function replaceTodoWithCreateTest(testFilePath, repoDir) {
-    // This method is an alternative to prepareFileForUpdateScenario
-    // TODO: Does not support scoped variables and updates in POM files
+function replaceTodoWithCreateTest(testFilePath, repoDir) {
     const absoluteTestFilePath = path_1.default.join(repoDir, testFilePath);
     const fileContent = fs_1.default.readFileSync(absoluteTestFilePath, "utf-8");
     const todoRegex = /\/\/ TODO\(agent(?:\s+on\s+(\w+))?\):\s*(.*)/;
@@ -104,10 +44,11 @@ async function replaceTodoWithCreateTest(testFilePath, repoDir) {
     if (!todoMatch) {
         throw new Error(`No "// TODO(agent):" or "// TODO(agent on pageName):" comment found in file: ${testFilePath}`);
     }
-    const [, pageVarName] = todoMatch;
+    const [, pageVarName, todoContent] = todoMatch;
     const pageVariable = pageVarName || "page"; // Default to "page" if not specified
     fs_1.default.writeFileSync(absoluteTestFilePath, fileContent.replace(todoRegex, (_, __, todoText) => `await createTest("${todoText.replace(/"/g, '\\"')}", ${pageVariable});`));
-    await addImportForMethod(absoluteTestFilePath, "createTest");
+    addImportForMethod(absoluteTestFilePath, "createTest");
+    return todoContent;
 }
 async function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }) {
     const testFileContent = fs_1.default.readFileSync(specPath, "utf-8");
@@ -124,51 +65,6 @@ async function markTestAsOnly({ testCaseName, testCaseSuites, specPath, }) {
         fs_1.default.writeFileSync(specPath, updatedTestFileContent);
     }
 }
-async function prepareFileForMasterAgent({ testCase, specPath, trace, }) {
-    const prepareFileSpan = trace?.span({
-        name: "prepare-file-for-master-agent",
-    });
-    new logger_1.CustomLogger({ useReporter: true }).log(`Preparing file for master agent. [view trace](${trace?.getTraceUrl()})`);
-    const { name, suites } = testCase;
-    // check if the spec file exists
-    // if no then create a new file with test and expect imports
-    if (!fs_1.default.existsSync(specPath)) {
-        const fileCreateSpan = prepareFileSpan?.span({
-            name: "create-file",
-            input: { specPath },
-        });
-        fs_1.default.writeFileSync(specPath, "");
-        const fileContentWithImports = (0, web_1.addNewImport)("", ["test", "expect"], (0, web_1.getFixtureImportPath)(specPath));
-        fs_1.default.writeFileSync(specPath, fileContentWithImports, "utf-8");
-        fileCreateSpan?.end({ output: { specPath, fileContentWithImports } });
-    }
-    const existingContents = fs_1.default.readFileSync(specPath, "utf-8");
-    const { testBlock } = (0, web_1.getTypescriptTestBlock)({
-        scenarioName: name,
-        suites,
-        content: existingContents,
-    });
-    if (!testBlock) {
-        const newTestBlock = await (0, create_test_block_1.createEmptyTestCaseBlock)({
-            trace: prepareFileSpan,
-            testCase,
-            file: specPath,
-        });
-        fs_1.default.writeFileSync(specPath, `${existingContents} \n\n ${newTestBlock}`, "utf-8");
-        const updatedContent = (0, web_1.injectCodeSnippetBySuiteChain)({
-            testFileContent: existingContents,
-            suites: testCase.suites,
-            codeSnippet: `\n\n${newTestBlock}`,
-        });
-        fs_1.default.writeFileSync(specPath, updatedContent, "utf-8");
-    }
-    const updatePath = await prepareFileForUpdateScenario({
-        testCase,
-        specPath,
-        trace,
-    });
-    return updatePath;
-}
 function newContentsWithTestOnly(existingContents, originalTestBlock, updatedTestBlock, parentDescribeBlock) {
     if (!parentDescribeBlock) {
         const testMarkedAsOnly = updatedTestBlock.replace("test(", "test.only(");

package/dist/agent/chat/agent-loop.d.ts

@@ -1,17 +1,18 @@
 import { TraceClient } from "@empiricalrun/llm";
-import { IChatModel } from "@empiricalrun/llm/chat";
-import { CanonicalMessage, PendingToolCall, ReporterFunction, StreamingMessageReporterFunc, SupportedChatModels, ToolDefinition } from "@empiricalrun/shared-types";
-export declare function chatAgentLoop<T = CanonicalMessage>({ chatModel, selectedModel, reporter, streamingMessageReporter, trace, signal, featureFlags, tools, onPendingToolCall, onLLMResponse, systemPrompt, }: {
-    chatModel: IChatModel<T>;
-    selectedModel: SupportedChatModels;
+import { IChatCanonicalModel, IChatModel } from "@empiricalrun/llm/chat";
+import { CanonicalMessage, PendingToolCall, ReporterFunction, StreamingMessageReporterFunc, SupportedChatModels, ToolsForLLM } from "@empiricalrun/shared-types";
+type AgentLoopParams<T> = {
+    chatModel: IChatModel<T> | IChatCanonicalModel;
     reporter: ReporterFunction;
     streamingMessageReporter?: StreamingMessageReporterFunc;
     trace?: TraceClient;
     signal?: AbortSignal;
     featureFlags: string[];
-    tools: ToolDefinition[];
+    tools: ToolsForLLM;
     onPendingToolCall: (toolCalls: PendingToolCall[]) => Promise<void>;
     onLLMResponse?: (response: T, selectedModel: SupportedChatModels) => Promise<void>;
     systemPrompt: string;
-}): Promise<void>;
+};
+export declare function chatAgentLoop<T extends CanonicalMessage = CanonicalMessage>({ chatModel, reporter, streamingMessageReporter, trace, signal, featureFlags, tools, onPendingToolCall, onLLMResponse, systemPrompt, }: AgentLoopParams<T>): Promise<void>;
+export {};
 //# sourceMappingURL=agent-loop.d.ts.map

package/dist/agent/chat/agent-loop.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,4BAA4B,EAC5B,mBAAmB,EACnB,cAAc,EACf,MAAM,4BAA4B,CAAC;AAKpC,wBAAsB,aAAa,CAAC,CAAC,GAAG,gBAAgB,EAAE,EACxD,SAAS,EACT,aAAa,EACb,QAAQ,EACR,wBAAwB,EACxB,KAAK,EACL,MAAM,EACN,YAAY,EACZ,KAAK,EACL,iBAAiB,EACjB,aAAa,EACb,YAAY,GACb,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACzB,aAAa,EAAE,mBAAmB,CAAC;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,wBAAwB,CAAC,EAAE,4BAA4B,CAAC;IACxD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,iBAAiB,EAAE,CAAC,SAAS,EAAE,eAAe,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,aAAa,CAAC,EAAE,CACd,QAAQ,EAAE,CAAC,EACX,aAAa,EAAE,mBAAmB,KAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,CA2DhB"}
+{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,4BAA4B,EAC5B,mBAAmB,EACnB,WAAW,EACZ,MAAM,4BAA4B,CAAC;AAKpC,KAAK,eAAe,CAAC,CAAC,IAAI;IACxB,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAAC;IAC/C,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,wBAAwB,CAAC,EAAE,4BAA4B,CAAC;IACxD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,EAAE,WAAW,CAAC;IACnB,iBAAiB,EAAE,CAAC,SAAS,EAAE,eAAe,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,aAAa,CAAC,EAAE,CACd,QAAQ,EAAE,CAAC,EACX,aAAa,EAAE,mBAAmB,KAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,wBAAsB,aAAa,CACjC,CAAC,SAAS,gBAAgB,GAAG,gBAAgB,EAC7C,EACA,SAAS,EACT,QAAQ,EACR,wBAAwB,EACxB,KAAK,EACL,MAAM,EACN,YAAY,EACZ,KAAK,EACL,iBAAiB,EACjB,aAAa,EACb,YAAY,GACb,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA2CpC"}