@empiricalrun/test-gen 0.51.3 → 0.51.5

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @empiricalrun/test-gen
 
+## 0.51.5
+
+### Patch Changes
+
+- ae47eed: feat: add type checking to text editor tool call
+- beb704e: chore: move claude chat stuff to make space for gemini
+- 3d9c868: fix: system prompt for computer use agent
+- Updated dependencies [beb704e]
+- Updated dependencies [f283696]
+  - @empiricalrun/llm@0.11.4
+
+## 0.51.4
+
+### Patch Changes
+
+- 7613f20: fix: update test-run-fetcher tool call response to include project
+- f57ab88: fix: add playwright best practices to system prompt
+- 83a4788: feat: use cua in chat agent tool call with feature flag
+- a8c11eb: fix: tool call log should show error
+- Updated dependencies [307a1cf]
+  - @empiricalrun/llm@0.11.3
+
 ## 0.51.3
 
 ### Patch Changes

package/dist/agent/chat.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AA+EA,wBAAsB,SAAS,CAAC,EAC9B,SAAwC,EACxC,mBAAmB,GACpB,EAAE;IACD,SAAS,CAAC,EAAE,4BAA4B,GAAG,4BAA4B,CAAC;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B,mBA4GA"}
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AA+FA,wBAAsB,SAAS,CAAC,EAC9B,SAAwC,EACxC,mBAAmB,GACpB,EAAE;IACD,SAAS,CAAC,EAAE,4BAA4B,GAAG,4BAA4B,CAAC;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B,mBAkHA"}

package/dist/agent/chat.js

@@ -4,13 +4,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.chatAgent = void 0;
-const claude_1 = require("@empiricalrun/llm/claude");
+const chat_1 = require("@empiricalrun/llm/chat");
 const path_1 = __importDefault(require("path"));
 const picocolors_1 = require("picocolors");
+const web_1 = require("../bin/utils/platform/web");
 const human_in_the_loop_1 = require("../human-in-the-loop");
-const browser_agent_1 = require("../tools/browser-agent");
 const diagnosis_fetcher_1 = require("../tools/diagnosis-fetcher");
 const grep_1 = require("../tools/grep");
+const test_gen_browser_1 = require("../tools/test-gen-browser");
 const test_run_1 = require("../tools/test-run");
 const test_run_fetcher_1 = require("../tools/test-run-fetcher");
 const zod_schema_1 = require("../tools/zod-schema");
@@ -20,6 +21,8 @@ You are a helpful assistant that can answer questions and help with tasks.
 You are given a set of tools to use to fulfill the user's request. Read their descriptions to
 understand what each tool does.
 
+# Tools
+
 For example, if the user asks you to run a test, you could use the runTest tool.
 Once the test is run, you will receive the results in the form of a JSON object.
 Summarize the results in a few sentences.
@@ -46,6 +49,7 @@ the actual code to click on the login button. If you are fixing a failing test,
 around the failing line of code, so that it can be replaced/modified.
 
 # Repo context
+
 You are running as a CLI tool inside the directory of the repo where this test file is located. Here is
 the repo directory structure:
 
@@ -55,27 +59,38 @@ While specifying paths to files, use relative paths from the current working dir
 - Correct path: "tests/lesson.spec.ts"
 - Incorrect path: "/repo/tests/lesson.spec.ts" or "${path_1.default.basename(process.cwd())}/tests/lesson.spec.ts"
 
+# Rules for fixing Playwright tests
+
+You must follow these rules while adding new tests or modifying existing tests. There can be exceptions to these rules, but
+ONLY when explicitly asked for by the user.
+
+1. Do not add any conditional logic or try catch blocks in a test. A good test deterministically tests a user scenario
+2. Trust Playwright's ability to auto-wait while taking actions on elements. For example, do not add checks on locator.isVisible() before clicking on it: Playwright already does this
+3. Do not add waitForTimeout or waitForLoadState in a test. Playwright will automatically wait for the page to load.
+4. You can't delete some steps from the test to make it pass. The test needs to accomplish its objective (which is to validate a particular user scenario)
+
 # Proactiveness
+
 You are allowed to be proactive, but only when the user asks you to do something. You should strive to
 strike a balance between:
 1. Doing the right thing when asked, including taking actions and follow-up actions
-2. Not surprising the user with actions you take without asking
+2. Not surprising the user with actions you take without asking. It is okay to ask the user for confirmation before taking actions.
 `;
 const tools = [
     test_run_1.runTestTool,
-    browser_agent_1.browserAgentTool,
+    test_gen_browser_1.generateTestWithBrowserAgent,
     diagnosis_fetcher_1.diagnosisTool,
     grep_1.grepTool,
     test_run_fetcher_1.testRunTool,
 ];
 const toolExecutors = {
     ...Object.fromEntries(tools.map((tool) => [tool.schema.name, tool.execute])),
-    str_replace_editor: claude_1.strReplaceEditorTool,
+    str_replace_editor: (input) => (0, chat_1.strReplaceEditorTool)(input, web_1.validateTypescript),
 };
 async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForChatState, }) {
     const ora = (await import("ora")).default;
     let userPrompt = undefined;
-    let chatState = useDiskForChatState ? claude_1.ChatState.load() : new claude_1.ChatState(false);
+    let chatState = useDiskForChatState ? chat_1.ChatState.load() : new chat_1.ChatState(false);
     const handleSigInt = () => {
         console.log(`\n${(0, picocolors_1.gray)("Usage summary -> " + chatState.getUsageSummary())}`);
         process.exit(0);
@@ -129,7 +144,12 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
                 throw new Error(`Tool ${toolUse.name} not found`);
             }
             const toolResult = await toolExecutor(toolUse.input);
-            spinner.succeed(`Tool ${toolUse.name} completed`);
+            if (toolResult.isError) {
+                spinner.fail(`Tool ${toolUse.name} failed with error: ${toolResult.result}`);
+            }
+            else {
+                spinner.succeed(`Tool ${toolUse.name} completed`);
+            }
             chatState.pushMessage({
                 role: "user",
                 content: [
@@ -144,10 +164,10 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
             continue;
         }
         const spinner = ora("Claude is working...").start();
-        const response = await (0, claude_1.createClaudeMessage)({
+        const response = await (0, chat_1.createClaudeMessage)({
             systemPrompt,
             messages: chatState.getMessagesForCreateCompletion(),
-            tools: tools.map((tool) => (0, claude_1.convertOpenAISchemaToAnthropic)((0, zod_schema_1.zodToOpenAITool)(tool.schema))),
+            tools: tools.map((tool) => (0, chat_1.convertOpenAISchemaToAnthropic)((0, zod_schema_1.zodToOpenAITool)(tool.schema))),
             model: chatModel,
             withStrReplaceEditor: true,
         });

package/dist/agent/cua/index.d.ts

@@ -3,7 +3,7 @@ export declare function startPlaywrightCodegen(page: Page): Promise<void>;
 /**
  * Run the loop that executes computer actions until no 'computer_call' is found.
  */
-export declare function executeUsingComputerUseAgent({ page, task, }: {
+export declare function createTestUsingComputerUseAgent({ page, task, }: {
     page: Page;
     task: string;
 }): Promise<{

package/dist/agent/cua/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,sBAAsB,CAAC,IAAI,EAAE,IAAI,iBAoBtD;AAED;;GAEG;AACH,wBAAsB,4BAA4B,CAAC,EACjD,IAAI,EACJ,IAAI,GACL,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC,CAkGD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,sBAAsB,CAAC,IAAI,EAAE,IAAI,iBAoBtD;AAED;;GAEG;AACH,wBAAsB,+BAA+B,CAAC,EACpD,IAAI,EACJ,IAAI,GACL,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC,CAmGD"}

package/dist/agent/cua/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.executeUsingComputerUseAgent = exports.startPlaywrightCodegen = void 0;
+exports.createTestUsingComputerUseAgent = exports.startPlaywrightCodegen = void 0;
 const openai_1 = __importDefault(require("openai"));
 const utils_1 = require("../browsing/utils");
 const computer_1 = require("./computer");
@@ -37,7 +37,7 @@ exports.startPlaywrightCodegen = startPlaywrightCodegen;
 /**
  * Run the loop that executes computer actions until no 'computer_call' is found.
  */
-async function executeUsingComputerUseAgent({ page, task, }) {
+async function createTestUsingComputerUseAgent({ page, task, }) {
     let generatedCode = "";
     await (0, utils_1.injectPwLocatorGenerator)(page);
     const screenshotBytes = await (0, computer_1.getScreenshot)(page);
@@ -110,6 +110,7 @@ async function executeUsingComputerUseAgent({ page, task, }) {
                     environment: "browser",
                 },
             ],
+            instructions: INSTRUCTIONS,
             input: [
                 {
                     call_id: lastCallId,
@@ -129,4 +130,4 @@ async function executeUsingComputerUseAgent({ page, task, }) {
         importPaths: [],
     };
 }
-exports.executeUsingComputerUseAgent = executeUsingComputerUseAgent;
+exports.createTestUsingComputerUseAgent = createTestUsingComputerUseAgent;

package/dist/agent/master/browser-tests/index.spec.js

@@ -75,7 +75,7 @@ click on maverick inside ford dropdown`,
 (0, fixtures_1.test)("cua agent can click icons accurately", async ({ page, server }) => {
     await page.goto(`${server.baseURL}/icons-navbar.html`);
     await (0, fixtures_1.expect)(page.getByText("select an icon")).toBeVisible();
-    const response = await (0, run_1.executeUsingComputerUseAgent)({
+    const response = await (0, run_1.createTestUsingComputerUseAgent)({
         task: `click on the gear icon`,
         page,
     });

package/dist/agent/master/run.d.ts

@@ -1,7 +1,7 @@
 import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
 import { ScopeVars } from "../../types";
-export { executeUsingComputerUseAgent } from "../cua";
+export { createTestUsingComputerUseAgent } from "../cua";
 export declare const IS_ALLOWED_TO_USE_SKILLS = false;
 export declare function createTestUsingMasterAgent({ task, page, testCase, specPath, options, scopeVars, }: {
     task: string;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAAE,4BAA4B,EAAE,MAAM,QAAQ,CAAC;AAKtD,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAqB9C,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwRA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAAE,+BAA+B,EAAE,MAAM,QAAQ,CAAC;AAKzD,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAqB9C,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwRA"}

package/dist/agent/master/run.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createTestUsingMasterAgent = exports.IS_ALLOWED_TO_USE_SKILLS = exports.executeUsingComputerUseAgent = void 0;
+exports.createTestUsingMasterAgent = exports.IS_ALLOWED_TO_USE_SKILLS = exports.createTestUsingComputerUseAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const actions_1 = require("../../actions");
 const skill_1 = require("../../actions/skill");
@@ -20,7 +20,7 @@ const execute_browser_action_1 = require("./execute-browser-action");
 const execute_skill_action_1 = require("./execute-skill-action");
 const next_action_1 = require("./next-action");
 var cua_1 = require("../cua");
-Object.defineProperty(exports, "executeUsingComputerUseAgent", { enumerable: true, get: function () { return cua_1.executeUsingComputerUseAgent; } });
+Object.defineProperty(exports, "createTestUsingComputerUseAgent", { enumerable: true, get: function () { return cua_1.createTestUsingComputerUseAgent; } });
 const MAX_ERROR_COUNT = 2;
 // Disabling skills as we're seeing false usage with chat agent
 exports.IS_ALLOWED_TO_USE_SKILLS = false;

package/dist/bin/utils/scenarios/index.d.ts

@@ -1,6 +1,8 @@
 import type { TestGenConfig } from "@empiricalrun/shared-types";
 import { CliOptions } from "../index";
-export declare function buildTokenFromOptions(options: Omit<CliOptions, "token">): string;
+export declare function buildTokenFromOptions(options: Omit<CliOptions, "token">, featureFlags?: {
+    useComputerUseAgent?: boolean;
+}): string;
 export declare function buildTestConfigFromOptions(options: Omit<CliOptions, "token">): TestGenConfig;
 export declare function loadTestConfigs(testGenToken: string): TestGenConfig;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/scenarios/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/scenarios/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAIV,aAAa,EAEd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAetC,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,GACjC,MAAM,CAgBR;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,GACjC,aAAa,CAaf;AAED,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,aAAa,CAmBnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/scenarios/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAIV,aAAa,EAEd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAetC,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,EAClC,YAAY,CAAC,EAAE;IACb,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B,GACA,MAAM,CAqBR;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,GACjC,aAAa,CAaf;AAED,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,aAAa,CAmBnE"}

package/dist/bin/utils/scenarios/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadTestConfigs = exports.buildTestConfigFromOptions = exports.buildTokenFromOptions = void 0;
-function buildTokenFromOptions(options) {
+function buildTokenFromOptions(options, featureFlags) {
     const genConfig = buildTestConfigFromOptions(options);
     const requestConfig = {
         specPath: genConfig.specPath,
@@ -12,7 +12,12 @@ function buildTokenFromOptions(options) {
         suites: genConfig.testCase.suites,
         ai_gist: genConfig.testCase.ai_gist,
         build: genConfig.build,
-        options: genConfig.options,
+        options: genConfig.options
+            ? {
+                ...genConfig.options,
+                ...featureFlags,
+            }
+            : featureFlags,
         environment: genConfig.environment,
         testErrorDiagnosis: genConfig.testErrorDiagnosis,
     };

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAepC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAyC3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAepC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAsD3E"}

package/dist/index.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTest = void 0;
 const llm_1 = require("@empiricalrun/llm");
+const cua_1 = require("./agent/cua");
 const run_1 = require("./agent/master/run");
 const scenarios_1 = require("./bin/utils/scenarios");
 const client_1 = __importDefault(require("./file/client"));
@@ -26,29 +27,41 @@ async function createTest(task, page, scope) {
     try {
         const testConfigArg = process.env.TEST_GEN_TOKEN;
         const testGenConfig = (0, scenarios_1.loadTestConfigs)(testConfigArg);
-        (0, reporter_1.setReporterConfig)({
-            projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-            testSessionId: testGenConfig.options?.metadata.testSessionId,
-            generationId: testGenConfig.options?.metadata.generationId,
-        });
-        (0, session_1.setSessionDetails)({
-            sessionId: testGenConfig.options?.metadata.testSessionId,
-            generationId: testGenConfig.options?.metadata.generationId,
-            testCaseId: testGenConfig.testCase.id,
-            projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-        });
+        if (testGenConfig.options && testGenConfig.options.metadata) {
+            (0, reporter_1.setReporterConfig)({
+                projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+                testSessionId: testGenConfig.options?.metadata.testSessionId,
+                generationId: testGenConfig.options?.metadata.generationId,
+            });
+            (0, session_1.setSessionDetails)({
+                sessionId: testGenConfig.options?.metadata.testSessionId,
+                generationId: testGenConfig.options?.metadata.generationId,
+                testCaseId: testGenConfig.testCase.id,
+                projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+            });
+        }
         const fileService = new client_1.default();
-        const { testCase, specPath } = testGenConfig;
-        const { code, importPaths } = await (0, run_1.createTestUsingMasterAgent)({
-            testCase,
-            specPath,
-            page,
-            task,
-            options: {
-                ...testGenConfig.options,
-            },
-            scopeVars: scope,
-        });
+        const useComputerUseAgent = testGenConfig.options?.useComputerUseAgent;
+        let agentResult;
+        if (useComputerUseAgent) {
+            agentResult = await (0, cua_1.createTestUsingComputerUseAgent)({
+                task,
+                page,
+            });
+        }
+        else {
+            agentResult = await (0, run_1.createTestUsingMasterAgent)({
+                task,
+                page,
+                testCase: testGenConfig.testCase,
+                specPath: testGenConfig.specPath,
+                options: {
+                    ...testGenConfig.options,
+                },
+                scopeVars: scope,
+            });
+        }
+        const { code, importPaths } = agentResult;
         await fileService.updateTest({
             task,
             generatedCode: code,

package/dist/tools/test-gen-browser.d.ts

@@ -0,0 +1,3 @@
+import type { Tool } from "./types";
+export declare const generateTestWithBrowserAgent: Tool;
+//# sourceMappingURL=test-gen-browser.d.ts.map

package/dist/tools/test-gen-browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAmDpC,eAAO,MAAM,4BAA4B,EAAE,IAiE1C,CAAC"}

package/dist/tools/{browser-agent.js → test-gen-browser.js}

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.browserAgentTool = void 0;
+exports.generateTestWithBrowserAgent = void 0;
 const zod_1 = require("zod");
 const run_1 = require("../agent/browsing/run");
 const utils_1 = require("../agent/browsing/utils");
@@ -49,7 +49,7 @@ test("Example test code", async ({ page }) => {
 });
 \`\`\`
 `;
-exports.browserAgentTool = {
+exports.generateTestWithBrowserAgent = {
     schema: {
         name: "generateTestWithBrowserAgent",
         description: BROWSER_AGENT_DESCRIPTION,
@@ -57,6 +57,14 @@ exports.browserAgentTool = {
     },
     execute: async (input) => {
         const { testName, testSuites, fileName, changeToMake, project } = input;
+        const playwrightConfig = await (0, utils_1.readPlaywrightConfig)(process.cwd());
+        const validProjectNames = await (0, utils_1.getValidProjectNames)(playwrightConfig);
+        if (!validProjectNames.includes(project)) {
+            return {
+                isError: true,
+                result: `Invalid project name: ${project}. Valid project names are: ${validProjectNames.join(", ")}`,
+            };
+        }
         try {
             await (0, utils_1.replaceTodoWithCreateTest)({
                 testCaseName: testName,
@@ -70,14 +78,6 @@ exports.browserAgentTool = {
                 result: `Error running tool: ${error}`,
             };
         }
-        const playwrightConfig = await (0, utils_1.readPlaywrightConfig)(process.cwd());
-        const validProjectNames = await (0, utils_1.getValidProjectNames)(playwrightConfig);
-        if (!validProjectNames.includes(project)) {
-            return {
-                isError: true,
-                result: `Invalid project name: ${project}. Valid project names are: ${validProjectNames.join(", ")}`,
-            };
-        }
         const { isError, error } = await (0, run_1.generateTestsUsingMasterAgent)({
             testFilePath: fileName,
             filePathToUpdate: fileName,
@@ -86,6 +86,8 @@ exports.browserAgentTool = {
                 name: testName,
                 file: fileName,
                 prompt: changeToMake,
+            }, {
+                useComputerUseAgent: true,
             }),
             repoDir: process.cwd(),
         });

package/dist/tools/test-run-fetcher/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAWrC,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOnE;AAED,eAAO,MAAM,WAAW,EAAE,IAwHzB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAWrC,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOnE;AAED,eAAO,MAAM,WAAW,EAAE,IA4HzB,CAAC"}

package/dist/tools/test-run-fetcher/index.js

@@ -52,55 +52,57 @@ exports.testRunTool = {
         //   1. Truncate stack trace to last 300 characters
         //   2. Remove request/response headers from network metadata
         //   3. Focus on failed tests only
-        if (data.data?.test_run?.flattenedSummaryDetails) {
-            data.data.test_run.flattenedSummaryDetails =
-                data.data.test_run.flattenedSummaryDetails.map((detail) => {
-                    // Truncate stack trace to last 300 characters
-                    if (detail.failed_run_metadata?.stack) {
-                        const stack = detail.failed_run_metadata.stack;
-                        // TODO: change to last 10 lines
-                        detail.failed_run_metadata.stack =
-                            stack.length > 300 ? `...${stack.slice(-300)}` : stack;
-                    }
-                    // Remove headers from network metadata
-                    if (detail.network_metadata?.failed_calls) {
-                        detail.network_metadata.failed_calls =
-                            detail.network_metadata.failed_calls.map((call) => ({
-                                ...call,
-                                request: { ...call.request, headers: [] },
-                                response: { ...call.response, headers: [] },
-                            }));
-                    }
-                    if (detail.network_metadata?.failed_calls_within_time_range) {
-                        detail.network_metadata.failed_calls_within_time_range =
-                            detail.network_metadata.failed_calls_within_time_range.map((call) => ({
-                                ...call,
-                                request: { ...call.request, headers: [] },
-                                response: { ...call.response, headers: [] },
-                            }));
-                    }
-                    return detail;
-                });
+        const testRun = data.data?.test_run;
+        if (testRun?.flattenedSummaryDetails) {
+            testRun.flattenedSummaryDetails = testRun.flattenedSummaryDetails.map((detail) => {
+                // Truncate stack trace to last 300 characters
+                if (detail.failed_run_metadata?.stack) {
+                    const stack = detail.failed_run_metadata.stack;
+                    // TODO: change to last 10 lines
+                    detail.failed_run_metadata.stack =
+                        stack.length > 300 ? `...${stack.slice(-300)}` : stack;
+                }
+                // Remove headers from network metadata
+                if (detail.network_metadata?.failed_calls) {
+                    detail.network_metadata.failed_calls =
+                        detail.network_metadata.failed_calls.map((call) => ({
+                            ...call,
+                            request: { ...call.request, headers: [] },
+                            response: { ...call.response, headers: [] },
+                        }));
+                }
+                if (detail.network_metadata?.failed_calls_within_time_range) {
+                    detail.network_metadata.failed_calls_within_time_range =
+                        detail.network_metadata.failed_calls_within_time_range.map((call) => ({
+                            ...call,
+                            request: { ...call.request, headers: [] },
+                            response: { ...call.response, headers: [] },
+                        }));
+                }
+                return detail;
+            });
         }
-        const failedTests = data.data?.test_run?.flattenedSummaryDetails.filter((detail) => detail.status === "failed");
+        const failedTests = testRun?.flattenedSummaryDetails.filter((detail) => detail.status === "failed");
         const failedTestInMarkdown = failedTests
             ?.map((detail) => {
             const { nesting } = detail.failed_run_metadata;
             return {
                 testName: nesting.slice(1).join(" > "),
                 fileName: extractPathAfterSourceRepo(detail.failed_run_metadata.location.file),
+                project: detail.test_project,
                 detail,
             };
         })
-            .map((test) => {
+            .map((res) => {
             return `
-### Test name: ${test.testName}
-File: ${test.fileName}
+### Test name: ${res.testName}
+File: ${res.fileName}
+Project: ${res.project}
 
 Failure data below
 
 \`\`\`json
-${JSON.stringify(test.detail, null, 2)}
+${JSON.stringify(res.detail, null, 2)}
 \`\`\`
 `;
         })
@@ -111,10 +113,10 @@ ${JSON.stringify(test.detail, null, 2)}
 ## Run info
 - Run ID: ${runId}
 - Repository: ${repoName}
-- Started at: ${data.data?.test_run?.testRun.run_started_at}
-- Ended at: ${data.data?.test_run?.testRun.run_ended_at}
-- Duration: ${data.data?.test_run?.testRun.duration} seconds
-- Environment name: ${data.data?.test_run?.testRun.environment_name}
+- Started at: ${testRun?.testRun.run_started_at}
+- Ended at: ${testRun?.testRun.run_ended_at}
+- Duration: ${testRun?.testRun.duration} seconds
+- Environment name: ${testRun?.testRun.environment_name}
 
 ## Failed tests
 ${failedTestInMarkdown}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.51.3",
+  "version": "0.51.5",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -74,7 +74,7 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "zod": "^3.23.8",
-    "@empiricalrun/llm": "^0.11.2",
+    "@empiricalrun/llm": "^0.11.4",
     "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/reporter": "^0.23.2",
     "@empiricalrun/test-run": "^0.7.6"
@@ -92,7 +92,7 @@
     "js-levenshtein": "^1.1.6",
     "playwright": "1.47.1",
     "ts-patch": "^3.3.0",
-    "@empiricalrun/shared-types": "0.0.6"
+    "@empiricalrun/shared-types": "0.0.7"
   },
   "scripts": {
     "dev": "tspc --build --watch",

package/dist/tools/browser-agent.d.ts

@@ -1,3 +0,0 @@
-import type { Tool } from "./types";
-export declare const browserAgentTool: Tool;
-//# sourceMappingURL=browser-agent.d.ts.map

package/dist/tools/browser-agent.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"browser-agent.d.ts","sourceRoot":"","sources":["../../src/tools/browser-agent.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAmDpC,eAAO,MAAM,gBAAgB,EAAE,IA4D9B,CAAC"}