@empiricalrun/test-gen 0.47.2 → 0.47.4

package/dist/agent/master/planner.js

@@ -4,7 +4,6 @@ exports.runtimePlannerWithScreenshot = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
 const constants_1 = require("../../constants");
-const utils_1 = require("../utils");
 async function runtimePlannerWithScreenshot({ trace, task, conversation, pages, page, currentPage, }) {
     const buffer = await page.screenshot({
     //This is done to improve element annotation accuracy, anyways it doesn't annotate elements which are out of viewport
@@ -120,7 +119,7 @@ async function runtimePlannerWithScreenshot({ trace, task, conversation, pages,
     });
     const toolCallResp = (response?.tool_calls || [])[0];
     if (toolCallResp) {
-        const toolCall = (0, utils_1.parseJson)(toolCallResp.function.arguments);
+        const toolCall = JSON.parse(toolCallResp.function.arguments);
         const output = {
             pageName: toolCall.pageName,
             isDone: toolCall.isDone,

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAWxC,OAAO,EAAE,4BAA4B,EAAE,MAAM,QAAQ,CAAC;AAuBtD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwRA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAAE,4BAA4B,EAAE,MAAM,QAAQ,CAAC;AAuBtD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwRA"}

package/dist/agent/master/run.js

@@ -15,7 +15,6 @@ const utils_2 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
 const run_1 = require("../planner/run");
 const run_time_planner_1 = require("../planner/run-time-planner");
-const utils_3 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
 const execute_browser_action_1 = require("./execute-browser-action");
 const execute_skill_action_1 = require("./execute-skill-action");
@@ -164,7 +163,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                 await testgenUpdatesReporter.sendMessage("Agent is not able to figure out next action since element is not visible on screen.");
                 break;
             }
-            const args = (0, utils_3.parseJson)(nextAction.toolCallArgs);
+            const args = JSON.parse(nextAction.toolCallArgs);
             const masterAgentActionSpan = masterAgentSpan?.span({
                 name: "master-agent-execute-action",
             });

package/dist/agent/master/scroller.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAG7C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAchD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AA2ZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}
+{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAG7C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAahD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AA2ZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}

package/dist/agent/master/scroller.js

@@ -5,7 +5,6 @@ const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
 const constants_1 = require("../../constants");
 const reporter_1 = require("../../reporter");
-const utils_1 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
 const element_annotation_1 = require("./element-annotation");
 let usedAnnotations = [];
@@ -165,7 +164,7 @@ Follow the instructions before responding:
     const toolCall = completion?.tool_calls?.[0];
     scrollSpan?.end({ output: toolCall });
     if (toolCall) {
-        const args = (0, utils_1.parseJson)(toolCall.function.arguments);
+        const args = JSON.parse(toolCall.function.arguments);
         isVisible = args.is_visible || false;
     }
     else {
@@ -303,7 +302,7 @@ ${annotationKeysString}`,
     const toolCall = completion?.tool_calls?.[0];
     annotationsSpan?.end({ output: toolCall });
     if (toolCall) {
-        const args = (0, utils_1.parseJson)(toolCall.function.arguments);
+        const args = JSON.parse(toolCall.function.arguments);
         const isAnnotationPresentInKeys = annotationKeys.some((annotation) => annotation.elementID === args.element_annotation);
         if (args.element_annotation !== "NA" && isAnnotationPresentInKeys) {
             usedAnnotations.push(args.element_annotation);

package/dist/agent/planner/run-time-planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;CAC1B;;;;GA+FA"}
+{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;CAC1B;;;;GA+FA"}

package/dist/agent/planner/run-time-planner.js

@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.runtimePlanner = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are given a list of successfully executed actions that are done towards completing a task (which\nis also provided to you). Your goal is to analyse the list and determine if the task is completed.\n\nIf the task is not fully completed, identify which specific actions are missing\nand suggest next steps to complete the task. Assume that the conversation provided\nis entirely truthful and no additional actions were performed beyond those listed.\n\nThese actions were executed by AI agents using Playwright on a browser. These agents\nalready have access to browser tabs to execute actions. If there is a pending action,\none of the agents will execute it in the browser. However, they need your help to\nchoose which browser tab (= page) to use for the next action.\n\nTo fulfil your goal, follow these steps:\n- Divide the task into individual actions.\n- Compare each task action against the actions listed in the successfully executed actions list.\n- Identify which actions have been executed and which have not.\n- If all actions are executed, respond with the task as done.\n- If any actions are missing, respond with the task as not done, listing all actions\n  and specifying which are complete and which are missing.\n- If provided with list of pages, based on the next pending action and previously executed\n  action, identify the page on which next action needs to be taken\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\n----\n\nSuccessfully executed actions:\n{{successfulActions}}\n\n----\n\nList of pages with their current URLs:\n{{pagesSummary}}\n\n\n{{/section}}\n";
-const utils_1 = require("../utils");
 async function runtimePlanner({ trace, task, successfulActions, pages, currentPage, }) {
     const runTimePlannerSpan = trace?.span({
         name: "runtime-planner",
@@ -76,7 +75,7 @@ async function runtimePlanner({ trace, task, successfulActions, pages, currentPa
     });
     const toolCallResp = (response?.tool_calls || [])[0];
     if (toolCallResp) {
-        const toolCall = (0, utils_1.parseJson)(toolCallResp.function.arguments);
+        const toolCall = JSON.parse(toolCallResp.function.arguments);
         const output = {
             pageName: toolCall.pageName,
             isDone: toolCall.isDone,

package/dist/bin/index.js

@@ -9,6 +9,7 @@ const commander_1 = require("commander");
 const dotenv_1 = __importDefault(require("dotenv"));
 const run_1 = require("../agent/browsing/run");
 const utils_1 = require("../agent/browsing/utils");
+const chat_1 = require("../agent/chat");
 const repo_edit_1 = require("../agent/codegen/repo-edit");
 const run_2 = require("../agent/codegen/run");
 const diagnosis_agent_1 = require("../agent/diagnosis-agent");
@@ -31,14 +32,12 @@ process.on("beforeExit", async () => await flushEvents());
 process.on("exit", async () => await flushEvents());
 process.on("SIGINT", async () => await flushEvents());
 process.on("SIGTERM", async () => await flushEvents());
-async function resolveAgentUsingTask({ testCase, trace, }) {
-    const { response } = await (0, infer_agent_1.inferAgentBasedTask)({
-        task: testCase.steps.join("\n"),
-        trace,
+async function runChatAgent(prompt) {
+    return await (0, chat_1.chatAgent)({
+        prompt,
     });
-    return response;
 }
-async function runAgent(testGenConfig, testGenToken) {
+async function runAgentsWorkflow(testGenConfig, testGenToken) {
     const logger = new logger_1.CustomLogger();
     const { specPath, testCase } = testGenConfig;
     if (process.env.LOG_URL) {
@@ -89,29 +88,10 @@ async function runAgent(testGenConfig, testGenToken) {
         });
         return;
     }
-    // TODO: this needs to be moved to an orchestrator which decides what needs to be done first before executing the sub tasks
-    if (testGenConfig.testErrorDiagnosis &&
-        testGenConfig.testErrorDiagnosis.failingLine &&
-        // TODO: fix this hardcoding of user prompt - ideally its an auto fix intent
-        testCase.steps[0]?.toLowerCase().trim() == "can you please fix the test") {
-        const { task: updatedTask } = await (0, diagnosis_agent_1.createTaskUsingFailureDiagnosis)({
-            options: testGenConfig.options,
-            trace,
-            diagnosis: testGenConfig.testErrorDiagnosis,
-        });
-        if (updatedTask) {
-            testCase.steps = [updatedTask];
-        }
-    }
-    if (!agent || agent === "auto") {
-        agent = await resolveAgentUsingTask({
-            testCase,
-            trace,
-        });
-    }
-    logger.success(`Generating test using ${agent} agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
     if (testGenConfig.testErrorDiagnosis &&
         testGenConfig.testErrorDiagnosis.failingLine) {
+        // If failure context is available, we can enrich the user prompt to contain
+        // the failure context
         const requestedChangeResp = await (0, enrich_prompt_1.enrichPromptWithFailingLine)({
             trace,
             testBlock: testGenConfig.testErrorDiagnosis.failingLine,
@@ -119,7 +99,28 @@ async function runAgent(testGenConfig, testGenToken) {
             suggestionForFix: testCase.steps.join("\n"),
         });
         testCase.steps = [requestedChangeResp.output];
+        // For "auto-fix" we use the user prompt (which is hard-coded in the dashboard
+        // entrypoints), and invoke more interesting enrichment
+        if (
+        // TODO: fix this hardcoding of user prompt - ideally its an auto fix intent
+        testCase.steps[0]?.toLowerCase().trim() == "can you please fix the test") {
+            const { task: updatedTask } = await (0, diagnosis_agent_1.createTaskUsingFailureDiagnosis)({
+                trace,
+                diagnosis: testGenConfig.testErrorDiagnosis,
+            });
+            if (updatedTask) {
+                testCase.steps = [updatedTask];
+            }
+        }
+    }
+    if (!agent || agent === "auto") {
+        const { response } = await (0, infer_agent_1.inferAgentBasedTask)({
+            task: testCase.steps.join("\n"),
+            trace,
+        });
+        agent = response;
     }
+    logger.success(`Generating test using ${agent} agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
     if (agent === "plan") {
         const task = testCase.steps.join("\n");
         const plan = await (0, run_3.planTask)({
@@ -132,11 +133,18 @@ async function runAgent(testGenConfig, testGenToken) {
         await new reporter_1.TestGenUpdatesReporter().sendMessage(plan);
     }
     else if (agent === "code") {
-        await (0, run_2.generateTest)(testCase, specPath, testGenConfig.options, trace);
+        await (0, run_2.generateTestWithCodegen)({
+            testCase,
+            file: specPath,
+            trace,
+        });
     }
     else {
-        // this assumes we have only one scenario in test config
-        const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)(testGenConfig, trace);
+        const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)({
+            testCase,
+            specPath,
+            trace,
+        });
         void (0, session_1.updateSessionStatus)(testGenConfig.options?.metadata.testSessionId, {
             status: "agent_live_session_started",
         });
@@ -155,10 +163,11 @@ async function runAgent(testGenConfig, testGenToken) {
     const program = new commander_1.Command();
     program
         .option("--token <token>", "Test generation token")
+        .option("--prompt <prompt>", "Prompt for the chat agent")
         .option("--name <test-name>", "Name of the test case")
-        .option("--prompt <prompt>", "Prompt for the test case")
         .option("--file <test-file>", "File path of the test case (inside tests dir)")
         .option("--suites <suites>", "Comma separated list of describe blocks")
+        .option("--use-chat", "Use chat agent (and not the workflow)")
         .parse(process.argv);
     const options = program.opts();
     const completedOptions = await (0, utils_2.validateAndCompleteCliOptions)(options);
@@ -181,10 +190,16 @@ async function runAgent(testGenConfig, testGenToken) {
     });
     let testGenFailed = false;
     let agentUsed;
+    // Download the build if repo has a download script
+    await (0, test_build_1.downloadBuild)(testGenConfig.build || {});
     try {
-        // download the build if it exists
-        await (0, test_build_1.downloadBuild)(testGenConfig.build || {});
-        agentUsed = await runAgent(testGenConfig, testGenToken);
+        if (completedOptions.useChat) {
+            await runChatAgent(completedOptions.prompt);
+            return;
+        }
+        else {
+            agentUsed = await runAgentsWorkflow(testGenConfig, testGenToken);
+        }
     }
     catch (e) {
         testGenFailed = true;

package/dist/bin/utils/index.d.ts

@@ -4,6 +4,7 @@ export interface CliOptions {
     file?: string;
     prompt?: string;
     suites?: string;
+    useChat?: boolean;
 }
 export declare function validateAndCompleteCliOptions(options: CliOptions): Promise<CliOptions>;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAQD,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,UAAU,CAAC,CAmDrB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAQD,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,UAAU,CAAC,CAyDrB"}

package/dist/bin/utils/index.js

@@ -6,12 +6,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.validateAndCompleteCliOptions = void 0;
 const inquirer_1 = __importDefault(require("inquirer"));
 async function validateAndCompleteCliOptions(options) {
+    // For existing flow between dashboard <> test-gen (via ci-worker)
     const hasToken = !!options.token;
     if (hasToken) {
         return options;
     }
+    let requiredFields = ["name", "file", "prompt"];
+    // For new chat flow in local CLI usage, only prompt is required
+    if (options.useChat) {
+        requiredFields = ["prompt"];
+    }
     const questions = [];
-    if (!options.name) {
+    if (!options.name && requiredFields.includes("name")) {
         questions.push({
             type: "input",
             name: "name",
@@ -19,7 +25,7 @@ async function validateAndCompleteCliOptions(options) {
             validate: (input) => input.trim().length > 0 || "Test name is required",
         });
     }
-    if (!options.file) {
+    if (!options.file && requiredFields.includes("file")) {
         questions.push({
             type: "input",
             name: "file",
@@ -27,7 +33,7 @@ async function validateAndCompleteCliOptions(options) {
             validate: (input) => input.trim().length > 0 || "Test file path is required",
         });
     }
-    if (!options.prompt) {
+    if (!options.prompt && requiredFields.includes("prompt")) {
         questions.push({
             type: "editor",
             name: "prompt",

package/dist/file/server.d.ts

@@ -2,12 +2,14 @@ export declare class FileService {
     private port;
     private filePath;
     private repoDir;
+    private server;
     constructor({ port, repoDir }: {
         port: number;
         repoDir: string;
     });
     setFilePath(filePath: string): void;
     startFileService(): Promise<number>;
+    stop(): Promise<void>;
 }
 export declare function startFileService(): Promise<void>;
 //# sourceMappingURL=server.d.ts.map

package/dist/file/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAWA,qBAAa,WAAW;IACtB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;gBAEjB,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAKhE,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAyC1C;AAED,wBAAsB,gBAAgB,kBAAK"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAWA,qBAAa,WAAW;IACtB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAA4C;gBAE9C,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAKhE,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IA0CnC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB5B;AAED,wBAAsB,gBAAgB,kBAAK"}

package/dist/file/server.js

@@ -13,6 +13,7 @@ class FileService {
     port = 0;
     filePath = "";
     repoDir = "";
+    server;
     constructor({ port, repoDir }) {
         this.port = port;
         this.repoDir = repoDir;
@@ -43,7 +44,23 @@ class FileService {
             return res.send({ success: false });
         });
         return new Promise((resolve) => {
-            app.listen(this.port, () => resolve(this.port));
+            this.server = app.listen(this.port, () => resolve(this.port));
+        });
+    }
+    async stop() {
+        return new Promise((resolve, reject) => {
+            if (!this.server) {
+                resolve();
+                return;
+            }
+            this.server.close((err) => {
+                if (err) {
+                    reject(err);
+                    return;
+                }
+                this.server = undefined;
+                resolve();
+            });
         });
     }
 }

package/dist/tools/browser-agent.d.ts

@@ -0,0 +1,16 @@
+import { OpenAI } from "openai";
+export declare const schema: OpenAI.Chat.Completions.ChatCompletionTool;
+export declare const generateTestWithBrowserAgentTool: ({ testName, fileName, changeToMake, }: {
+    testName: string;
+    fileName: string;
+    changeToMake: string;
+}) => Promise<{
+    result: string;
+    gitPatch: string;
+    error?: undefined;
+} | {
+    result: string;
+    error: string;
+    gitPatch?: undefined;
+}>;
+//# sourceMappingURL=browser-agent.d.ts.map

package/dist/tools/browser-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-agent.d.ts","sourceRoot":"","sources":["../../src/tools/browser-agent.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAQhC,eAAO,MAAM,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBA0B5C,CAAC;AAEF,eAAO,MAAM,gCAAgC;cAKjC,MAAM;cACN,MAAM;kBACF,MAAM;;;;;;;;;EAsCrB,CAAC"}

package/dist/tools/browser-agent.js

@@ -0,0 +1,76 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateTestWithBrowserAgentTool = exports.schema = void 0;
+const path_1 = __importDefault(require("path"));
+const run_1 = require("../agent/browsing/run");
+const utils_1 = require("../agent/browsing/utils");
+const scenarios_1 = require("../bin/utils/scenarios");
+const git_1 = require("../utils/git");
+exports.schema = {
+    type: "function",
+    function: {
+        name: "generateTestWithBrowserAgent",
+        description: "Create or modify a test case with browser agent. This is useful when the modifications involve changing a selector or executing browser interactions (like click, fill, etc)",
+        parameters: {
+            type: "object",
+            properties: {
+                testName: {
+                    type: "string",
+                    description: "The name of the test to create or modify",
+                },
+                fileName: {
+                    type: "string",
+                    description: "The name of the file where the test is located. File name must end with .spec.ts",
+                },
+                changeToMake: {
+                    type: "string",
+                    description: "The change to make to the test",
+                },
+            },
+            required: ["testName", "fileName", "changeToMake"],
+        },
+    },
+};
+const generateTestWithBrowserAgentTool = async ({ testName, fileName, changeToMake, }) => {
+    const testCase = {
+        id: 0,
+        name: testName,
+        filePath: fileName,
+        suites: [], // TODO: Support suites
+        steps: [changeToMake],
+    };
+    const filePathFromCwd = path_1.default.join("tests", fileName);
+    const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)({
+        testCase,
+        specPath: filePathFromCwd,
+    });
+    const { isError, error } = await (0, run_1.generateTestsUsingMasterAgent)({
+        testFilePath: filePathFromCwd,
+        filePathToUpdate,
+        // TODO: Remove this hardcoded project name
+        pwProjectsFilter: ["chromium"],
+        testGenToken: (0, scenarios_1.buildTokenFromOptions)({
+            name: testName,
+            file: fileName,
+            prompt: changeToMake,
+        }),
+        repoDir: process.cwd(),
+    });
+    if (!isError) {
+        const gitPatch = (0, git_1.getGitDiff)(filePathFromCwd);
+        return {
+            result: "Test was generated successfully",
+            gitPatch,
+        };
+    }
+    else {
+        return {
+            result: "Test was not generated successfully",
+            error,
+        };
+    }
+};
+exports.generateTestWithBrowserAgentTool = generateTestWithBrowserAgentTool;

package/dist/tools/codegen-agent.d.ts

@@ -0,0 +1,9 @@
+import { TestCase } from "@empiricalrun/shared-types";
+import { OpenAI } from "openai";
+export declare const schema: OpenAI.Chat.Completions.ChatCompletionTool;
+export declare const generateTestWithCodegenTool: ({ testName, fileName, changeToMake, }: {
+    testName: string;
+    fileName: string;
+    changeToMake: string;
+}) => Promise<void | TestCase[]>;
+//# sourceMappingURL=codegen-agent.d.ts.map

package/dist/tools/codegen-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codegen-agent.d.ts","sourceRoot":"","sources":["../../src/tools/codegen-agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAIhC,eAAO,MAAM,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBA0B5C,CAAC;AAEF,eAAO,MAAM,2BAA2B;cAK5B,MAAM;cACN,MAAM;kBACF,MAAM;gCAcrB,CAAC"}

package/dist/tools/codegen-agent.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateTestWithCodegenTool = exports.schema = void 0;
+const run_1 = require("../agent/codegen/run");
+exports.schema = {
+    type: "function",
+    function: {
+        name: "generateTestWithCodegen",
+        description: "Create or modify a test case with code generation. This is useful when modifications can be done with TypeScript only, and don't require any browser interactions or element selectors.",
+        parameters: {
+            type: "object",
+            properties: {
+                testName: {
+                    type: "string",
+                    description: "The name of the test to create or modify",
+                },
+                fileName: {
+                    type: "string",
+                    description: "The name of the file where the test is located. File name must end with .spec.ts",
+                },
+                changeToMake: {
+                    type: "string",
+                    description: "The change to make to the test",
+                },
+            },
+            required: ["testName", "fileName", "changeToMake"],
+        },
+    },
+};
+const generateTestWithCodegenTool = async ({ testName, fileName, changeToMake, }) => {
+    const testCase = {
+        id: 0,
+        name: testName,
+        filePath: fileName,
+        suites: [], // TODO: Support suites
+        steps: [changeToMake],
+    };
+    const result = await (0, run_1.generateTestWithCodegen)({
+        testCase,
+        file: fileName,
+    });
+    return result;
+};
+exports.generateTestWithCodegenTool = generateTestWithCodegenTool;

package/dist/tools/test-run.d.ts

@@ -0,0 +1,10 @@
+import { OpenAI } from "openai";
+export declare const schema: OpenAI.Chat.Completions.ChatCompletionTool;
+export declare const runTestTool: ({ testName, fileName, }: {
+    testName: string;
+    fileName: string;
+}) => Promise<{
+    hasTestPassed: boolean;
+    summaryJson: any;
+}>;
+//# sourceMappingURL=test-run.d.ts.map

package/dist/tools/test-run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,eAAO,MAAM,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAqB5C,CAAC;AAEF,eAAO,MAAM,WAAW;cAIZ,MAAM;cACN,MAAM;;;;EASjB,CAAC"}

package/dist/tools/test-run.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runTestTool = exports.schema = void 0;
+const test_run_1 = require("@empiricalrun/test-run");
+exports.schema = {
+    type: "function",
+    function: {
+        name: "runTest",
+        description: "Run a test",
+        parameters: {
+            type: "object",
+            properties: {
+                testName: {
+                    type: "string",
+                    description: "The name of the test to run",
+                },
+                fileName: {
+                    type: "string",
+                    description: "The name of the file where the test is located. File name must end with .spec.ts",
+                },
+            },
+            required: ["testName", "fileName"],
+        },
+    },
+};
+const runTestTool = async ({ testName, fileName, }) => {
+    // TODO: Remove this hardcoded project name
+    const result = await (0, test_run_1.runSingleTest)({
+        testName,
+        fileName,
+        projects: ["chromium"],
+    });
+    return result;
+};
+exports.runTestTool = runTestTool;

package/dist/utils/git.d.ts

@@ -0,0 +1,2 @@
+export declare function getGitDiff(filepath: string): string;
+//# sourceMappingURL=git.d.ts.map

package/dist/utils/git.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/utils/git.ts"],"names":[],"mappings":"AAEA,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKnD"}

package/dist/utils/git.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getGitDiff = void 0;
+const child_process_1 = require("child_process");
+function getGitDiff(filepath) {
+    const diff = (0, child_process_1.execSync)(`git diff ${filepath}`, {
+        encoding: "utf-8",
+    });
+    return diff;
+}
+exports.getGitDiff = getGitDiff;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.47.2",
+  "version": "0.47.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -75,7 +75,8 @@
     "typescript": "^5.3.3",
     "@empiricalrun/llm": "^0.9.36",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/reporter": "^0.23.1"
+    "@empiricalrun/reporter": "^0.23.1",
+    "@empiricalrun/test-run": "^0.7.1"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",

package/dist/agent/utils.d.ts

@@ -1,2 +0,0 @@
-export declare function parseJson(args: string): any;
-//# sourceMappingURL=utils.d.ts.map

package/dist/agent/utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/agent/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,OAMrC"}

package/dist/agent/utils.js

@@ -1,12 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseJson = void 0;
-function parseJson(args) {
-    try {
-        return JSON.parse(args);
-    }
-    catch (e) {
-        console.error(`Failed to parse JSON with args ${args}`, e);
-    }
-}
-exports.parseJson = parseJson;