@empiricalrun/test-gen 0.42.18 → 0.42.20

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @empiricalrun/test-gen
 
+## 0.42.20
+
+### Patch Changes
+
+- 9488f81: fix: infer-agent prompt template
+- b625749: chore: move infer-agent and planner prompts to handlebars
+- 5fb977c: test: clean up some unreliable tests
+- 8914542: feat: run planner inside master agent flow
+
+## 0.42.19
+
+### Patch Changes
+
+- c36efe4: chore: remove any type for get-next-action output
+- ebb0bfa: feat: support images in handlebar prompts
+- 63ed479: fix: remove verification and looping inside browsing agent
+
 ## 0.42.18
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts

@@ -1,20 +1,18 @@
 import { LLM, TraceClient } from "@empiricalrun/llm";
 import { Page } from "playwright";
 import { PlaywrightActions } from "../../actions";
-import { CustomLogger } from "../../bin/logger";
 import { TestGenConfigOptions } from "../../types";
 export type BrowsingAgentOptions = Partial<TestGenConfigOptions> & {
     htmlSanitize?: {
         disallowedStrings?: string[];
     };
 };
-export declare function executeTaskUsingBrowsingAgent({ trace, action, logger, page, options, llm, actions, }: {
+export declare function executeTaskUsingBrowsingAgent({ action, page, actions, llm, options, trace, }: {
     action: string;
-    trace?: TraceClient;
-    logger: CustomLogger;
     page: Page;
-    options: BrowsingAgentOptions;
-    llm: LLM;
     actions: PlaywrightActions;
+    llm: LLM;
+    trace?: TraceClient;
+    options: BrowsingAgentOptions;
 }): Promise<string[] | undefined>;
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAOnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,CAiJhC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAKlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAKnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,EACP,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,CAiEhC"}

package/dist/agent/browsing/index.js

@@ -2,148 +2,70 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeTaskUsingBrowsingAgent = void 0;
 const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a browser automation agent who is given a task to generate code for navigation and assertion. This task is your\ngoal and you must achieve it.\n\nYou will be provided with already executed actions and basis that you need to pick the next step to achieve the task.\nRemember that the goal must be achieved.\n\nYou will be provided with the web page snapshot in the form of Document Object Model. Based on the goal and available\ntool calls you need to pick the appropriate tool call.\n\nInstructions:\n- Take actions one at a time. Do not try to take multiple actions\n- You can respond with multiple assertions in one shot\n- Do not repeat the same actions again otherwise your response will be marked INVALID\n- Avoid repeating errors which we got while executing the last action\n- Stick to the task provided to you and mark the task done once the task is complete\n- Do not execute any action which is not mentioned in the task\n- Do not repeat actions which are already executed more than twice otherwise your response will be marked INVALID\n- Always refer to \"Executed actions\" before deciding your next action for completion of the task.\n- End the task done if all actions required for task are executed\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\nCurrent page snapshot:\n{{pageSnapshot}}\n{{/section}}";
+const lib_1 = require("../../prompts/lib");
 const reporter_1 = require("../../reporter");
-const session_1 = require("../../session");
 const html_1 = require("../../utils/html");
 const utils_1 = require("../utils");
-const verification_1 = require("../verification");
 const o1_completion_1 = require("./o1-completion");
-const utils_2 = require("./utils");
-async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, options, llm, actions, }) {
-    let isTaskDone = false;
-    const executedActions = [];
-    let lastActionExecTrace = "";
+async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, options, trace, }) {
     let generatedCodeSteps = [];
     const tools = actions.getBrowsingActionSchemas();
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
-    while (!isTaskDone) {
-        const browsingAgentSpan = trace?.span({
-            name: `browsing-agent`,
-            input: {
-                action,
-            },
-        });
-        if (await (0, session_1.shouldStopSession)()) {
-            break;
-        }
-        const pageContentSpan = browsingAgentSpan?.span({
-            name: "page-content",
-        });
-        const pageContent = await page.content();
-        pageContentSpan?.end({ output: { pageContent } });
-        const sanitizationSpan = browsingAgentSpan?.span({
-            name: "page-sanitization",
-        });
-        const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
-        sanitizationSpan?.end({ output: { pageSnapshot } });
-        const promptSpan = browsingAgentSpan?.span({ name: "page-prompt" });
-        // extract all successful actions
-        const successfulActions = executedActions
-            .filter((a) => !a.isError)
-            .map((a) => a.action);
-        if (successfulActions.length > 0) {
-            const verificationAgentResp = await (0, verification_1.verificationAgent)({
-                trace: browsingAgentSpan,
-                task: action,
-                conversation: ["Successfully executed actions", ...successfulActions],
-            });
-            isTaskDone = verificationAgentResp.isDone;
-            logger.log(`isTaskDone: ${isTaskDone}`);
-            logger.log(`reason: ${verificationAgentResp.reason}`);
-            if (isTaskDone) {
-                browsingAgentSpan?.event({ name: "task-done" });
-                browsingAgentSpan?.end({
-                    output: {
-                        taskDone: true,
-                        reason: verificationAgentResp.reason,
-                        code: generatedCodeSteps,
-                    },
-                });
-                break;
-            }
-        }
-        const messages = await (0, utils_2.getPromptForNextAction)({
-            pageSnapshot,
-            previousActions: successfulActions,
-            task: action,
-            lastActionErrors: lastActionExecTrace ? [lastActionExecTrace] : [],
-            promptType: "browsing-agent-as-tool",
-        });
-        promptSpan?.end({ output: { messages } });
-        let completion;
-        completion = await (0, o1_completion_1.getO1Completion)({
-            //@ts-ignore
+    const browsingAgentSpan = trace?.span({
+        name: `browsing-agent`,
+        input: {
+            action,
+        },
+    });
+    const pageContentSpan = browsingAgentSpan?.span({
+        name: "page-content",
+    });
+    const pageContent = await page.content();
+    pageContentSpan?.end({ output: { pageContent } });
+    const sanitizationSpan = browsingAgentSpan?.span({
+        name: "page-sanitization",
+    });
+    const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
+    sanitizationSpan?.end({ output: { pageSnapshot } });
+    const promptSpan = browsingAgentSpan?.span({ name: "page-prompt" });
+    const messages = await (0, lib_1.compilePrompt)(promptTemplate_0, { pageSnapshot, task: action });
+    promptSpan?.end({ output: { messages } });
+    let completion;
+    completion = await (0, o1_completion_1.getO1Completion)({
+        messages,
+        tools,
+        trace: browsingAgentSpan,
+    });
+    // If O1 completion fails due to any reason, resort to old flow
+    if (!completion) {
+        completion = await llm.createChatCompletion({
             messages,
             tools,
             trace: browsingAgentSpan,
+            model: options.model || constants_1.DEFAULT_MODEL,
+            modelParameters: {
+                ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                ...options.modelParameters,
+                tool_choice: "required",
+            },
         });
-        // If O1 completion fails due to any reason, resort to old flow
-        if (!completion) {
-            completion = await llm.createChatCompletion({
-                messages,
-                tools,
-                trace: browsingAgentSpan,
-                model: options.model || constants_1.DEFAULT_MODEL,
-                modelParameters: {
-                    ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                    ...options.modelParameters,
-                    tool_choice: "required",
-                },
-            });
-        }
-        const toolCalls = completion?.tool_calls || [];
-        // LLM might respond with empty tool_calls and we can go into endless loop
-        // if we donot record this action and mark it as error
-        if (!toolCalls.length) {
-            executedActions.push({
-                isError: true,
-                action: "",
-            });
-        }
-        const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
-        for (const i in toolCalls) {
-            const toolCall = toolCalls[i];
-            if (await (0, session_1.shouldStopSession)()) {
-                break;
-            }
-            try {
-                const code = await actions.executeAction(toolCall.function.name, (0, utils_1.parseJson)(toolCall.function.arguments), toolCallsSpan);
-                if (code) {
-                    generatedCodeSteps.push(code);
-                }
-                executedActions.push({
-                    isError: false,
-                    action: (0, utils_1.parseJson)(toolCall.function.arguments)?.reason,
-                });
-                lastActionExecTrace = "";
-            }
-            catch (e) {
-                // TODO: implement feedback loop to llm
-                executedActions.push({
-                    isError: true,
-                    action: (0, utils_1.parseJson)(toolCall.function.arguments)?.reason,
-                });
-                lastActionExecTrace = e.message;
-                void testgenUpdatesReporter.sendMessage(e.message);
-                logger.error(lastActionExecTrace, e);
+    }
+    const toolCalls = completion?.tool_calls || [];
+    const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
+    for (const i in toolCalls) {
+        const toolCall = toolCalls[i];
+        try {
+            const code = await actions.executeAction(toolCall.function.name, (0, utils_1.parseJson)(toolCall.function.arguments), toolCallsSpan);
+            if (code) {
+                generatedCodeSteps.push(code);
             }
         }
-        toolCallsSpan?.end({ output: { toolCalls } });
-        // mark task as done if llm is stuck in loop
-        if (executedActions.length >= 3) {
-            const lastThreeActions = executedActions.slice(-3);
-            const lastThreeActionsFailed = lastThreeActions.every((a) => a.isError);
-            // get last 3 lines of code
-            const isStuckInLoop = actions.isStuckInLoop();
-            if (lastThreeActionsFailed || isStuckInLoop) {
-                // TODO: this should be sent to dashboard
-                const error = "Agent is not able to figure out next browser action, ending retries";
-                logger.error(error);
-                await testgenUpdatesReporter.sendMessage(error);
-                throw Error(error);
-            }
+        catch (e) {
+            void testgenUpdatesReporter.sendMessage(e.message);
         }
     }
+    toolCallsSpan?.end({ output: { toolCalls } });
     return generatedCodeSteps;
 }
 exports.executeTaskUsingBrowsingAgent = executeTaskUsingBrowsingAgent;

package/dist/agent/browsing/utils.d.ts

@@ -29,13 +29,6 @@ export declare function readPlaywrightConfig(): Promise<PlaywrightTestConfig>;
  * @returns
  */
 export declare function detectProjectName(testFilePath: string, playwrightConfig: PlaywrightTestConfig, pwProjectsFilter?: string[]): Promise<string>;
-export declare function getPromptForNextAction({ pageSnapshot, task, previousActions, lastActionErrors, promptType, }: {
-    pageSnapshot: string;
-    task: string;
-    previousActions: string[];
-    lastActionErrors: string[];
-    promptType?: string;
-}): Promise<import("openai/resources/index.mjs").ChatCompletionMessageParam[]>;
 export declare class TeardownManager {
     private directory;
     constructor(directory: string);

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuHxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuHxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TeardownManager = exports.getPromptForNextAction = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunMasterAgent = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
-const llm_1 = require("@empiricalrun/llm");
+exports.TeardownManager = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunMasterAgent = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const minimatch_1 = require("minimatch");
 const path_1 = __importDefault(require("path"));
@@ -348,17 +347,6 @@ async function detectProjectName(testFilePath, playwrightConfig, pwProjectsFilte
     return filteredProjectNames[0];
 }
 exports.detectProjectName = detectProjectName;
-async function getPromptForNextAction({ pageSnapshot = "", task = "", previousActions = [], lastActionErrors = [], promptType = "browsing-agent-next-action", }) {
-    const previousActionsStr = previousActions.join("\n\n ---- \n\n");
-    const prompt = await (0, llm_1.getPrompt)(promptType, {
-        pageSnapshot,
-        previousActionsStr,
-        task,
-        lastActionErrors,
-    });
-    return prompt;
-}
-exports.getPromptForNextAction = getPromptForNextAction;
 class TeardownManager {
     directory;
     constructor(directory) {

package/dist/agent/codegen/create-test-block.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAavE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAqDA"}
+{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAcvE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAqDA"}

package/dist/agent/codegen/create-test-block.js

@@ -7,8 +7,8 @@ const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write an empty test block.\nBased on the inputs you need to create an empty playwright test block with correctly imported fixture.\n\nThe test will contain a test name which you will need to use to build the empty test case block.\n\nYou will be provided with current tests, fixtures and page object models for you to use and create test case block as\nper the task provided to you.\n\nBefore responding you need to ensure that the code change is minimal and the change is reusable across tests. You need\nto ensure the code follows DRY principle.\n\nHere is the list of current tests and fixtures:\n\n{{testFiles}}\n\nHere is the list of current page object models:\n\n{{pageFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nFollowing is the test scenario for which you need to write the empty test case block:\ntest name:\n{{scenarioName}}\n\ntask:\ncreate an empty test case block for the following test steps:\n{{scenario}}\n\ntest file path: {{scenarioFile}}\n\n------\n\nYou also need to ensure that the empty test case block has a starting page to begin test.\n\nIn order to identify the right page with which the test should start, follow the steps:\n- based on the similarities with other test cases mentioned in the file, identify the right page fixture to be imported\n- Read the page fixture methods step by step. Identify whether the fixture handles navigating to a page.\n- Identify whether other tests using the page fixture had to add separate steps for navigation or not\n- Based on the above analysis there will be following cases and choose either for the given test scenario:\n-- Case 1: if the test case scenario provided inside the task mentions about page navigation, then use that page\nnavigation. skip other cases if this case is satisfied.\n-- Case 2: refer other test cases which import similar fixtures and infer the first page navigation of this test case.\nYou should prefer tests which are in the same file. Tests within same file have higher overlaps in first page\nnavigation.\n- Once the page fixture is decided, look for userContext fixture in files. If its available then add \"userContext\" to\nthe test case block\n\n\n\nFollow these instructions before responding with output:\n- Read the code line by line and achieve the task provided to you\n- Read the dependencies of the code block by scanning through file paths and file provided to you. refer the same file\npath while responding with update\n- Focus only on the test case provided and associated JS methods called from the test case.\n- Respond only with the new empty test case block to be created and nothing else.\n- DO NOT respond with any backticks or markdown syntax\n- If \"userContext\" fixture is available in fixtures file, ensure importing that fixture in the test case block.\n- Provide a reason based on the test steps provided to you on why you chose the fixture or page.goto statement. The\nreason should be one of the list steps provided to you and mention why the case was chosen\n{{/section}}";
+const lib_1 = require("../../prompts/lib");
 const session_1 = require("../../session");
-const promptBuilder_1 = require("./promptBuilder");
 async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
     const logger = new logger_1.CustomLogger({ useReporter: false });
     logger.log("Creating new test block");
@@ -29,7 +29,7 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
     const promptSpan = trace?.span({
         name: "build-create-empty-test-case-prompt",
     });
-    const prompt = await (0, promptBuilder_1.compilePrompt)(promptTemplate_0, {
+    const prompt = await (0, lib_1.compilePrompt)(promptTemplate_0, {
         testFiles: context.codePrompt,
         pageFiles: context.pomPrompt,
         scenarioName: testCase.name,

package/dist/agent/codegen/lexical-scoped-vars.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lexical-scoped-vars.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/lexical-scoped-vars.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAGnD,wBAAsB,oBAAoB,CAAC,EACzC,KAAK,EACL,IAAI,EACJ,cAAc,EACd,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,qBAoDA"}
+{"version":3,"file":"lexical-scoped-vars.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/lexical-scoped-vars.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AASrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,wBAAsB,oBAAoB,CAAC,EACzC,KAAK,EACL,IAAI,EACJ,cAAc,EACd,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,qBAoDA"}

package/dist/agent/codegen/lexical-scoped-vars.js

@@ -4,12 +4,12 @@ exports.getLexicalScopedVars = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software engineer tasked with analysing Typescript code to identify all variables available in the lexical\nscope at a specific reference point within a file. You will be given a file that contains multiple Playwright tests or\npage object models, along with a reference point inside the file. Your goal is to evaluate the list of all variables\navailable in the lexical scope at that reference point.\n\nTo accomplish this, you need to evaluate the Abstract Syntax Tree (AST) and accumulate all variables that are in the\nlexical scope, which includes:\n1. Variables declared within the test before the reference point.\n2. Arguments of the function.\n3. Variables defined in the parent scope. Identify all variables available in the lexical scope at a specific execution\nreference point within a file, considering only those variables that have been declared and assigned prior to the\nexecution of this point in the code.\n4. Global variables defined in the file.\n\nBefore responding:\n- Ignore variables imported from the `\"./pages\"` path.\n- keep in mind temporal dead zone phenomenon before responding with variables\n{{/section}}\n\n{{#section \"user\"}}\nFile:\n{{testFile}}\n\nReference point:\n{{referencePoint}}\n{{/section}}";
-const promptBuilder_1 = require("./promptBuilder");
+const lib_1 = require("../../prompts/lib");
 async function getLexicalScopedVars({ trace, file, referencePoint, options, }) {
     const fetchLexicalScopedVarsSpan = trace?.span({
         name: "lexical-scoped-vars",
     });
-    const messages = await (0, promptBuilder_1.compilePrompt)(promptTemplate_0, {
+    const messages = await (0, lib_1.compilePrompt)(promptTemplate_0, {
         testFile: file || "",
         referencePoint: referencePoint || "",
     });

package/dist/agent/infer-agent/index.d.ts

@@ -1,5 +1,6 @@
 import { TraceClient } from "@empiricalrun/llm";
-import { Agent, TestGenConfigOptions } from "../../types";
+import { Agent } from "@empiricalrun/shared-types";
+import { TestGenConfigOptions } from "../../types";
 export declare function inferAgentBasedTask({ task, options, trace, }: {
     task: string;
     options?: TestGenConfigOptions;

package/dist/agent/infer-agent/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/infer-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG1D,wBAAsB,mBAAmB,CAAC,EACxC,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,KAAK,CAAA;CAAE,CAAC,CA4E/B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/infer-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAWnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAGnD,wBAAsB,mBAAmB,CAAC,EACxC,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,KAAK,CAAA;CAAE,CAAC,CAmE/B"}

package/dist/agent/infer-agent/index.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.inferAgentBasedTask = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer specializing in Playwright end-to-end tests. You are given a task which is a part of an\nend-to-end test scenario. The task may involve updating an existing end-to-end test case or writing a new test case from\nscratch. Tests involve user interactions (e.g. click on element) or other actions supported by Playwright (e.g.\nintercept network requests)\n\nYour objective is to identify whether the task requires accessing a web browser or not.\n\nTo fulfill your objective, answer the following questions:\n\n1. Does it require you to interact with a UI element in the browser? Examples of interactions are click, fill, type, key\npress, assert visibility of the element. Actions that interact with network requests are not UI element interactions.\n\n2. Is the locator of this UI element given to you in the task? Locators look like `getByText(...)`, `getByTestId(...)`\nand other locator methods in Playwright\n\n3. Decide if you need a browser: if you need to interact with a UI element AND you are NOT given the locator for that\nelement, you WILL NEED a browser.\n\n4. If you NEED a browser, then respond with answer as \"master\", otherwise respond with \"code\"\n\n\n# Example 1\n## Input\nTask:\nin this test don't delete the agent and remove steps after that\n\n## Output\n- ui_interaction_to_be_performed: There is no interaction here\n- ui_element_to_interact_with: No element specified\n- has_locator_for_that_element: No element specified\n- reasoning_for_browser_required: No interaction hence browser is not required\n- answer: code\n\n# Example 2\n## Input\nTask:\nin the swapfast test, replace the selectTokenForSwap method. Instead we will do this\\nclick on token button - this will\nshow a drawer\\nEnter usd in the search field that shows up in the drawer\\nSelect USDC.axl for Cosmos Hub - very\nimportant to choose this instead of USDC.axl on Osmosis\n\n## Output\n- ui_interaction_to_be_performed: Click on token button\n- ui_element_to_interact_with: Token button\n- has_locator_for_that_element: false\n- reasoning_for_browser_required: Task requires interacting with a UI element and identifying its locator which needs a\nbrowser\n- answer: master\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n{{/section}}";
+const lib_1 = require("../../prompts/lib");
 const session_1 = require("../../session");
 const session = (0, session_1.getSessionDetails)();
 async function inferAgentBasedTask({ task, options, trace, }) {
@@ -20,16 +22,7 @@ async function inferAgentBasedTask({ task, options, trace, }) {
             options,
         },
     });
-    const promptSpan = inferAgentSpan?.span({
-        name: "infer-agent-prompt",
-        input: {
-            task,
-        },
-    });
-    const prompt = await (0, llm_1.getPrompt)("infer-agent", {
-        task,
-    });
-    promptSpan?.end({ output: { prompt } });
+    const messages = (0, lib_1.compilePrompt)(promptTemplate_0, { task });
     const llm = new llm_1.LLM({
         trace: inferAgentSpan,
         provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
@@ -37,7 +30,7 @@ async function inferAgentBasedTask({ task, options, trace, }) {
         providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
     });
     const firstShotMessage = await llm.createChatCompletion({
-        messages: prompt,
+        messages,
         modelParameters: {
             ...constants_1.DEFAULT_MODEL_PARAMETERS,
             ...options?.modelParameters,

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAgG9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}

package/dist/agent/master/element-annotation.js

@@ -2,8 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getAnnotationKeys = exports.getElementAnnotation = void 0;
 const llm_1 = require("@empiricalrun/llm");
-const vision_1 = require("@empiricalrun/llm/vision");
 const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements\npresent in the image.\n\nYou will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation\nis done by drawing a red box around the element and a small yellow box on it which contains unique element id.\n\nYou are given a Annotations which contains list of unique element id and description of the element separated by \":\".\n\nYou are also given the description of the element on which the action needs to be taken. The description includes\ninformation about how the element looks, it's position etc.\n\nYour task is to provide the annotation of the element on which the action needs to be performed based on the element\ndescription.\n\nFollow steps to fulfil your task:\n- Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each\nelement.\n- For describing each element Id\n-- iterate over each element Id in annotation list\n-- check if the description is already present for the element Id in the Annotation provided to you. If present skip\ndescribing it and use it as is.\n-- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or\nicon enclosed in the element.\n- Respond with the mapped element Ids as \"enriched_annotations\"\n- Based on the description provided to you and the enriched annotations, first identify the element Id whose description\nmatches the task provided\n\nNote:\n- Ensure providing the description of all the elements in the list.\n- Don't update the description if its already present in the given annotations\n- Replace all the \"NA\" with description of the element. Its position, how does it look like etc.\n- There should be no \"NA\" present in any of the element description\n{{/section}}\n\n{{#section \"user\"}}\nElement description:\n{{elementDescription}}\n\nAnnotations:\n{{annotations}}\n\n{{image annotatedScreenshot}}\n{{/section}}";
+const lib_1 = require("../../prompts/lib");
 const utils_1 = require("../utils");
 const annotationToolAction = {
     name: "element_annotation",
@@ -66,58 +67,11 @@ async function getElementAnnotation({ elementDescription, annotations, annotated
             preference,
         },
     });
-    const systemMessage = {
-        role: "system",
-        content: `You are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements present in the image.
-  
-  You will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation is done by drawing a red box around the element and a small yellow box on it which contains unique element id.
-  
-  You are given a Annotations which contains list of  unique element id  and description of the element separated by ":". 
-  
-  You are also given the description of the element on which the action needs to be taken. The description includes information about how the element looks, it's position etc.
-  
-  Your task is to provide the annotation of the element on which the action needs to be performed based on the element description.
-  
-  Follow steps to fulfil your task:
-  - Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each element. 
-  - For describing each element Id
-  -- iterate over each element Id in annotation list
-  -- check if the description is already present for the element Id in the Annotation provided to you. If present skip describing it and use it as is.
-  -- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or icon enclosed in the element. 
-  - Respond with the mapped element Ids as "enriched_annotations"
-  - Based on the description provided to you and the enriched annotations, first identify the element Id whose description matches the task provided
-  
-  Note:
-  - Ensure providing the description of all the elements in the list.
-  - Don't update the description if its already present in the given annotations
-  - Replace all the "NA" with description of the element. Its position, how does it look like etc. 
-  - There should be no "NA" present in any of the element description
-  `,
-    };
-    const userMessage = {
-        role: "user",
-        content: [
-            {
-                type: "text",
-                text: `
-  Element description:
-  ${elementDescription}
-  
-  Annotations:
-  ${annotations}`,
-            },
-            {
-                type: "image_url",
-                image_url: {
-                    url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, annotatedScreenshot),
-                },
-            },
-        ],
-    };
-    const messages = [
-        systemMessage,
-        userMessage,
-    ];
+    const messages = (0, lib_1.compilePrompt)(promptTemplate_0, {
+        elementDescription,
+        annotations,
+        annotatedScreenshot,
+    }, options);
     llm =
         llm ||
             new llm_1.LLM({

package/dist/agent/master/run.d.ts

@@ -1,10 +1,11 @@
 import { Page } from "playwright";
 import { ScopeVars, TestCase } from "../../types";
 import { BrowsingAgentOptions } from "../browsing";
-export declare function createTestUsingMasterAgent({ task, page, testCase, options, scopeVars, }: {
+export declare function createTestUsingMasterAgent({ task, page, testCase, specPath, options, scopeVars, }: {
     task: string;
     page: Page;
     testCase?: TestCase;
+    specPath?: string;
     options: BrowsingAgentOptions;
     scopeVars?: ScopeVars;
 }): Promise<{

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA4BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA+UA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA0WA"}

package/dist/agent/master/run.js

@@ -13,6 +13,7 @@ const session_1 = require("../../session");
 const browsing_1 = require("../browsing");
 const utils_2 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
+const run_1 = require("../planner/run");
 const run_time_planner_1 = require("../planner/run-time-planner");
 const utils_3 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
@@ -29,8 +30,9 @@ function getPageVariables(stateVariables) {
     }, {});
     return pages;
 }
-async function createTestUsingMasterAgent({ task, page, testCase, options, scopeVars, }) {
+async function createTestUsingMasterAgent({ task, page, testCase, specPath, options, scopeVars, }) {
     const useActionSpecificAnnotations = options?.useActionSpecificAnnotations || false;
+    const usePlannerInMaster = options?.usePlannerInMaster || false;
     const logger = new logger_1.CustomLogger({ useReporter: false });
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     const session = (0, session_1.getSessionDetails)();
@@ -74,6 +76,19 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         });
     }
     skill_1.testCaseSkills.updateSkills(skills);
+    if (usePlannerInMaster && testCase && specPath) {
+        void testgenUpdatesReporter.sendMessage(`Planner is working on task: ${task}`);
+        logger.log(`Planner is working on task: ${task}`);
+        const plan = await (0, run_1.planTask)({
+            task,
+            specPath,
+            trace,
+        });
+        void testgenUpdatesReporter.sendMessage(`Here is the plan:\n${plan}`);
+        logger.log(`Here is the plan:\n${plan}`);
+        // Will assume this is the task hereon
+        task = plan;
+    }
     const actions = new actions_1.PlaywrightActions(testGenPage, scopeVars);
     await (0, utils_2.injectPwLocatorGenerator)(page);
     trace?.update({ input: { task } });
@@ -150,11 +165,11 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
             const masterAgentActionSpan = masterAgentSpan?.span({
                 name: "master-agent-execute-action",
             });
+            output = {
+                action: args.action || args.skill,
+                reason: args.reason,
+            };
             try {
-                output = {
-                    action: args.action || args.skill,
-                    reason: args.reason,
-                };
                 void testGenReporter.sendMessage(output.reason);
                 logger.log(`Next Action: ${output.action}`);
                 if (toolCall.actionType === skill_1.SKILL_USAGE) {
@@ -197,6 +212,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                         options,
                     });
                     if (annotationKeys.length > 0) {
+                        // TODO: this string has newline characters that makes it harder to read
                         const annotationMapString = annotationKeys
                             ?.map((a) => `${a.elementID}: ${a.text}`)
                             .join("\n");
@@ -254,7 +270,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                         generatedCodeSteps = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
                             trace: masterAgentActionSpan,
                             action: output.action,
-                            logger,
                             page,
                             options,
                             llm,
@@ -307,8 +322,8 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         trace?.update({ input: { task }, output: { output } });
         masterAgentSpan?.end({
             output: {
-                action: output.action,
-                reason: output.reason,
+                action: output?.action,
+                reason: output?.reason,
                 code: generatedCodeSteps,
             },
         });

package/dist/agent/planner/run-time-planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAKrD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GAiHA"}
+{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAKrD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GAkHA"}

package/dist/agent/planner/run-time-planner.js

@@ -40,7 +40,8 @@ To fulfil your goal, follow these steps:
         {
             role: "user",
             content: `
-Task: ${task}
+Task:
+${task}
 
 ----
 

package/dist/agent/planner/run.d.ts

@@ -1,7 +1,6 @@
 import { TraceClient } from "@empiricalrun/llm";
-import { TestCase } from "../../types";
-export declare function planTask({ testCase, specPath, trace, }: {
-    testCase: TestCase;
+export declare function planTask({ task, specPath, trace, }: {
+    task: string;
     specPath: string;
     trace?: TraceClient;
 }): Promise<string>;

package/dist/agent/planner/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAsBvC,wBAAsB,QAAQ,CAAC,EAC7B,QAAQ,EACR,QAAQ,EACR,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,mBA0DA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AA6BrD,wBAAsB,QAAQ,CAAC,EAC7B,IAAI,EACJ,QAAQ,EACR,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,mBAyCA"}

package/dist/agent/planner/run.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.planTask = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const context_1 = require("../../bin/utils/context");
+const lib_1 = require("../../prompts/lib");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert software engineer in test. You are given a task to provide a high level plan to create a test for a\ngiven scenario.\n\nYou will be provided with already added tests and page object object models which you can use to plan out how to write\nthe test.\n\nThe expected plan should be a list of bullet points and each bullet point is a step in the test.\nYou will be provided with app knowledge as well, which can help you groom the steps in the tests.\n{{/section}}\n\n{{#section \"user\"}}\nApp knowledge\n{{appKnowledge}}\n\n-----\n\nCurrent tests and page object model references\n\n{{fileContext}}\n\n------\n\nTask:\n\n{{task}}\n\n------\n\nFollow the steps to create a test plan:\n- create sub tasks\n- read the task step by step and create sub tasks from the given task\n- Ensure no new steps are added which are not mentioned in the task\n- enriched sub tasks\n- Read the app knowledge provided to you and enrich the verified sub tasks based on the provided information in app\nknowledge.\n- Fill in the missing information in the verified sub tasks based on the app knowledge.\n- based on the type of task, add sub tasks to the verified sub tasks based on the matching criteria\n- final plan:\n- Once all the subtasks are enriched, list all the sub tasks as bullet points\n- Each bullet point should be one of the following actions:\n- Open page, Click on, Fill in, Assert, hover on, press, extract textContent\n- Do not respond with points which do not start with above actions.\n\nFollow the steps before responding\n- The steps should only contain bullet points on list of steps for the test\n- Do not add any other assertion which is not mentioned in the task or app knowledge\n- Respond with <create_sub_tasks></create_sub_tasks>\n<enriched_sub_tasks></enriched_sub_tasks> and <final_plan></final_plan>\n- The final plan should not mention reference to the knowledge base used to generate it\n{{/section}}";
 function extractTestPlan(input) {
     const result = {
         createSubTasks: "",
@@ -20,9 +22,7 @@ function extractTestPlan(input) {
     }
     return result;
 }
-async function planTask({ testCase, specPath, trace, }) {
-    const task = testCase.steps.join("\n");
-    // TODO: fix this
+async function planTask({ task, specPath, trace, }) {
     let fileContext = "";
     try {
         const { pomPrompt, testFileContent } = await (0, context_1.contextForGeneration)(specPath);
@@ -37,24 +37,15 @@ ${pomPrompt}
     const plannerSpan = trace?.span({
         name: "planner",
         input: {
-            testCase,
+            task,
         },
     });
     const appKnowledge = await (0, context_1.fetchAppKnowledge)();
-    const promptSpan = plannerSpan?.span({
-        name: "planner-prompt",
-        input: {
-            appKnowledge,
-            fileContext,
-            testCase,
-        },
-    });
-    const messages = await (0, llm_1.getPrompt)("planner", {
+    const messages = (0, lib_1.compilePrompt)(promptTemplate_0, {
         appKnowledge,
         fileContext,
         task,
-    }, 2);
-    promptSpan?.end({ output: { messages } });
+    });
     const llm = new llm_1.LLM({
         provider: "openai",
         //TODO: change to o1

package/dist/bin/index.js

@@ -102,8 +102,9 @@ async function runAgent(testGenConfig) {
         testCase.steps = [requestedChangeResp.output];
     }
     if (agent === "plan") {
+        const task = testCase.steps.join("\n");
         const plan = await (0, run_3.planTask)({
-            testCase,
+            task,
             specPath,
             trace,
         });

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAQpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAsC3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAQpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAuC3E"}

package/dist/index.js

@@ -31,8 +31,10 @@ async function createTest(task, page, scope) {
         projectRepoName: testGenConfig.options?.metadata.projectRepoName,
     });
     const fileService = new client_1.default(Number(port));
+    const { testCase, specPath } = testGenConfig;
     const { code, importPaths } = await (0, run_1.createTestUsingMasterAgent)({
-        testCase: testGenConfig.testCase,
+        testCase,
+        specPath,
         page,
         task,
         options: {

package/dist/prompts/lib/index.d.ts

@@ -0,0 +1,8 @@
+import { LLMProvider } from "@empiricalrun/llm";
+import OpenAI from "openai";
+type PromptOptions = {
+    modelProvider?: LLMProvider;
+};
+export declare function compilePrompt<T extends object>(promptTemplate: string, params: T, options?: PromptOptions): OpenAI.Chat.Completions.ChatCompletionMessageParam[];
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/prompts/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/prompts/lib/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAyF5B,KAAK,aAAa,GAAG;IACnB,aAAa,CAAC,EAAE,WAAW,CAAC;CAC7B,CAAC;AAEF,wBAAgB,aAAa,CAAC,CAAC,SAAS,MAAM,EAC5C,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,CAAC,EACT,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAwCtD"}

package/dist/prompts/lib/index.js

@@ -0,0 +1,118 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compilePrompt = void 0;
+const vision_1 = require("@empiricalrun/llm/vision");
+const handlebars_1 = __importDefault(require("handlebars"));
+const constants_1 = require("../../constants");
+class SectionManager {
+    sections = {};
+    getSection(name) {
+        return this.sections[name] || "";
+    }
+    setSection(name, content) {
+        this.sections[name] = content;
+    }
+    getAllSections() {
+        return this.sections;
+    }
+}
+const IMAGE_TOKEN_PREFIX = "[[[HANDLEBARS_IMAGE:";
+const IMAGE_TOKEN_SUFFIX = "]]]";
+function createHandlebarsEnv() {
+    const HandlebarsEnv = handlebars_1.default.create();
+    const sectionManager = new SectionManager();
+    HandlebarsEnv.registerHelper("section", function (name, options) {
+        const content = options.fn(this);
+        sectionManager.setSection(name, content);
+        return ""; // don't output anything in place
+    });
+    HandlebarsEnv.registerHelper("image", function (imageParam) {
+        const tokenPayload = JSON.stringify({ url: imageParam });
+        // Use encodeURIComponent to avoid conflicts with special characters.
+        const token = `${IMAGE_TOKEN_PREFIX}${encodeURIComponent(tokenPayload)}${IMAGE_TOKEN_SUFFIX}`;
+        return token;
+    });
+    HandlebarsEnv.registerHelper("images", function (imagesParam) {
+        if (!Array.isArray(imagesParam))
+            return "";
+        return imagesParam
+            .map((url) => {
+            const tokenPayload = JSON.stringify({ url });
+            return `${IMAGE_TOKEN_PREFIX}${encodeURIComponent(tokenPayload)}${IMAGE_TOKEN_SUFFIX}`;
+        })
+            .join("");
+    });
+    return { HandlebarsEnv, sectionManager };
+}
+function processSectionContent(content) {
+    if (!content.includes(IMAGE_TOKEN_PREFIX)) {
+        return content.trim();
+    }
+    const segments = [];
+    const regex = /\[\[\[HANDLEBARS_IMAGE:(.*?)\]\]\]/g;
+    let lastIndex = 0;
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+        // Get the text before the token.
+        const textPart = content.slice(lastIndex, match.index).trim();
+        if (textPart) {
+            segments.push({ type: "text", text: textPart });
+        }
+        // Decode the token payload.
+        try {
+            const payloadJson = decodeURIComponent(match[1]);
+            const payload = JSON.parse(payloadJson);
+            segments.push({ type: "image_url", image_url: { url: payload.url } });
+        }
+        catch (err) {
+            // If decoding/parsing fails, treat the token as literal text.
+            segments.push({ type: "text", text: match[0] });
+        }
+        lastIndex = match.index + match[0].length;
+    }
+    const remaining = content.slice(lastIndex).trim();
+    if (remaining) {
+        segments.push({ type: "text", text: remaining });
+    }
+    return segments;
+}
+function compilePrompt(promptTemplate, params, options) {
+    const { HandlebarsEnv, sectionManager } = createHandlebarsEnv();
+    const template = HandlebarsEnv.compile(promptTemplate, { noEscape: true });
+    template(params);
+    const sections = sectionManager.getAllSections();
+    // TODO: system cannot have images, we can add validation for that
+    const system = sections["system"];
+    const user = sections["user"];
+    if (!system || !user) {
+        // TODO: support templates that have only one section
+        throw new Error("Both system and user sections must be defined in the template");
+    }
+    const systemContent = processSectionContent(system);
+    const userContent = processSectionContent(user);
+    let userContentCorrectedForImageFormat = userContent;
+    if (Array.isArray(userContent)) {
+        const provider = options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER;
+        userContentCorrectedForImageFormat = userContent.map((c) => {
+            if (c.type === "image_url") {
+                return {
+                    ...c,
+                    image_url: {
+                        url: (0, vision_1.imageFormatForProvider)(provider, c.image_url.url),
+                    },
+                };
+            }
+            else {
+                return c;
+            }
+        });
+    }
+    return [
+        { role: "system", content: systemContent },
+        { role: "user", content: userContentCorrectedForImageFormat },
+    ];
+}
+exports.compilePrompt = compilePrompt;

package/dist/types/index.d.ts

@@ -1,4 +1,5 @@
-import { LLMModel, LLMProvider, ModelParameters, TraceClient } from "@empiricalrun/llm";
+import { ModelParameters, TraceClient } from "@empiricalrun/llm";
+import { Agent, LLMModel, LLMProvider } from "@empiricalrun/shared-types";
 import OpenAI from "openai";
 import { TestGenPage } from "../page";
 export type FileContent = {
@@ -10,14 +11,11 @@ export type Environment = {
     slug: string;
     playwrightProjects: string[];
 };
-export type Agent = "code" | "master" | "auto" | "plan";
 export type TestGenConfigOptions = {
     agent: Agent;
     model: LLMModel;
     modelProvider: LLMProvider;
     modelParameters?: ModelParameters;
-    useActionSpecificAnnotations?: boolean;
-    useStrReplace?: boolean;
     metadata: {
         testSessionId: number;
         generationId: number;
@@ -26,6 +24,9 @@ export type TestGenConfigOptions = {
         projectName: string;
         environment: "development" | "production";
     };
+    useActionSpecificAnnotations?: boolean;
+    useStrReplace?: boolean;
+    usePlannerInMaster?: boolean;
 };
 export type Build = {
     url?: string;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAExD,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,kBAAkB,CAAC,EAAE,yBAAyB,CAAC;IAC/C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACpC,iBAAiB,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAClE,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACjE,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,KAChD;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;IAEF,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,kBAAkB,CAAC,EAAE,yBAAyB,CAAC;IAC/C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACpC,iBAAiB,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAClE,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACjE,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,KAChD;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.42.18",
+  "version": "0.42.20",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -73,8 +73,8 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "@empiricalrun/llm": "^0.9.32",
-    "@empiricalrun/reporter": "^0.23.1",
-    "@empiricalrun/r2-uploader": "^0.3.8"
+    "@empiricalrun/r2-uploader": "^0.3.8",
+    "@empiricalrun/reporter": "^0.23.1"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",
@@ -86,7 +86,8 @@
     "@types/md5": "^2.3.5",
     "js-levenshtein": "^1.1.6",
     "playwright": "1.47.1",
-    "ts-patch": "^3.3.0"
+    "ts-patch": "^3.3.0",
+    "@empiricalrun/shared-types": "0.0.0"
   },
   "scripts": {
     "dev": "tspc --build --watch",

package/dist/agent/codegen/promptBuilder.d.ts

@@ -1,3 +0,0 @@
-import OpenAI from "openai";
-export declare function compilePrompt<T extends object>(promptTemplate: string, params: T): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
-//# sourceMappingURL=promptBuilder.d.ts.map

package/dist/agent/codegen/promptBuilder.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"promptBuilder.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/promptBuilder.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AA6B5B,wBAAsB,aAAa,CAAC,CAAC,SAAS,MAAM,EAClD,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,CAAC,GACR,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC,CAe/D"}

package/dist/agent/codegen/promptBuilder.js

@@ -1,44 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.compilePrompt = void 0;
-const handlebars_1 = __importDefault(require("handlebars"));
-class SectionManager {
-    sections = {};
-    getSection(name) {
-        return this.sections[name] || "";
-    }
-    setSection(name, content) {
-        this.sections[name] = content;
-    }
-    getAllSections() {
-        return this.sections;
-    }
-}
-function createHandlebarsEnv() {
-    const HandlebarsEnv = handlebars_1.default.create();
-    const sectionManager = new SectionManager();
-    HandlebarsEnv.registerHelper("section", function (name, options) {
-        const content = options.fn(this);
-        sectionManager.setSection(name, content);
-        return ""; // Don't output anything in place
-    });
-    return { HandlebarsEnv, sectionManager };
-}
-async function compilePrompt(promptTemplate, params) {
-    const { HandlebarsEnv, sectionManager } = createHandlebarsEnv();
-    const template = HandlebarsEnv.compile(promptTemplate, { noEscape: true });
-    template(params);
-    const { system, user } = sectionManager.getAllSections();
-    if (!system || !user) {
-        // TODO: support templates that have only one section
-        throw new Error("Both system and user sections must be defined in the template");
-    }
-    return [
-        { role: "system", content: system },
-        { role: "user", content: user },
-    ];
-}
-exports.compilePrompt = compilePrompt;

package/dist/agent/verification/index.d.ts

@@ -1,13 +0,0 @@
-import { TraceClient } from "@empiricalrun/llm";
-/**
- * This agent is used to verify whether the task is done basis the conversation history
- */
-export declare function verificationAgent({ trace, task, conversation, }: {
-    trace?: TraceClient;
-    conversation: string[];
-    task: string;
-}): Promise<{
-    isDone: boolean;
-    reason: string;
-}>;
-//# sourceMappingURL=index.d.ts.map

package/dist/agent/verification/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA+EA"}

package/dist/agent/verification/index.js

@@ -1,84 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.verificationAgent = void 0;
-const llm_1 = require("@empiricalrun/llm");
-const utils_1 = require("../utils");
-/**
- * This agent is used to verify whether the task is done basis the conversation history
- */
-async function verificationAgent({ trace, task, conversation, }) {
-    const verificationAgentSpan = trace?.span({
-        name: "verification-agent",
-        input: {
-            task,
-            conversation,
-        },
-    });
-    const messages = await (0, llm_1.getPrompt)("agent-steps-verification", {
-        task,
-        conversation: conversation.join("\n"),
-    }, 5);
-    const llm = new llm_1.LLM({ provider: "openai" });
-    const response = await llm.createChatCompletion({
-        trace: verificationAgentSpan,
-        traceName: "verification-agent-llm",
-        model: "gpt-4o",
-        messages,
-        tools: [
-            {
-                type: "function",
-                function: {
-                    name: "task_done",
-                    description: "end the task by calling this method",
-                    parameters: {
-                        type: "object",
-                        properties: {
-                            actions: {
-                                type: "string",
-                                description: "actions extracted from task",
-                            },
-                            successful_actions: {
-                                type: "string",
-                                description: "successful actions mentioned in the conversation",
-                            },
-                            reason: {
-                                type: "string",
-                                description: "reasoning for identification of task status",
-                            },
-                            isDone: {
-                                type: "boolean",
-                                description: "whether the task is done",
-                            },
-                        },
-                        required: ["isDone", "reason"],
-                    },
-                },
-            },
-        ],
-        modelParameters: {
-            tool_choice: "required",
-            temperature: 0.5,
-        },
-    });
-    const toolCallResp = (response?.tool_calls || [])[0];
-    if (toolCallResp) {
-        const toolCall = (0, utils_1.parseJson)(toolCallResp.function.arguments);
-        const output = {
-            isDone: toolCall.isDone,
-            reason: toolCall.reason,
-        };
-        verificationAgentSpan?.end({
-            output,
-        });
-        return output;
-    }
-    const output = {
-        isDone: false,
-        reason: "LLM failed to generate a valid response",
-    };
-    verificationAgentSpan?.end({
-        output,
-    });
-    return output;
-}
-exports.verificationAgent = verificationAgent;

package/dist/evals/verification-agent.evals.d.ts

@@ -1,4 +0,0 @@
-import { EvaluateFn } from "./type";
-export declare const verifierAgentEvaluate: EvaluateFn;
-export default verifierAgentEvaluate;
-//# sourceMappingURL=verification-agent.evals.d.ts.map

package/dist/evals/verification-agent.evals.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"verification-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/verification-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,qBAAqB,EAAE,UAgBnC,CAAC;AAEF,eAAe,qBAAqB,CAAC"}

package/dist/evals/verification-agent.evals.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.verifierAgentEvaluate = void 0;
-const verification_1 = require("../agent/verification");
-const verifierAgentEvaluate = async ({ item, trace }) => {
-    const { conversation = [], task = "" } = item.input;
-    const output = await (0, verification_1.verificationAgent)({
-        conversation,
-        trace,
-        task,
-    });
-    return {
-        scores: [
-            {
-                name: "equality",
-                value: item.expectedOutput.isDone === output.isDone ? 1 : 0,
-            },
-        ],
-        output,
-    };
-};
-exports.verifierAgentEvaluate = verifierAgentEvaluate;
-exports.default = exports.verifierAgentEvaluate;