@empiricalrun/test-gen 0.42.17 → 0.42.19

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @empiricalrun/test-gen
 
+## 0.42.19
+
+### Patch Changes
+
+- c36efe4: chore: remove any type for get-next-action output
+- ebb0bfa: feat: support images in handlebar prompts
+- 63ed479: fix: remove verification and looping inside browsing agent
+
+## 0.42.18
+
+### Patch Changes
+
+- 6f876ea: fix: remove code generation feedback loop from repo edit
+- 658451e: fix: tracing for vitests
+- Updated dependencies [658451e]
+  - @empiricalrun/llm@0.9.32
+
 ## 0.42.17
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts

@@ -1,20 +1,18 @@
 import { LLM, TraceClient } from "@empiricalrun/llm";
 import { Page } from "playwright";
 import { PlaywrightActions } from "../../actions";
-import { CustomLogger } from "../../bin/logger";
 import { TestGenConfigOptions } from "../../types";
 export type BrowsingAgentOptions = Partial<TestGenConfigOptions> & {
     htmlSanitize?: {
         disallowedStrings?: string[];
     };
 };
-export declare function executeTaskUsingBrowsingAgent({ trace, action, logger, page, options, llm, actions, }: {
+export declare function executeTaskUsingBrowsingAgent({ action, page, actions, llm, options, trace, }: {
     action: string;
-    trace?: TraceClient;
-    logger: CustomLogger;
     page: Page;
-    options: BrowsingAgentOptions;
-    llm: LLM;
     actions: PlaywrightActions;
+    llm: LLM;
+    trace?: TraceClient;
+    options: BrowsingAgentOptions;
 }): Promise<string[] | undefined>;
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAOnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,CAiJhC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAKlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAKnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,EACP,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,CAiEhC"}

package/dist/agent/browsing/index.js

@@ -2,148 +2,70 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeTaskUsingBrowsingAgent = void 0;
 const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a browser automation agent who is given a task to generate code for navigation and assertion. This task is your\ngoal and you must achieve it.\n\nYou will be provided with already executed actions and basis that you need to pick the next step to achieve the task.\nRemember that the goal must be achieved.\n\nYou will be provided with the web page snapshot in the form of Document Object Model. Based on the goal and available\ntool calls you need to pick the appropriate tool call.\n\nInstructions:\n- Take actions one at a time. Do not try to take multiple actions\n- You can respond with multiple assertions in one shot\n- Do not repeat the same actions again otherwise your response will be marked INVALID\n- Avoid repeating errors which we got while executing the last action\n- Stick to the task provided to you and mark the task done once the task is complete\n- Do not execute any action which is not mentioned in the task\n- Do not repeat actions which are already executed more than twice otherwise your response will be marked INVALID\n- Always refer to \"Executed actions\" before deciding your next action for completion of the task.\n- End the task done if all actions required for task are executed\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\nCurrent page snapshot:\n{{pageSnapshot}}\n{{/section}}";
+const lib_1 = require("../../prompts/lib");
 const reporter_1 = require("../../reporter");
-const session_1 = require("../../session");
 const html_1 = require("../../utils/html");
 const utils_1 = require("../utils");
-const verification_1 = require("../verification");
 const o1_completion_1 = require("./o1-completion");
-const utils_2 = require("./utils");
-async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, options, llm, actions, }) {
-    let isTaskDone = false;
-    const executedActions = [];
-    let lastActionExecTrace = "";
+async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, options, trace, }) {
     let generatedCodeSteps = [];
     const tools = actions.getBrowsingActionSchemas();
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
-    while (!isTaskDone) {
-        const browsingAgentSpan = trace?.span({
-            name: `browsing-agent`,
-            input: {
-                action,
-            },
-        });
-        if (await (0, session_1.shouldStopSession)()) {
-            break;
-        }
-        const pageContentSpan = browsingAgentSpan?.span({
-            name: "page-content",
-        });
-        const pageContent = await page.content();
-        pageContentSpan?.end({ output: { pageContent } });
-        const sanitizationSpan = browsingAgentSpan?.span({
-            name: "page-sanitization",
-        });
-        const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
-        sanitizationSpan?.end({ output: { pageSnapshot } });
-        const promptSpan = browsingAgentSpan?.span({ name: "page-prompt" });
-        // extract all successful actions
-        const successfulActions = executedActions
-            .filter((a) => !a.isError)
-            .map((a) => a.action);
-        if (successfulActions.length > 0) {
-            const verificationAgentResp = await (0, verification_1.verificationAgent)({
-                trace: browsingAgentSpan,
-                task: action,
-                conversation: ["Successfully executed actions", ...successfulActions],
-            });
-            isTaskDone = verificationAgentResp.isDone;
-            logger.log(`isTaskDone: ${isTaskDone}`);
-            logger.log(`reason: ${verificationAgentResp.reason}`);
-            if (isTaskDone) {
-                browsingAgentSpan?.event({ name: "task-done" });
-                browsingAgentSpan?.end({
-                    output: {
-                        taskDone: true,
-                        reason: verificationAgentResp.reason,
-                        code: generatedCodeSteps,
-                    },
-                });
-                break;
-            }
-        }
-        const messages = await (0, utils_2.getPromptForNextAction)({
-            pageSnapshot,
-            previousActions: successfulActions,
-            task: action,
-            lastActionErrors: lastActionExecTrace ? [lastActionExecTrace] : [],
-            promptType: "browsing-agent-as-tool",
-        });
-        promptSpan?.end({ output: { messages } });
-        let completion;
-        completion = await (0, o1_completion_1.getO1Completion)({
-            //@ts-ignore
+    const browsingAgentSpan = trace?.span({
+        name: `browsing-agent`,
+        input: {
+            action,
+        },
+    });
+    const pageContentSpan = browsingAgentSpan?.span({
+        name: "page-content",
+    });
+    const pageContent = await page.content();
+    pageContentSpan?.end({ output: { pageContent } });
+    const sanitizationSpan = browsingAgentSpan?.span({
+        name: "page-sanitization",
+    });
+    const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
+    sanitizationSpan?.end({ output: { pageSnapshot } });
+    const promptSpan = browsingAgentSpan?.span({ name: "page-prompt" });
+    const messages = await (0, lib_1.compilePrompt)(promptTemplate_0, { pageSnapshot, task: action });
+    promptSpan?.end({ output: { messages } });
+    let completion;
+    completion = await (0, o1_completion_1.getO1Completion)({
+        messages,
+        tools,
+        trace: browsingAgentSpan,
+    });
+    // If O1 completion fails due to any reason, resort to old flow
+    if (!completion) {
+        completion = await llm.createChatCompletion({
             messages,
             tools,
             trace: browsingAgentSpan,
+            model: options.model || constants_1.DEFAULT_MODEL,
+            modelParameters: {
+                ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                ...options.modelParameters,
+                tool_choice: "required",
+            },
         });
-        // If O1 completion fails due to any reason, resort to old flow
-        if (!completion) {
-            completion = await llm.createChatCompletion({
-                messages,
-                tools,
-                trace: browsingAgentSpan,
-                model: options.model || constants_1.DEFAULT_MODEL,
-                modelParameters: {
-                    ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                    ...options.modelParameters,
-                    tool_choice: "required",
-                },
-            });
-        }
-        const toolCalls = completion?.tool_calls || [];
-        // LLM might respond with empty tool_calls and we can go into endless loop
-        // if we donot record this action and mark it as error
-        if (!toolCalls.length) {
-            executedActions.push({
-                isError: true,
-                action: "",
-            });
-        }
-        const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
-        for (const i in toolCalls) {
-            const toolCall = toolCalls[i];
-            if (await (0, session_1.shouldStopSession)()) {
-                break;
-            }
-            try {
-                const code = await actions.executeAction(toolCall.function.name, (0, utils_1.parseJson)(toolCall.function.arguments), toolCallsSpan);
-                if (code) {
-                    generatedCodeSteps.push(code);
-                }
-                executedActions.push({
-                    isError: false,
-                    action: (0, utils_1.parseJson)(toolCall.function.arguments)?.reason,
-                });
-                lastActionExecTrace = "";
-            }
-            catch (e) {
-                // TODO: implement feedback loop to llm
-                executedActions.push({
-                    isError: true,
-                    action: (0, utils_1.parseJson)(toolCall.function.arguments)?.reason,
-                });
-                lastActionExecTrace = e.message;
-                void testgenUpdatesReporter.sendMessage(e.message);
-                logger.error(lastActionExecTrace, e);
+    }
+    const toolCalls = completion?.tool_calls || [];
+    const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
+    for (const i in toolCalls) {
+        const toolCall = toolCalls[i];
+        try {
+            const code = await actions.executeAction(toolCall.function.name, (0, utils_1.parseJson)(toolCall.function.arguments), toolCallsSpan);
+            if (code) {
+                generatedCodeSteps.push(code);
             }
         }
-        toolCallsSpan?.end({ output: { toolCalls } });
-        // mark task as done if llm is stuck in loop
-        if (executedActions.length >= 3) {
-            const lastThreeActions = executedActions.slice(-3);
-            const lastThreeActionsFailed = lastThreeActions.every((a) => a.isError);
-            // get last 3 lines of code
-            const isStuckInLoop = actions.isStuckInLoop();
-            if (lastThreeActionsFailed || isStuckInLoop) {
-                // TODO: this should be sent to dashboard
-                const error = "Agent is not able to figure out next browser action, ending retries";
-                logger.error(error);
-                await testgenUpdatesReporter.sendMessage(error);
-                throw Error(error);
-            }
+        catch (e) {
+            void testgenUpdatesReporter.sendMessage(e.message);
         }
     }
+    toolCallsSpan?.end({ output: { toolCalls } });
     return generatedCodeSteps;
 }
 exports.executeTaskUsingBrowsingAgent = executeTaskUsingBrowsingAgent;

package/dist/agent/browsing/utils.d.ts

@@ -29,13 +29,6 @@ export declare function readPlaywrightConfig(): Promise<PlaywrightTestConfig>;
  * @returns
  */
 export declare function detectProjectName(testFilePath: string, playwrightConfig: PlaywrightTestConfig, pwProjectsFilter?: string[]): Promise<string>;
-export declare function getPromptForNextAction({ pageSnapshot, task, previousActions, lastActionErrors, promptType, }: {
-    pageSnapshot: string;
-    task: string;
-    previousActions: string[];
-    lastActionErrors: string[];
-    promptType?: string;
-}): Promise<import("openai/resources/index.mjs").ChatCompletionMessageParam[]>;
 export declare class TeardownManager {
     private directory;
     constructor(directory: string);

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuHxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuHxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TeardownManager = exports.getPromptForNextAction = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunMasterAgent = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
-const llm_1 = require("@empiricalrun/llm");
+exports.TeardownManager = exports.detectProjectName = exports.readPlaywrightConfig = exports.canRunMasterAgent = exports.injectPwLocatorGenerator = exports.prepareFileForMasterAgent = exports.prepareBrowsingAgentTask = exports.isRegExp = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const minimatch_1 = require("minimatch");
 const path_1 = __importDefault(require("path"));
@@ -348,17 +347,6 @@ async function detectProjectName(testFilePath, playwrightConfig, pwProjectsFilte
     return filteredProjectNames[0];
 }
 exports.detectProjectName = detectProjectName;
-async function getPromptForNextAction({ pageSnapshot = "", task = "", previousActions = [], lastActionErrors = [], promptType = "browsing-agent-next-action", }) {
-    const previousActionsStr = previousActions.join("\n\n ---- \n\n");
-    const prompt = await (0, llm_1.getPrompt)(promptType, {
-        pageSnapshot,
-        previousActionsStr,
-        task,
-        lastActionErrors,
-    });
-    return prompt;
-}
-exports.getPromptForNextAction = getPromptForNextAction;
 class TeardownManager {
     directory;
     constructor(directory) {

package/dist/agent/codegen/create-test-block.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAavE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAqDA"}
+{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAcvE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAqDA"}

package/dist/agent/codegen/create-test-block.js

@@ -7,8 +7,8 @@ const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write an empty test block.\nBased on the inputs you need to create an empty playwright test block with correctly imported fixture.\n\nThe test will contain a test name which you will need to use to build the empty test case block.\n\nYou will be provided with current tests, fixtures and page object models for you to use and create test case block as\nper the task provided to you.\n\nBefore responding you need to ensure that the code change is minimal and the change is reusable across tests. You need\nto ensure the code follows DRY principle.\n\nHere is the list of current tests and fixtures:\n\n{{testFiles}}\n\nHere is the list of current page object models:\n\n{{pageFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nFollowing is the test scenario for which you need to write the empty test case block:\ntest name:\n{{scenarioName}}\n\ntask:\ncreate an empty test case block for the following test steps:\n{{scenario}}\n\ntest file path: {{scenarioFile}}\n\n------\n\nYou also need to ensure that the empty test case block has a starting page to begin test.\n\nIn order to identify the right page with which the test should start, follow the steps:\n- based on the similarities with other test cases mentioned in the file, identify the right page fixture to be imported\n- Read the page fixture methods step by step. Identify whether the fixture handles navigating to a page.\n- Identify whether other tests using the page fixture had to add separate steps for navigation or not\n- Based on the above analysis there will be following cases and choose either for the given test scenario:\n-- Case 1: if the test case scenario provided inside the task mentions about page navigation, then use that page\nnavigation. skip other cases if this case is satisfied.\n-- Case 2: refer other test cases which import similar fixtures and infer the first page navigation of this test case.\nYou should prefer tests which are in the same file. Tests within same file have higher overlaps in first page\nnavigation.\n- Once the page fixture is decided, look for userContext fixture in files. If its available then add \"userContext\" to\nthe test case block\n\n\n\nFollow these instructions before responding with output:\n- Read the code line by line and achieve the task provided to you\n- Read the dependencies of the code block by scanning through file paths and file provided to you. refer the same file\npath while responding with update\n- Focus only on the test case provided and associated JS methods called from the test case.\n- Respond only with the new empty test case block to be created and nothing else.\n- DO NOT respond with any backticks or markdown syntax\n- If \"userContext\" fixture is available in fixtures file, ensure importing that fixture in the test case block.\n- Provide a reason based on the test steps provided to you on why you chose the fixture or page.goto statement. The\nreason should be one of the list steps provided to you and mention why the case was chosen\n{{/section}}";
+const lib_1 = require("../../prompts/lib");
 const session_1 = require("../../session");
-const promptBuilder_1 = require("./promptBuilder");
 async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
     const logger = new logger_1.CustomLogger({ useReporter: false });
     logger.log("Creating new test block");
@@ -29,7 +29,7 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
     const promptSpan = trace?.span({
         name: "build-create-empty-test-case-prompt",
     });
-    const prompt = await (0, promptBuilder_1.compilePrompt)(promptTemplate_0, {
+    const prompt = await (0, lib_1.compilePrompt)(promptTemplate_0, {
         testFiles: context.codePrompt,
         pageFiles: context.pomPrompt,
         scenarioName: testCase.name,

package/dist/agent/codegen/generate-code-apply-changes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate-code-apply-changes.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/generate-code-apply-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AASrE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAOhD,OAAO,EAAsB,UAAU,EAAsB,MAAM,SAAS,CAAC;AAqL7E,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,UAkD3D;AAgED,wBAAsB,2BAA2B,CAAC,EAChD,IAAI,EACJ,KAAK,EACL,MAAM,EACN,gBAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gBAAgB,EAAE,MAAM,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAyLxB"}
+{"version":3,"file":"generate-code-apply-changes.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/generate-code-apply-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQrE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAOhD,OAAO,EAAsB,UAAU,EAAE,MAAM,SAAS,CAAC;AAqLzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,UAkD3D;AA2DD,wBAAsB,2BAA2B,CAAC,EAChD,IAAI,EACJ,KAAK,EACL,MAAM,EACN,gBAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gBAAgB,EAAE,MAAM,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuIxB"}

package/dist/agent/codegen/generate-code-apply-changes.js

@@ -264,12 +264,11 @@ In order to execute the task, FOLLOW BELOW STEPS:
 }
 function deDupUpdatedFiles(updatedFiles) {
     return updatedFiles.filter((change, index, self) => index ===
-        self.findIndex((existing) => existing.filePath === change.filePath &&
-            existing.oldCode === change.oldCode &&
-            existing.newCode === change.newCode));
+        self.findIndex((existing) => existing.filePath === change.filePath));
 }
 async function generateCodeAndApplyChanges({ task, trace, logger, getRelevantFiles, }) {
-    let planRetries = 5;
+    // Reducing this from 5 to 3, if the required changes are getting missed will change it back.
+    let planRetries = 3;
     let updatedFiles = [];
     while (planRetries--) {
         const generateCodeAndApplyChangesSpan = trace?.span({
@@ -300,120 +299,81 @@ async function generateCodeAndApplyChanges({ task, trace, logger, getRelevantFil
             await (0, llm_1.flushAllTraces)();
             return deDupUpdatedFiles(updatedFiles);
         }
-        let strReplaceRetries = 3;
-        while (strReplaceRetries--) {
-            const promptForStrReplace = [
-                {
-                    role: "system",
-                    content: systemPromptBuilderForRepoEdit(files),
-                },
-                {
-                    role: "user",
-                    content: userPromptBuilderForStrReplace(strReplacePlan),
-                },
-            ];
-            const codeEditorSpan = generateCodeAndApplyChangesSpan?.span({
-                name: "code-editor-agent",
-                input: {
-                    prompt: promptForStrReplace,
-                },
-            });
-            const llm = new llm_1.LLM({
-                trace: codeEditorSpan,
-                provider: "anthropic",
-                defaultModel: "claude-3-5-sonnet-20240620",
-                providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
-            });
-            const completion = await llm.createChatCompletion({
-                messages: promptForStrReplace,
-                modelParameters: {
-                    ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                    temperature: 0.1,
-                    tool_choice: "required",
-                },
-                trace: codeEditorSpan,
-                tools: getCodeEditorToolCalls(),
-            });
-            codeEditorSpan?.end({ output: { completion } });
-            if (!completion?.tool_calls || completion?.tool_calls?.length === 0) {
-                break;
-            }
-            let codeEditorToolCalls = completion.tool_calls;
-            // Filter out the tool calls which are for creating new files
-            const createFileToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.CREATE_FILE);
-            if (createFileToolCalls.length > 0) {
-                console.log(`create_file tool calls: `, createFileToolCalls);
-            }
-            await Promise.all(createFileToolCalls.map((tc) => {
-                return (async () => {
-                    const args = (0, utils_1.parseJson)(tc.function.arguments);
-                    updatedFiles.push({
-                        filePath: args.filePath,
-                        oldCode: "",
-                        newCode: args.code,
-                        reason: args.reason,
-                    });
-                    await fs_extra_1.default.mkdir((0, path_1.dirname)(args.filePath), { recursive: true });
-                    await fs_extra_1.default.writeFile(args.filePath, args.code, "utf-8");
-                    console.log(`Created file: ${args.filePath}`);
-                })();
-            }));
-            const strReplaceToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.STR_REPLACE);
-            if (strReplaceToolCalls.length > 0) {
-                console.log(`str_replace tool calls: `, strReplaceToolCalls);
-            }
-            // Filter out the tool calls which are for replacing code in existing files
-            const fileChanges = strReplaceToolCalls
-                .map((toolCall) => (0, utils_1.parseJson)(toolCall.function.arguments))
-                .filter((f) => f.filePath && fs_extra_1.default.existsSync(f.filePath));
-            updatedFiles.push(...fileChanges);
-            let failedCodeUpdates;
-            // applyChangesResponse contains the errors occurred while applying the changes
-            failedCodeUpdates = await (0, utils_2.applyFileChangesUsingStrReplace)({
-                trace: codeEditorSpan,
-                fileChanges,
-                logger,
-            });
-            // Filter out the responses having errors
-            failedCodeUpdates = failedCodeUpdates.filter((f) => f?.error);
-            // Filter out the tool calls which have errors
-            const toolCallsWithErrors = codeEditorToolCalls.filter((toolCall) => {
-                const args = (0, utils_1.parseJson)(toolCall.function.arguments);
-                return failedCodeUpdates.find((response) => response.filePath === args.filePath);
-            });
-            if (failedCodeUpdates.length === 0) {
-                break;
-            }
-            logger?.log(`Failed to apply changes, retrying...`, failedCodeUpdates);
-            const feedback = failedCodeUpdates
-                .map((updates) => `For file ${updates.filePath}: ${updates.errorMessage}`)
-                .join("\n");
-            promptForStrReplace.push({
-                role: "assistant",
-                tool_calls: toolCallsWithErrors,
-            });
-            toolCallsWithErrors.forEach((toolCall) => {
-                promptForStrReplace.push({
-                    role: "tool",
-                    tool_call_id: toolCall.id,
-                    content: `
-Errors while executing the changes provided in above tool call:
-${feedback}
-
-Please fix the errors and return the updated code.
-
-FOLLOW BELOW STEPS TO FIX THE ISSUES:
-- First read the error message and understand the issue.
-- Go through the new code block and current file code, to figure out the root cause of the issue.
-- Compile the steps that you need to follow to fix the issue.
-- Check the test names, to ensure that the changes are applied to the correct test.
-- Use separate 'str_replace' tool to make the changes for each update.
-- Return the updated code in the same format as provided in the tool call.
-
-NOTE: ONLY MAKE THE CHANGES TO FIX THE ISSUES MENTIONED IN THE ERROR MESSAGE AND NOTHING ELSE. NO EXTRA CODE REFACTORING IS REQUIRED.
-    `,
+        const promptForStrReplace = [
+            {
+                role: "system",
+                content: systemPromptBuilderForRepoEdit(files),
+            },
+            {
+                role: "user",
+                content: userPromptBuilderForStrReplace(strReplacePlan),
+            },
+        ];
+        const codeEditorSpan = generateCodeAndApplyChangesSpan?.span({
+            name: "code-editor-agent",
+            input: {
+                prompt: promptForStrReplace,
+            },
+        });
+        const llm = new llm_1.LLM({
+            trace: codeEditorSpan,
+            provider: "anthropic",
+            defaultModel: "claude-3-5-sonnet-20240620",
+            providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
+        });
+        const completion = await llm.createChatCompletion({
+            messages: promptForStrReplace,
+            modelParameters: {
+                ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                temperature: 0.1,
+                tool_choice: "required",
+            },
+            trace: codeEditorSpan,
+            tools: getCodeEditorToolCalls(),
+        });
+        codeEditorSpan?.end({ output: { completion } });
+        if (!completion?.tool_calls || completion?.tool_calls?.length === 0) {
+            continue;
+        }
+        // Filter out the tool calls which are for creating new files
+        const createFileToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.CREATE_FILE);
+        if (createFileToolCalls.length > 0) {
+            console.log(`create_file tool calls: `, createFileToolCalls);
+        }
+        await Promise.all(createFileToolCalls.map((tc) => {
+            return (async () => {
+                const args = (0, utils_1.parseJson)(tc.function.arguments);
+                updatedFiles.push({
+                    filePath: args.filePath,
+                    oldCode: "",
+                    newCode: args.code,
+                    reason: args.reason,
                 });
-            });
+                await fs_extra_1.default.mkdir((0, path_1.dirname)(args.filePath), { recursive: true });
+                await fs_extra_1.default.writeFile(args.filePath, args.code, "utf-8");
+                console.log(`Created file: ${args.filePath}`);
+            })();
+        }));
+        const strReplaceToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.STR_REPLACE);
+        if (strReplaceToolCalls.length > 0) {
+            console.log(`str_replace tool calls: `, strReplaceToolCalls);
+        }
+        // Filter out the tool calls which are for replacing code in existing files
+        const fileChanges = strReplaceToolCalls
+            .map((toolCall) => (0, utils_1.parseJson)(toolCall.function.arguments))
+            .filter((f) => f.filePath && fs_extra_1.default.existsSync(f.filePath));
+        // We add all the suggested changes to the updatedFiles array
+        // This is used to validate and format files later
+        updatedFiles.push(...fileChanges);
+        // applyChangesResponse contains the errors occurred while applying the changes
+        const updates = await (0, utils_2.applyFileChangesUsingStrReplace)({
+            trace: codeEditorSpan,
+            fileChanges,
+            logger,
+        });
+        const failedCodeUpdates = updates.filter((f) => f?.error);
+        if (failedCodeUpdates.length > 0) {
+            logger?.log(`Failed to apply changes, retrying...`, failedCodeUpdates);
         }
     }
     return deDupUpdatedFiles(updatedFiles);

package/dist/agent/codegen/lexical-scoped-vars.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lexical-scoped-vars.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/lexical-scoped-vars.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAGnD,wBAAsB,oBAAoB,CAAC,EACzC,KAAK,EACL,IAAI,EACJ,cAAc,EACd,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,qBAoDA"}
+{"version":3,"file":"lexical-scoped-vars.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/lexical-scoped-vars.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AASrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,wBAAsB,oBAAoB,CAAC,EACzC,KAAK,EACL,IAAI,EACJ,cAAc,EACd,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,qBAoDA"}

package/dist/agent/codegen/lexical-scoped-vars.js

@@ -4,12 +4,12 @@ exports.getLexicalScopedVars = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software engineer tasked with analysing Typescript code to identify all variables available in the lexical\nscope at a specific reference point within a file. You will be given a file that contains multiple Playwright tests or\npage object models, along with a reference point inside the file. Your goal is to evaluate the list of all variables\navailable in the lexical scope at that reference point.\n\nTo accomplish this, you need to evaluate the Abstract Syntax Tree (AST) and accumulate all variables that are in the\nlexical scope, which includes:\n1. Variables declared within the test before the reference point.\n2. Arguments of the function.\n3. Variables defined in the parent scope. Identify all variables available in the lexical scope at a specific execution\nreference point within a file, considering only those variables that have been declared and assigned prior to the\nexecution of this point in the code.\n4. Global variables defined in the file.\n\nBefore responding:\n- Ignore variables imported from the `\"./pages\"` path.\n- keep in mind temporal dead zone phenomenon before responding with variables\n{{/section}}\n\n{{#section \"user\"}}\nFile:\n{{testFile}}\n\nReference point:\n{{referencePoint}}\n{{/section}}";
-const promptBuilder_1 = require("./promptBuilder");
+const lib_1 = require("../../prompts/lib");
 async function getLexicalScopedVars({ trace, file, referencePoint, options, }) {
     const fetchLexicalScopedVarsSpan = trace?.span({
         name: "lexical-scoped-vars",
     });
-    const messages = await (0, promptBuilder_1.compilePrompt)(promptTemplate_0, {
+    const messages = await (0, lib_1.compilePrompt)(promptTemplate_0, {
         testFile: file || "",
         referencePoint: referencePoint || "",
     });

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAgG9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}

package/dist/agent/master/element-annotation.js

@@ -2,8 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getAnnotationKeys = exports.getElementAnnotation = void 0;
 const llm_1 = require("@empiricalrun/llm");
-const vision_1 = require("@empiricalrun/llm/vision");
 const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements\npresent in the image.\n\nYou will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation\nis done by drawing a red box around the element and a small yellow box on it which contains unique element id.\n\nYou are given a Annotations which contains list of unique element id and description of the element separated by \":\".\n\nYou are also given the description of the element on which the action needs to be taken. The description includes\ninformation about how the element looks, it's position etc.\n\nYour task is to provide the annotation of the element on which the action needs to be performed based on the element\ndescription.\n\nFollow steps to fulfil your task:\n- Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each\nelement.\n- For describing each element Id\n-- iterate over each element Id in annotation list\n-- check if the description is already present for the element Id in the Annotation provided to you. If present skip\ndescribing it and use it as is.\n-- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or\nicon enclosed in the element.\n- Respond with the mapped element Ids as \"enriched_annotations\"\n- Based on the description provided to you and the enriched annotations, first identify the element Id whose description\nmatches the task provided\n\nNote:\n- Ensure providing the description of all the elements in the list.\n- Don't update the description if its already present in the given annotations\n- Replace all the \"NA\" with description of the element. Its position, how does it look like etc.\n- There should be no \"NA\" present in any of the element description\n{{/section}}\n\n{{#section \"user\"}}\nElement description:\n{{elementDescription}}\n\nAnnotations:\n{{annotations}}\n\n{{image annotatedScreenshot}}\n{{/section}}";
+const lib_1 = require("../../prompts/lib");
 const utils_1 = require("../utils");
 const annotationToolAction = {
     name: "element_annotation",
@@ -66,58 +67,11 @@ async function getElementAnnotation({ elementDescription, annotations, annotated
             preference,
         },
     });
-    const systemMessage = {
-        role: "system",
-        content: `You are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements present in the image.
-  
-  You will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation is done by drawing a red box around the element and a small yellow box on it which contains unique element id.
-  
-  You are given a Annotations which contains list of  unique element id  and description of the element separated by ":". 
-  
-  You are also given the description of the element on which the action needs to be taken. The description includes information about how the element looks, it's position etc.
-  
-  Your task is to provide the annotation of the element on which the action needs to be performed based on the element description.
-  
-  Follow steps to fulfil your task:
-  - Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each element. 
-  - For describing each element Id
-  -- iterate over each element Id in annotation list
-  -- check if the description is already present for the element Id in the Annotation provided to you. If present skip describing it and use it as is.
-  -- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or icon enclosed in the element. 
-  - Respond with the mapped element Ids as "enriched_annotations"
-  - Based on the description provided to you and the enriched annotations, first identify the element Id whose description matches the task provided
-  
-  Note:
-  - Ensure providing the description of all the elements in the list.
-  - Don't update the description if its already present in the given annotations
-  - Replace all the "NA" with description of the element. Its position, how does it look like etc. 
-  - There should be no "NA" present in any of the element description
-  `,
-    };
-    const userMessage = {
-        role: "user",
-        content: [
-            {
-                type: "text",
-                text: `
-  Element description:
-  ${elementDescription}
-  
-  Annotations:
-  ${annotations}`,
-            },
-            {
-                type: "image_url",
-                image_url: {
-                    url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, annotatedScreenshot),
-                },
-            },
-        ],
-    };
-    const messages = [
-        systemMessage,
-        userMessage,
-    ];
+    const messages = (0, lib_1.compilePrompt)(promptTemplate_0, {
+        elementDescription,
+        annotations,
+        annotatedScreenshot,
+    }, options);
     llm =
         llm ||
             new llm_1.LLM({

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA4BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA+UA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA4BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAoVA"}

package/dist/agent/master/run.js

@@ -150,11 +150,11 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
             const masterAgentActionSpan = masterAgentSpan?.span({
                 name: "master-agent-execute-action",
             });
+            output = {
+                action: args.action || args.skill,
+                reason: args.reason,
+            };
             try {
-                output = {
-                    action: args.action || args.skill,
-                    reason: args.reason,
-                };
                 void testGenReporter.sendMessage(output.reason);
                 logger.log(`Next Action: ${output.action}`);
                 if (toolCall.actionType === skill_1.SKILL_USAGE) {
@@ -254,7 +254,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                         generatedCodeSteps = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
                             trace: masterAgentActionSpan,
                             action: output.action,
-                            logger,
                             page,
                             options,
                             llm,
@@ -307,8 +306,8 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         trace?.update({ input: { task }, output: { output } });
         masterAgentSpan?.end({
             output: {
-                action: output.action,
-                reason: output.reason,
+                action: output?.action,
+                reason: output?.reason,
                 code: generatedCodeSteps,
             },
         });

package/dist/prompts/lib/index.d.ts

@@ -0,0 +1,8 @@
+import { LLMProvider } from "@empiricalrun/llm";
+import OpenAI from "openai";
+type PromptOptions = {
+    modelProvider?: LLMProvider;
+};
+export declare function compilePrompt<T extends object>(promptTemplate: string, params: T, options?: PromptOptions): OpenAI.Chat.Completions.ChatCompletionMessageParam[];
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/prompts/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/prompts/lib/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAyF5B,KAAK,aAAa,GAAG;IACnB,aAAa,CAAC,EAAE,WAAW,CAAC;CAC7B,CAAC;AAEF,wBAAgB,aAAa,CAAC,CAAC,SAAS,MAAM,EAC5C,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,CAAC,EACT,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAwCtD"}

package/dist/prompts/lib/index.js

@@ -0,0 +1,118 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compilePrompt = void 0;
+const vision_1 = require("@empiricalrun/llm/vision");
+const handlebars_1 = __importDefault(require("handlebars"));
+const constants_1 = require("../../constants");
+class SectionManager {
+    sections = {};
+    getSection(name) {
+        return this.sections[name] || "";
+    }
+    setSection(name, content) {
+        this.sections[name] = content;
+    }
+    getAllSections() {
+        return this.sections;
+    }
+}
+const IMAGE_TOKEN_PREFIX = "[[[HANDLEBARS_IMAGE:";
+const IMAGE_TOKEN_SUFFIX = "]]]";
+function createHandlebarsEnv() {
+    const HandlebarsEnv = handlebars_1.default.create();
+    const sectionManager = new SectionManager();
+    HandlebarsEnv.registerHelper("section", function (name, options) {
+        const content = options.fn(this);
+        sectionManager.setSection(name, content);
+        return ""; // don't output anything in place
+    });
+    HandlebarsEnv.registerHelper("image", function (imageParam) {
+        const tokenPayload = JSON.stringify({ url: imageParam });
+        // Use encodeURIComponent to avoid conflicts with special characters.
+        const token = `${IMAGE_TOKEN_PREFIX}${encodeURIComponent(tokenPayload)}${IMAGE_TOKEN_SUFFIX}`;
+        return token;
+    });
+    HandlebarsEnv.registerHelper("images", function (imagesParam) {
+        if (!Array.isArray(imagesParam))
+            return "";
+        return imagesParam
+            .map((url) => {
+            const tokenPayload = JSON.stringify({ url });
+            return `${IMAGE_TOKEN_PREFIX}${encodeURIComponent(tokenPayload)}${IMAGE_TOKEN_SUFFIX}`;
+        })
+            .join("");
+    });
+    return { HandlebarsEnv, sectionManager };
+}
+function processSectionContent(content) {
+    if (!content.includes(IMAGE_TOKEN_PREFIX)) {
+        return content.trim();
+    }
+    const segments = [];
+    const regex = /\[\[\[HANDLEBARS_IMAGE:(.*?)\]\]\]/g;
+    let lastIndex = 0;
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+        // Get the text before the token.
+        const textPart = content.slice(lastIndex, match.index).trim();
+        if (textPart) {
+            segments.push({ type: "text", text: textPart });
+        }
+        // Decode the token payload.
+        try {
+            const payloadJson = decodeURIComponent(match[1]);
+            const payload = JSON.parse(payloadJson);
+            segments.push({ type: "image_url", image_url: { url: payload.url } });
+        }
+        catch (err) {
+            // If decoding/parsing fails, treat the token as literal text.
+            segments.push({ type: "text", text: match[0] });
+        }
+        lastIndex = match.index + match[0].length;
+    }
+    const remaining = content.slice(lastIndex).trim();
+    if (remaining) {
+        segments.push({ type: "text", text: remaining });
+    }
+    return segments;
+}
+function compilePrompt(promptTemplate, params, options) {
+    const { HandlebarsEnv, sectionManager } = createHandlebarsEnv();
+    const template = HandlebarsEnv.compile(promptTemplate, { noEscape: true });
+    template(params);
+    const sections = sectionManager.getAllSections();
+    // TODO: system cannot have images, we can add validation for that
+    const system = sections["system"];
+    const user = sections["user"];
+    if (!system || !user) {
+        // TODO: support templates that have only one section
+        throw new Error("Both system and user sections must be defined in the template");
+    }
+    const systemContent = processSectionContent(system);
+    const userContent = processSectionContent(user);
+    let userContentCorrectedForImageFormat = userContent;
+    if (Array.isArray(userContent)) {
+        const provider = options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER;
+        userContentCorrectedForImageFormat = userContent.map((c) => {
+            if (c.type === "image_url") {
+                return {
+                    ...c,
+                    image_url: {
+                        url: (0, vision_1.imageFormatForProvider)(provider, c.image_url.url),
+                    },
+                };
+            }
+            else {
+                return c;
+            }
+        });
+    }
+    return [
+        { role: "system", content: systemContent },
+        { role: "user", content: userContentCorrectedForImageFormat },
+    ];
+}
+exports.compilePrompt = compilePrompt;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.42.17",
+  "version": "0.42.19",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -72,7 +72,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.31",
+    "@empiricalrun/llm": "^0.9.32",
     "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/reporter": "^0.23.1"
   },

package/dist/agent/codegen/promptBuilder.d.ts

@@ -1,3 +0,0 @@
-import OpenAI from "openai";
-export declare function compilePrompt<T extends object>(promptTemplate: string, params: T): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
-//# sourceMappingURL=promptBuilder.d.ts.map

package/dist/agent/codegen/promptBuilder.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"promptBuilder.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/promptBuilder.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AA6B5B,wBAAsB,aAAa,CAAC,CAAC,SAAS,MAAM,EAClD,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,CAAC,GACR,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC,CAe/D"}

package/dist/agent/codegen/promptBuilder.js

@@ -1,44 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.compilePrompt = void 0;
-const handlebars_1 = __importDefault(require("handlebars"));
-class SectionManager {
-    sections = {};
-    getSection(name) {
-        return this.sections[name] || "";
-    }
-    setSection(name, content) {
-        this.sections[name] = content;
-    }
-    getAllSections() {
-        return this.sections;
-    }
-}
-function createHandlebarsEnv() {
-    const HandlebarsEnv = handlebars_1.default.create();
-    const sectionManager = new SectionManager();
-    HandlebarsEnv.registerHelper("section", function (name, options) {
-        const content = options.fn(this);
-        sectionManager.setSection(name, content);
-        return ""; // Don't output anything in place
-    });
-    return { HandlebarsEnv, sectionManager };
-}
-async function compilePrompt(promptTemplate, params) {
-    const { HandlebarsEnv, sectionManager } = createHandlebarsEnv();
-    const template = HandlebarsEnv.compile(promptTemplate, { noEscape: true });
-    template(params);
-    const { system, user } = sectionManager.getAllSections();
-    if (!system || !user) {
-        // TODO: support templates that have only one section
-        throw new Error("Both system and user sections must be defined in the template");
-    }
-    return [
-        { role: "system", content: system },
-        { role: "user", content: user },
-    ];
-}
-exports.compilePrompt = compilePrompt;

package/dist/agent/verification/index.d.ts

@@ -1,13 +0,0 @@
-import { TraceClient } from "@empiricalrun/llm";
-/**
- * This agent is used to verify whether the task is done basis the conversation history
- */
-export declare function verificationAgent({ trace, task, conversation, }: {
-    trace?: TraceClient;
-    conversation: string[];
-    task: string;
-}): Promise<{
-    isDone: boolean;
-    reason: string;
-}>;
-//# sourceMappingURL=index.d.ts.map

package/dist/agent/verification/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA+EA"}

package/dist/agent/verification/index.js

@@ -1,84 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.verificationAgent = void 0;
-const llm_1 = require("@empiricalrun/llm");
-const utils_1 = require("../utils");
-/**
- * This agent is used to verify whether the task is done basis the conversation history
- */
-async function verificationAgent({ trace, task, conversation, }) {
-    const verificationAgentSpan = trace?.span({
-        name: "verification-agent",
-        input: {
-            task,
-            conversation,
-        },
-    });
-    const messages = await (0, llm_1.getPrompt)("agent-steps-verification", {
-        task,
-        conversation: conversation.join("\n"),
-    }, 5);
-    const llm = new llm_1.LLM({ provider: "openai" });
-    const response = await llm.createChatCompletion({
-        trace: verificationAgentSpan,
-        traceName: "verification-agent-llm",
-        model: "gpt-4o",
-        messages,
-        tools: [
-            {
-                type: "function",
-                function: {
-                    name: "task_done",
-                    description: "end the task by calling this method",
-                    parameters: {
-                        type: "object",
-                        properties: {
-                            actions: {
-                                type: "string",
-                                description: "actions extracted from task",
-                            },
-                            successful_actions: {
-                                type: "string",
-                                description: "successful actions mentioned in the conversation",
-                            },
-                            reason: {
-                                type: "string",
-                                description: "reasoning for identification of task status",
-                            },
-                            isDone: {
-                                type: "boolean",
-                                description: "whether the task is done",
-                            },
-                        },
-                        required: ["isDone", "reason"],
-                    },
-                },
-            },
-        ],
-        modelParameters: {
-            tool_choice: "required",
-            temperature: 0.5,
-        },
-    });
-    const toolCallResp = (response?.tool_calls || [])[0];
-    if (toolCallResp) {
-        const toolCall = (0, utils_1.parseJson)(toolCallResp.function.arguments);
-        const output = {
-            isDone: toolCall.isDone,
-            reason: toolCall.reason,
-        };
-        verificationAgentSpan?.end({
-            output,
-        });
-        return output;
-    }
-    const output = {
-        isDone: false,
-        reason: "LLM failed to generate a valid response",
-    };
-    verificationAgentSpan?.end({
-        output,
-    });
-    return output;
-}
-exports.verificationAgent = verificationAgent;

package/dist/evals/verification-agent.evals.d.ts

@@ -1,4 +0,0 @@
-import { EvaluateFn } from "./type";
-export declare const verifierAgentEvaluate: EvaluateFn;
-export default verifierAgentEvaluate;
-//# sourceMappingURL=verification-agent.evals.d.ts.map

package/dist/evals/verification-agent.evals.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"verification-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/verification-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,qBAAqB,EAAE,UAgBnC,CAAC;AAEF,eAAe,qBAAqB,CAAC"}

package/dist/evals/verification-agent.evals.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.verifierAgentEvaluate = void 0;
-const verification_1 = require("../agent/verification");
-const verifierAgentEvaluate = async ({ item, trace }) => {
-    const { conversation = [], task = "" } = item.input;
-    const output = await (0, verification_1.verificationAgent)({
-        conversation,
-        trace,
-        task,
-    });
-    return {
-        scores: [
-            {
-                name: "equality",
-                value: item.expectedOutput.isDone === output.isDone ? 1 : 0,
-            },
-        ],
-        output,
-    };
-};
-exports.verifierAgentEvaluate = verifierAgentEvaluate;
-exports.default = exports.verifierAgentEvaluate;