@empiricalrun/test-gen 0.46.7 → 0.46.9

package/dist/agent/master/browser-tests/skills.spec.js

@@ -0,0 +1,109 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const run_1 = require("../run");
+const fixtures_1 = require("./fixtures");
+fixtures_1.test.describe.configure({ mode: "default" });
+fixtures_1.test.beforeEach(async () => {
+    // Setup repo with POM file
+    fs_1.default.mkdirSync("pages");
+    const pomContent = `
+import type { Page } from "@playwright/test";
+
+export async function subscribeToBlog({ page, email }: { page: Page; email: string }) {
+  await page.getByPlaceholder("Enter your email").fill(email);
+  await page.getByRole("button", { name: "Subscribe" }).click();
+}
+
+export async function extractTitleForPost({ page, nth }: { page: Page; nth: number }) {
+  // nth is zero-indexed
+  const title = await page.locator("h2").nth(nth).textContent();
+  return title;
+}
+`;
+    fs_1.default.writeFileSync("pages/blog.ts", pomContent);
+});
+fixtures_1.test.afterEach(async () => {
+    // Clean up the repo in cwd
+    fs_1.default.rmSync("pages", { recursive: true });
+});
+(0, fixtures_1.test)("use skills to subscribe to blog", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `subscribe as user@example.com`,
+        page,
+        testCase: {
+            id: 1,
+            name: "subscribe to blog",
+            steps: ["subscribe as user@example.com"],
+            filePath: "blog.spec.ts",
+            suites: [],
+        },
+        options: {},
+    });
+    // Assert execution was successful
+    await (0, fixtures_1.expect)(page.getByText("Thanks for subscribing")).toBeVisible();
+    // Assert code generated is correct
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.importPaths[0]).toBe("pages/blog.ts");
+    (0, fixtures_1.expect)(response.code.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.code).toMatch(/await.*subscribeToBlog.*page.*email.*user@example\.com/);
+});
+(0, fixtures_1.test)("use skills to extract blog post title", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        // TODO: Extend this to click on "read more" and verify generated code
+        // uses the state variable created by the skill
+        task: `extract title of 2nd blog post`,
+        page,
+        testCase: {
+            id: 1,
+            name: "extract blog post title",
+            steps: ["extract title of 2nd blog post"],
+            filePath: "blog.spec.ts",
+            suites: [],
+        },
+        options: {},
+    });
+    // Assert code generated is correct
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.importPaths[0]).toBe("pages/blog.ts");
+    (0, fixtures_1.expect)(response.code.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.code).toMatch(/^const.*=.*await extractTitleForPost.*page.*nth:.*1/);
+});
+(0, fixtures_1.test)("use skills to subscribe with multiple pages", async ({ page, server, }) => {
+    await page.goto(`${server.baseURL}/icons-navbar.html`);
+    const blogPage = await page.context().newPage();
+    await blogPage.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `we have 2 pages open inside a web browser, and your task is to subscribe as user@example.com on the blogPage`,
+        page,
+        testCase: {
+            id: 1,
+            name: "subscribe to blog",
+            steps: [
+                "we have 2 pages open inside a web browser, and your task is to subscribe as user@example.com on the blogPage",
+            ],
+            filePath: "blog.spec.ts",
+            suites: [],
+        },
+        scopeVars: {
+            page,
+            blogPage,
+        },
+        options: {},
+    });
+    // Assert execution was successful
+    await (0, fixtures_1.expect)(blogPage.getByText("Thanks for subscribing")).toBeVisible();
+    // Assert code generated is correct
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.importPaths[0]).toBe("pages/blog.ts");
+    (0, fixtures_1.expect)(response.code.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.code).toMatch(/await.*subscribeToBlog.*blogPage.*email.*user@example\.com/);
+});

package/dist/agent/master/element-annotation.d.ts

@@ -1,6 +1,6 @@
 import { LLM, TraceClient } from "@empiricalrun/llm";
+import { TestGenConfigOptions } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
-import { BrowsingAgentOptions } from "../browsing";
 import { ActionType } from "./action-tool-calls";
 export declare function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, preference, }: {
     elementDescription: string;
@@ -8,17 +8,16 @@ export declare function getElementAnnotation({ elementDescription, annotations,
     annotatedScreenshot: string;
     trace?: TraceClient;
     llm?: LLM;
-    options?: BrowsingAgentOptions;
+    options?: TestGenConfigOptions;
     preference: AnnotationPreference;
 }): Promise<string | undefined>;
 export type AnnotationPreference = {
     actionType: "all" | ActionType.FILL | ActionType.ASSERT_TEXT | ActionType.SCROLL;
     assertionText?: string | undefined;
 };
-export declare function getAnnotationKeys({ page, preference, options, trace, }: {
+export declare function getAnnotationKeys({ page, preference, trace, }: {
     page: Page;
     preference: AnnotationPreference;
-    options: BrowsingAgentOptions;
     trace?: TraceClient;
 }): Promise<{
     annotationKeys: {

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA2DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAgBF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAoFD"}
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA2DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAgBF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAoFD"}

package/dist/agent/master/element-annotation.js

@@ -100,7 +100,7 @@ async function getElementAnnotation({ elementDescription, annotations, annotated
     return;
 }
 exports.getElementAnnotation = getElementAnnotation;
-async function getAnnotationKeys({ page, preference, options, trace, }) {
+async function getAnnotationKeys({ page, preference, trace, }) {
     const annotatedElements = await page.evaluate(({ preference, options }) => {
         // @ts-ignore
         // eslint-disable-next-line no-undef
@@ -139,7 +139,7 @@ async function getAnnotationKeys({ page, preference, options, trace, }) {
         });
         const fullPageHTML = document.documentElement.outerHTML;
         return { annotations, fullPageHTML };
-    }, { preference, options });
+    }, { preference, options: {} });
     const annotationKeys = await enrichAnnotations(annotatedElements.annotations, annotatedElements.fullPageHTML, trace);
     const annotationBuffer = await page.screenshot({
     // path: `screenshots/screenshot-${screenshotIndex++}.png`,

package/dist/agent/master/execute-browser-action.d.ts

@@ -0,0 +1,19 @@
+import { LLM, TraceClient } from "@empiricalrun/llm";
+import type { Page } from "@playwright/test";
+import { PlaywrightActions } from "../../actions";
+export declare function executeBrowserAction({ page, nextAction, flags, actions, llm, trace, }: {
+    page: Page;
+    nextAction: {
+        actionType: string;
+        toolCallArgs: string;
+    };
+    flags: {
+        useActionSpecificAnnotations: boolean;
+    };
+    actions: PlaywrightActions;
+    trace: TraceClient | undefined;
+    llm: LLM;
+}): Promise<{
+    generatedCodeSteps: string[];
+}>;
+//# sourceMappingURL=execute-browser-action.d.ts.map

package/dist/agent/master/execute-browser-action.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,GAAG,EAAE,GAAG,CAAC;CACV;;GAsIA"}

package/dist/agent/master/execute-browser-action.js

@@ -0,0 +1,123 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeBrowserAction = void 0;
+const reporter_1 = require("../../reporter");
+const browsing_1 = require("../browsing");
+const utils_1 = require("../utils");
+const action_tool_calls_1 = require("./action-tool-calls");
+const element_annotation_1 = require("./element-annotation");
+const with_hints_1 = require("./with-hints");
+async function executeBrowserAction({ page, nextAction, flags, actions, llm, trace, }) {
+    const args = (0, utils_1.parseJson)(nextAction.toolCallArgs);
+    let generatedCodeSteps = [];
+    let output = {
+        action: args.action,
+        reason: args.reason,
+    };
+    let shouldTriggerHintsFlow;
+    let hintsExecutionCompletion;
+    let elementAnnotation;
+    const actionType = nextAction.actionType;
+    let preference = {
+        actionType: "all",
+    };
+    if (flags.useActionSpecificAnnotations && (0, action_tool_calls_1.isValidActionType)(actionType)) {
+        switch (actionType) {
+            case action_tool_calls_1.ActionType.FILL:
+                preference = {
+                    actionType: action_tool_calls_1.ActionType.FILL,
+                };
+                break;
+            case action_tool_calls_1.ActionType.ASSERT_TEXT:
+                preference = {
+                    actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
+                    assertionText: args.assertion_text,
+                };
+                break;
+            default:
+                preference = {
+                    actionType: "all",
+                };
+        }
+    }
+    let { annotationKeys, annotatedPageScreenshot, annotationBuffer } = await (0, element_annotation_1.getAnnotationKeys)({
+        page,
+        preference,
+        trace,
+    });
+    if (annotationKeys.length > 0) {
+        // TODO: this string has newline characters that makes it harder to read
+        const annotationMapString = annotationKeys
+            ?.map((a) => `${a.elementID}: ${a.text}`)
+            .join("\n");
+        // Provides the annotations for all the element present on screen
+        // Also provides the annotation of element on which the action needs to be taken
+        elementAnnotation = await (0, element_annotation_1.getElementAnnotation)({
+            elementDescription: args.element_description,
+            annotations: annotationMapString,
+            annotatedScreenshot: annotatedPageScreenshot,
+            trace,
+            llm,
+            preference,
+        });
+        output.elementAnnotation = elementAnnotation;
+        console.log("Output: ", output);
+        const testGenReporter = new reporter_1.TestGenUpdatesReporter();
+        await testGenReporter.sendCurrentView(annotationBuffer);
+        const triggerHintsFlowSpan = trace?.span({
+            name: "trigger-hints-flow",
+            input: {
+                outputFromGetNextAction: output,
+                generatedAnnotations: annotationKeys,
+            },
+        });
+        // Provides the action whether its a click, fill etc.
+        const result = await (0, with_hints_1.triggerHintsFlow)({
+            outputFromGetNextAction: output,
+            generatedAnnotations: annotationKeys,
+            actions,
+            llm,
+            trace: triggerHintsFlowSpan,
+        });
+        shouldTriggerHintsFlow = result.shouldTriggerHintsFlow;
+        hintsExecutionCompletion = result.hintsExecutionCompletion;
+        triggerHintsFlowSpan?.end({
+            output: result,
+        });
+    }
+    if (shouldTriggerHintsFlow && hintsExecutionCompletion) {
+        const toolCalls = hintsExecutionCompletion?.tool_calls || [];
+        for (const i in toolCalls) {
+            const currentToolCall = toolCalls[i];
+            const code = await actions.executeAction(currentToolCall.function.name, {
+                ...JSON.parse(currentToolCall.function.arguments),
+                elementAnnotation,
+            }, trace);
+            if (code) {
+                generatedCodeSteps.push(code);
+            }
+        }
+        if (actions.isStuckInLoop()) {
+            throw new Error("Agent is not able to figure out next action when using hints");
+        }
+    }
+    else {
+        const browserAction = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
+            trace,
+            action: output.action,
+            page,
+            llm,
+            actions,
+        });
+        if (browserAction) {
+            output.action = browserAction.action;
+            if (browserAction.code) {
+                generatedCodeSteps.push(browserAction.code);
+            }
+        }
+    }
+    return {
+        generatedCodeSteps,
+    };
+}
+exports.executeBrowserAction = executeBrowserAction;

package/dist/agent/master/execute-skill-action.d.ts

@@ -0,0 +1,11 @@
+import { TraceClient } from "@empiricalrun/llm";
+import { PlaywrightActions } from "../../actions";
+export declare function executeSkillAction({ actions, nextAction, trace, }: {
+    actions: PlaywrightActions;
+    nextAction: {
+        actionType: string;
+        toolCallArgs: string;
+    };
+    trace: TraceClient | undefined;
+}): Promise<string | undefined>;
+//# sourceMappingURL=execute-skill-action.d.ts.map

package/dist/agent/master/execute-skill-action.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute-skill-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-skill-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,wBAAsB,kBAAkB,CAAC,EACvC,OAAO,EACP,UAAU,EACV,KAAK,GACN,EAAE;IACD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;CAChC,+BAeA"}

package/dist/agent/master/execute-skill-action.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeSkillAction = void 0;
+const skill_1 = require("../../actions/skill");
+const errors_1 = require("../../errors");
+const human_in_the_loop_1 = require("../../human-in-the-loop");
+const env_1 = require("../../utils/env");
+const utils_1 = require("../utils");
+async function executeSkillAction({ actions, nextAction, trace, }) {
+    const args = (0, utils_1.parseJson)(nextAction.toolCallArgs);
+    if (!(0, env_1.isRunningOnCloud)()) {
+        const { skillDetails } = (0, skill_1.extractSkillFromArgs)(args);
+        const feedback = await human_in_the_loop_1.humanLoop.getFeedback({
+            message: `Should I use this skill?\n\n${skillDetails?.usageExample}\n\nfrom: ${skillDetails?.filePath}\n\nAnswer with "yes" or "no"`,
+        });
+        if (feedback.toLowerCase().trim().includes("no")) {
+            throw new errors_1.HumanApprovalDenied("Skill usage approval not provided");
+        }
+    }
+    const code = await actions.executeAction(nextAction.actionType, args, trace);
+    if (code) {
+        return code;
+    }
+}
+exports.executeSkillAction = executeSkillAction;

package/dist/agent/master/next-action.d.ts

@@ -1,22 +1,19 @@
 import { LLM, TraceClient } from "@empiricalrun/llm";
+import { TestGenConfigOptions } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
 import { PlaywrightActions } from "../../actions";
 import { CustomLogger } from "../../bin/logger";
-import { BrowsingAgentOptions } from "../browsing";
-import { ActionType } from "./action-tool-calls";
-export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, actionTypes, disableSkills, page, logger, }: {
+export declare function getNextAction({ page, pageScreenshot, task, executedActions, failedActions, trace, llm, options, actions, disableSkills, logger, }: {
+    page: Page;
+    pageScreenshot: string[];
     task: string;
     executedActions: string[];
     failedActions: any[];
-    pageUrl: string;
     trace?: TraceClient;
     llm?: LLM;
-    options?: BrowsingAgentOptions;
-    pageScreenshot: string[];
+    options?: Partial<TestGenConfigOptions>;
     actions: PlaywrightActions;
-    actionTypes: typeof ActionType;
     disableSkills: boolean;
-    page: Page;
     logger?: CustomLogger;
 }): Promise<{
     actionType: string;

package/dist/agent/master/next-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAsB,MAAM,qBAAqB,CAAC;AAGrE,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,WAAW,EACX,aAAa,EACb,IAAI,EACJ,MAAM,GACP,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,EAAE,OAAO,UAAU,CAAC;IAC/B,aAAa,EAAE,OAAO,CAAC;IACvB,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CACP;IACE,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,GACD,SAAS,CACZ,CAuLA"}
+{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAWhD,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,cAAc,EACd,IAAI,EACJ,eAAe,EACf,aAAa,EACb,KAAK,EACL,GAAG,EACH,OAAO,EACP,OAAO,EACP,aAAa,EACb,MAAM,GACP,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACxC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CACP;IACE,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,GACD,SAAS,CACZ,CA8FA"}

package/dist/agent/master/next-action.js

@@ -2,13 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getNextAction = void 0;
 const llm_1 = require("@empiricalrun/llm");
-const vision_1 = require("@empiricalrun/llm/vision");
 const skill_1 = require("../../actions/skill");
 const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a web automation tool which is given a task to complete. You need to execute the\ntask provided to you with the help of web page screenshot, a browser automation tool or skills\nwhich are learnt while writing previous tests. \n  \nBrowser automation tool is a tool which uses Playwright and browser to execute action using\nnext_action tool call.\nSkill usage is a tool which helps to execute previously known pieces of code to achieve a task.\n\nYou will be provided with a screenshot of the webpage which you will use to extract the action\nthat needs to be taken.\n\nYou will be provided with previously executed actions by the browser automation tool and based\non the current screenshot and previously executed actions, you need to predict the next action\nto be taken.\n\nYou will also be provided with failed next action predicted by you, so that you can avoid\nsuggesting the same action again - which failed.\n\nThe next action should be as atomic as possible. e.g: scroll, click on an element, fill an input\nelement, assert, extract text from an element are valid next action as they are atomic in nature.\n\nYou also need to provide the action type using the list below, action type which is not present in\nthe list is invalid: {{validActionTypes}}\n\nYou will also be provided with skill usage tool which you can use to execute action. These skills\nare compound functions which helps you to complete your action.\n\nYou need to respond with either:\n- Next action to be taken by a browser automation tool \n- Use previously learnt skills in the form of tool call.\n  \nYou need to make a decision whether the given skill can be reused if \"YES\" respond with the\nskill else respond with the next action.\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\n-----\n\nPrevious executed actions:\n{{executedActions}}\n\n-----\n\nPrevious failed actions:\n{{failedActions}}\n\n-----\n\nYou are also provided with a page screenshot for you to decide the next action.\n\nCurrent page URL: {{pageUrl}}\n\nFollow the instructions before responding:\n- Divide the task into sub tasks\n- Using previously executed actions, identify tasks are complete and which tasks needs to be executed next.\n- You will be provided a skill usage action, if the testStep matches the next action then respond with the skill usage.\n- If responding with next action, ensure next action to be detailed and explicit about what action needs to be done. Provide all the information which can be extracted from the screenshot as a part of next action.\n- Mark task as complete only when executed actions provided to you indicates that the task is done.\n- Refer to the text and references available in the screenshot to create the next action.\n- Do not take any extra actions which are not required for the execution of the task\n- If there are no further actions required based on the task, then respond with task as done.\n- Do not recommend actions which are not available in the screenshot\n\nScreenshots:\n{{images pageScreenshots}}\n\n{{/section}}\n";
 const utils_1 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
 const scroller_1 = require("./scroller");
-async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, actionTypes, disableSkills, page, logger, }) {
+async function getNextAction({ page, pageScreenshot, task, executedActions, failedActions, trace, llm, options, actions, disableSkills, logger, }) {
+    const pageUrl = page.url();
     const nextActionSpan = trace?.span({
         name: "master-agent-next-action",
         input: {
@@ -22,96 +23,18 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
             skills: skill_1.testCaseSkills.getAvailableSkills(),
         },
     });
-    const promptSpan = nextActionSpan?.span({
-        name: "master-agent-prompt",
+    const messages = (0, llm_1.compilePrompt)(promptTemplate_0, {
+        validActionTypes: Object.values(action_tool_calls_1.ActionType).join(", "),
+        task,
+        executedActions: executedActions.map((a) => a).join("\n"),
+        failedActions: failedActions.map((a) => a).join("\n"),
+        pageUrl,
+        pageScreenshots: pageScreenshot,
     });
-    const screenshotsForPrompt = pageScreenshot.map((screenshot) => {
-        return {
-            type: "image_url",
-            image_url: {
-                url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, screenshot),
-            },
-        };
-    });
-    const systemMessage = {
-        role: "system",
-        content: `You are a web automation tool which is given a task to complete. You need to execute the task provided to you with the help of web page screenshot, a browser automation tool or skills which are learnt while writing previous tests. 
-  
-  Browser automation tool is a tool which uses Playwright and browser to execute action using next_action tool call.
-  Skill usage is a tool which helps to execute previously known pieces of code to achieve a task.
-  
-  You will be provided with a screenshot of the webpage which you will use to extract the action that needs to be taken.
-  
-  You will be provided with previously executed actions by the browser automation tool and based on the current screenshot and previously executed actions, you need to predict the next action to be taken.
-  
-  You will also be provided with failed next action predicted by you, so that you can avoid suggesting the same action again - which failed.
-  
-  The next action should be as atomic as possible.
-  e.g: scroll, click on an element, fill an input element, assert, extract text from an element are valid next action as they are atomic in nature.
-
-  You also need to provide the action type using the list below, action type which is not present in the list is invalid.
-  ${Object.values(actionTypes)}
-  
-  You will also be provided with skill usage tool which you can use to execute action. These skills are compound functions which helps you to complete your action.
-  
-  You need to respond with either:
-  - Next action to be taken by a browser automation tool 
-  - Use previously learnt skills in the form of tool call.
-  
-  You need to make a decision whether the given skill can be reused if "YES" respond with the skill else respond with the next action.`,
-    };
-    const userMessage = {
-        role: "user",
-        content: [
-            {
-                type: "text",
-                text: `Task:
-          ${task}
-          
-          ----
-          
-          Previous executed actions:
-          ${executedActions.map((a) => a).join("\n")}
-          
-          ----
-          
-          Previous failed actions:
-          ${failedActions.map((a) => a).join("\n")}
-          
-          ----
-          You are also provided with a page screenshot for you to decide the next action.
-          
-          Current page URL: ${pageUrl}
-          
-          
-          Follow the instructions before responding:
-          - Divide the task into sub tasks
-          - Using previously executed actions, identify tasks are complete and which tasks needs to be executed next.
-          - You will be provided a skill usage action, if the testStep matches the next action then respond with the skill usage.
-          - If responding with next action, ensure next action to be detailed and explicit about what action needs to be done. Provide all the information which can be extracted from the screenshot as a part of next action.
-          - Mark task as complete only when executed actions provided to you indicates that the task is done.
-          - Refer to the text and references available in the screenshot to create the next action.
-          - Do not take any extra actions which are not required for the execution of the task
-          - If there are no further actions required based on the task, then respond with task as done.
-          - Do not recommend actions which are not available in the screenshot
-          `,
-            },
-            {
-                type: "text",
-                text: pageScreenshot.length > 1 ? "Screenshots:" : "Screenshot:",
-            },
-            ...screenshotsForPrompt,
-        ],
-    };
-    const messages = [
-        systemMessage,
-        userMessage,
-    ];
     const actionSchemas = disableSkills || skill_1.testCaseSkills.getAvailableSkills().length === 0
         ? []
         : actions.getMasterActionSchemas();
     const tools = [...(0, action_tool_calls_1.getActionToolCalls)(), ...actionSchemas];
-    promptSpan?.end({ output: { messages } });
     llm =
         llm ||
             new llm_1.LLM({
@@ -154,13 +77,11 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
                         task,
                         executedActions,
                         failedActions,
-                        pageUrl,
                         trace,
                         llm,
                         options,
                         pageScreenshot: frames.map((frame) => frame.frameScreenshot),
                         actions,
-                        actionTypes,
                         disableSkills,
                         page,
                         logger,
@@ -169,7 +90,7 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
             }
             if (maxScrollRetries === -1) {
                 return {
-                    actionType: "unknown",
+                    actionType: action_tool_calls_1.ActionType.UNKNOWN,
                     toolCallArgs: "",
                 };
             }
@@ -179,6 +100,5 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
             toolCallArgs: toolCall?.function.arguments,
         };
     }
-    return;
 }
 exports.getNextAction = getNextAction;

package/dist/agent/master/run.d.ts

@@ -1,13 +1,12 @@
-import type { TestCase } from "@empiricalrun/shared-types";
+import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
 import { ScopeVars } from "../../types";
-import { BrowsingAgentOptions } from "../browsing";
 export declare function createTestUsingMasterAgent({ task, page, testCase, specPath, options, scopeVars, }: {
     task: string;
     page: Page;
     testCase?: TestCase;
     specPath?: string;
-    options: BrowsingAgentOptions;
+    options: Partial<TestGenConfigOptions>;
     scopeVars?: ScopeVars;
 }): Promise<{
     code: string;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAqBlC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAmYA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAgCxC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAoRA"}