@empiricalrun/playwright-utils 0.22.7 → 0.23.0

package/bundled/test-gen/agent/enrich-prompt/index.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.enrichPromptWithFailingLine = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software QA engineer who is asked to update an automated browser test written that failed during a test run.\n\nYou will be provided with the details of the failure and then a user prompt that details what needs to be changed.\n\n# Details of the failure\n### Failing line:\n{{testBlock}}\n\n### Failing test file path:\n{{testFilePath}}\n\nYour goal is to enrich the user message with \"Failing line\" and \"Failing test file path\", if the user's message contains\nrequests like correcting or replacing failing lines or statements.\n\nTo fulfil your goal, follow these steps:\n1. Identify the user intent and check if the user says anything similar to:\n- \"Replace the failing line\"\n- \"Replace the error statement\"\n- \"Replace the failing statement\"\n- \"Replace the failed user action\"\n2. The user message intent can also be to modify things in or around the \"failing line\". Lookout for this scenario as\nwell to enrich the user message\n3. If such keywords are present, only then enrich the user message by inserting \"Failing line\" and \"Failing test file\npath\" at appropriate locations.\n\nBefore submitting your response, ensure the following:\n- Do not generate code for the user prompt. Your task is just to enrich the user message\n- Enrich the user message ONLY if any of these keywords are present - \"failing line\", \"error statement\", \"failing\nstatement\", \"failed user action\"\n- Do not include any markdown syntax or backticks\n- Respond with <reason_for_output></reason_for_output>, <is_user_message_enriched></is_user_message_enriched> and\n<output></output> as xml tags\n- is_user_message_enriched can be either true or false\n- output should follow the format: \"Replace the line <code> with <whatever is required to be done> in <file_path>\"\n{{/section}}\n\n{{#section \"user\"}}\nUser message:\n{{userMessage}}\n{{/section}}";
+const utils_1 = require("./utils");
+const responseFormat = {
+    type: "json_schema",
+    json_schema: {
+        name: "analyze_user_message_and_conditionally_enrich_it",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                reason_for_output: {
+                    type: "string",
+                    description: "provide reason in both cases whether the user message was enriched or not",
+                },
+                is_user_message_enriched: {
+                    type: "boolean",
+                    description: "whether the user message should be enriched or not",
+                },
+                output: {
+                    type: "string",
+                    description: "This is the final output after analyzing the user message, and optionally enriching it.",
+                },
+            },
+            required: ["is_user_message_enriched", "output", "reason_for_output"],
+            additionalProperties: false,
+        },
+    },
+};
+const enrichPromptWithFailingLine = async ({ trace, testBlock, testFilePath, suggestionForFix, }) => {
+    let output = {
+        output: suggestionForFix,
+        is_user_message_enriched: false,
+        reason_for_output: "",
+    };
+    try {
+        const enrichedPromptSpan = trace?.span({
+            name: "enrich-prompt-with-failing-line",
+            input: {
+                testBlock,
+                testFilePath,
+                suggestionForFix,
+            },
+        });
+        const instructions = (0, llm_1.compilePrompt)(promptTemplate_0, {
+            testBlock,
+            testFilePath,
+            userMessage: suggestionForFix,
+        });
+        const llm = new llm_1.LLM({
+            trace: enrichedPromptSpan,
+            providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
+            provider: "anthropic",
+            defaultModel: "claude-3-5-sonnet-latest",
+        });
+        const llmResponse = await llm.createChatCompletion({
+            messages: instructions,
+            trace,
+            modelParameters: {
+                temperature: 0.5,
+            },
+            responseFormat,
+        });
+        const generateRequestedChangeResponse = (0, utils_1.extractGenerateRequestedChange)(llmResponse?.content);
+        output = {
+            output: generateRequestedChangeResponse.output,
+            is_user_message_enriched: generateRequestedChangeResponse.is_user_message_enriched,
+            reason_for_output: generateRequestedChangeResponse.reason_for_output,
+        };
+        enrichedPromptSpan?.end({ output: output });
+    }
+    catch (e) {
+        console.error(`Failed to generate requested change for test - "${testBlock}", suggestion - "${suggestionForFix}"`, e);
+    }
+    return output;
+};
+exports.enrichPromptWithFailingLine = enrichPromptWithFailingLine;

package/bundled/test-gen/agent/enrich-prompt/utils.d.ts

@@ -0,0 +1,6 @@
+export declare function extractGenerateRequestedChange(input: string): {
+    reason_for_output: string;
+    is_user_message_enriched: boolean;
+    output: string;
+};
+//# sourceMappingURL=utils.d.ts.map

package/bundled/test-gen/agent/enrich-prompt/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/enrich-prompt/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC7D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,wBAAwB,EAAE,OAAO,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;CAChB,CAQA"}

package/bundled/test-gen/agent/enrich-prompt/utils.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractGenerateRequestedChange = void 0;
+function extractGenerateRequestedChange(input) {
+    const regex = /<reason_for_output>([\s\S]*?)<\/reason_for_output>[\s\S]*?<is_user_message_enriched>([\s\S]*?)<\/is_user_message_enriched>[\s\S]*?<output>([\s\S]*?)<\/output>/g;
+    const match = regex.exec(input);
+    const reason_for_output = match[1]?.trim() || "";
+    const is_user_message_enriched = match[2]?.trim() === "true";
+    const output = match[3]?.trim() || "";
+    return { reason_for_output, is_user_message_enriched, output };
+}
+exports.extractGenerateRequestedChange = extractGenerateRequestedChange;

package/bundled/test-gen/agent/infer-agent/index.d.ts

@@ -0,0 +1,10 @@
+import { TraceClient } from "@empiricalrun/llm";
+import { Agent, TestGenConfigOptions } from "@empiricalrun/shared-types";
+export declare function inferAgentBasedTask({ task, options, trace, }: {
+    task: string;
+    options?: TestGenConfigOptions;
+    trace?: TraceClient;
+}): Promise<{
+    response: Agent;
+}>;
+//# sourceMappingURL=index.d.ts.map

package/bundled/test-gen/agent/infer-agent/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/infer-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAYzE,wBAAsB,mBAAmB,CAAC,EACxC,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,KAAK,CAAA;CAAE,CAAC,CAmE/B"}

package/bundled/test-gen/agent/infer-agent/index.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inferAgentBasedTask = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const constants_1 = require("../../constants");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer specializing in Playwright end-to-end tests. You are given a task which is a part of an\nend-to-end test scenario. The task may involve updating an existing end-to-end test case or writing a new test case from\nscratch. Tests involve user interactions (e.g. click on element) or other actions supported by Playwright (e.g.\nintercept network requests)\n\nYour objective is to identify whether the task requires accessing a web browser or not.\n\nTo fulfill your objective, answer the following questions:\n\n1. Does it require you to interact with a UI element in the browser? Examples of interactions are click, fill, type, key\npress, assert visibility of the element. Actions that interact with network requests are not UI element interactions.\n\n2. Is the locator of this UI element given to you in the task? Locators look like `getByText(...)`, `getByTestId(...)`\nand other locator methods in Playwright\n\n3. Decide if you need a browser: if you need to interact with a UI element AND you are NOT given the locator for that\nelement, you WILL NEED a browser.\n\n4. If you NEED a browser, then respond with answer as \"master\", otherwise respond with \"code\"\n\n\n# Example 1\n## Input\nTask:\nin this test don't delete the agent and remove steps after that\n\n## Output\n- ui_interaction_to_be_performed: There is no interaction here\n- ui_element_to_interact_with: No element specified\n- has_locator_for_that_element: No element specified\n- reasoning_for_browser_required: No interaction hence browser is not required\n- answer: code\n\n# Example 2\n## Input\nTask:\nin the swapfast test, replace the selectTokenForSwap method. Instead we will do this\\nclick on token button - this will\nshow a drawer\\nEnter usd in the search field that shows up in the drawer\\nSelect USDC.axl for Cosmos Hub - very\nimportant to choose this instead of USDC.axl on Osmosis\n\n## Output\n- ui_interaction_to_be_performed: Click on token button\n- ui_element_to_interact_with: Token button\n- has_locator_for_that_element: false\n- reasoning_for_browser_required: Task requires interacting with a UI element and identifying its locator which needs a\nbrowser\n- answer: master\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n{{/section}}";
+const session_1 = require("../../session");
+const session = (0, session_1.getSessionDetails)();
+async function inferAgentBasedTask({ task, options, trace, }) {
+    trace =
+        trace ||
+            llm_1.langfuseInstance?.trace({
+                name: "infer-agent-task",
+                id: crypto.randomUUID(),
+                release: session.version,
+            });
+    const inferAgentSpan = trace?.span({
+        name: "infer-agent",
+        input: {
+            task,
+            options,
+        },
+    });
+    const messages = (0, llm_1.compilePrompt)(promptTemplate_0, { task });
+    const llm = new llm_1.LLM({
+        trace: inferAgentSpan,
+        provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: options?.model || constants_1.DEFAULT_MODEL,
+        providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+    });
+    const firstShotMessage = await llm.createChatCompletion({
+        messages,
+        modelParameters: {
+            ...constants_1.DEFAULT_MODEL_PARAMETERS,
+            ...options?.modelParameters,
+            temperature: 1,
+        },
+        responseFormat: {
+            type: "json_schema",
+            json_schema: {
+                name: "infer-master-code",
+                strict: true,
+                schema: {
+                    type: "object",
+                    properties: {
+                        reason: {
+                            type: "string",
+                        },
+                        response: {
+                            type: "string",
+                        },
+                    },
+                    additionalProperties: false,
+                    required: ["reason", "response"],
+                },
+            },
+        },
+    });
+    const output = JSON.parse(firstShotMessage?.content || "{}");
+    inferAgentSpan?.end({
+        output: {
+            response: output.response,
+            reason: output.reason,
+        },
+    });
+    return {
+        response: output.response,
+    };
+}
+exports.inferAgentBasedTask = inferAgentBasedTask;

package/bundled/test-gen/agent/master/action-tool-calls.d.ts

@@ -0,0 +1,42 @@
+export declare enum ActionType {
+    FILL = "fill",
+    PAGE_GOTO = "page_goto",
+    CLICK = "click",
+    PRESS_ACTION = "keyboard_press_on_element",
+    ASSERT_TEXT = "assert_text",
+    HOVER = "hover_element",
+    SCROLL = "scroll",
+    OBSERVATION = "observation",
+    UNKNOWN = "unknown"
+}
+export declare function isValidActionType(value: string): value is ActionType;
+export declare function getActionToolCalls(): {
+    type: string;
+    function: {
+        name: string;
+        description: string;
+        parameters: {
+            type: string;
+            properties: {
+                reason: {
+                    type: string;
+                    description: string;
+                };
+                action: {
+                    type: string;
+                    description: string;
+                };
+                element_description: {
+                    type: string;
+                    description: string;
+                };
+                is_element_visible: {
+                    type: string;
+                    description: string;
+                };
+            };
+            required: string[];
+        };
+    };
+}[];
+//# sourceMappingURL=action-tool-calls.d.ts.map

package/bundled/test-gen/agent/master/action-tool-calls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"action-tool-calls.d.ts","sourceRoot":"","sources":["../../../src/agent/master/action-tool-calls.ts"],"names":[],"mappings":"AAAA,oBAAY,UAAU;IACpB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,YAAY,8BAA8B;IAC1C,WAAW,gBAAgB;IAC3B,KAAK,kBAAkB;IACvB,MAAM,WAAW;IACjB,WAAW,gBAAgB;IAC3B,OAAO,YAAY;CACpB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,UAAU,CAEpE;AAqDD,wBAAgB,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA8CjC"}

package/bundled/test-gen/agent/master/action-tool-calls.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getActionToolCalls = exports.isValidActionType = exports.ActionType = void 0;
+var ActionType;
+(function (ActionType) {
+    ActionType["FILL"] = "fill";
+    ActionType["PAGE_GOTO"] = "page_goto";
+    ActionType["CLICK"] = "click";
+    ActionType["PRESS_ACTION"] = "keyboard_press_on_element";
+    ActionType["ASSERT_TEXT"] = "assert_text";
+    ActionType["HOVER"] = "hover_element";
+    ActionType["SCROLL"] = "scroll";
+    ActionType["OBSERVATION"] = "observation";
+    ActionType["UNKNOWN"] = "unknown";
+})(ActionType || (exports.ActionType = ActionType = {}));
+function isValidActionType(value) {
+    return Object.values(ActionType).includes(value);
+}
+exports.isValidActionType = isValidActionType;
+const createActionCall = (name, description, additionalProperties = {}) => ({
+    type: "function",
+    function: {
+        name,
+        description,
+        parameters: {
+            type: "object",
+            properties: {
+                reason: {
+                    type: "string",
+                    description: "explain how this action will help to complete the task. the reason should align with the task provided",
+                },
+                action: {
+                    type: "string",
+                    description: `explain the next action in natural language.
+                The next action should be as atomic as possible, precise and should contain enough details about the action to be performed.
+                E.g. each click, key press, input, assert, observation should be a separate action.
+                Each action should take the task to completion, if not the action is invalid.
+                If the element is not visible then action should be scroll.`,
+                },
+                element_description: {
+                    type: "string",
+                    description: "The description of the element on which action needs to be taken, including its position, appearance, etc.",
+                },
+                is_element_visible: {
+                    type: "boolean",
+                    description: "A boolean to indicate whether the concerned element is visible in the screenshot.",
+                },
+                ...additionalProperties,
+            },
+            required: [
+                "reason",
+                "action",
+                "element_description",
+                "is_element_visible",
+                ...Object.keys(additionalProperties),
+            ],
+        },
+    },
+});
+function getActionToolCalls() {
+    return [
+        createActionCall(ActionType.ASSERT_TEXT, "assert whether the given element on the page is visible", {
+            assertion_text: {
+                type: "string",
+                description: "The text for which the visibility needs to be asserted.",
+            },
+        }),
+        createActionCall(ActionType.FILL, "fill the input element with a particular text"),
+        createActionCall(ActionType.CLICK, "click an element on the page"),
+        createActionCall(ActionType.PAGE_GOTO, "navigate the page to a new URL"),
+        createActionCall(ActionType.PRESS_ACTION, "Focuses the matching element and presses a combination of the keys"),
+        createActionCall(ActionType.HOVER, "hover over an element on the page"),
+        createActionCall(ActionType.SCROLL, "Scroll the page since element is not visible in the screenshot", {
+            element: {
+                type: "string",
+                description: "Element to look for after scrolling to complete the task.",
+            },
+        }),
+        createActionCall(ActionType.OBSERVATION, "Observation based on the task provided. If the task doesn't explicitly mention any browser interaction, and instead, asks questions to be answered — use this tool call", {
+            observation: {
+                type: "string",
+                description: "Observation based on the task provided.",
+            },
+        }),
+    ];
+}
+exports.getActionToolCalls = getActionToolCalls;

package/bundled/test-gen/agent/master/browser-tests/fixtures.d.ts

@@ -0,0 +1,9 @@
+export type ServerFixture = {
+    port: number;
+    baseURL: string;
+};
+export declare const expect: import("@playwright/test").Expect<{}>;
+export declare const test: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & {
+    server: ServerFixture;
+}>;
+//# sourceMappingURL=fixtures.d.ts.map

package/bundled/test-gen/agent/master/browser-tests/fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/browser-tests/fixtures.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,eAAO,MAAM,MAAM,uCAAc,CAAC;AAClC,eAAO,MAAM,IAAI;YAA6B,aAAa;EAsBzD,CAAC"}

package/bundled/test-gen/agent/master/browser-tests/fixtures.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.test = exports.expect = void 0;
+const test_1 = require("@playwright/test");
+const http_server_1 = __importDefault(require("http-server"));
+const path_1 = __importDefault(require("path"));
+exports.expect = test_1.test.expect;
+exports.test = test_1.test.extend({
+    server: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use, workerInfo) => {
+            const port = 2345 + workerInfo.workerIndex;
+            const server = http_server_1.default.createServer({
+                root: path_1.default.join(process.cwd(), "test-data"),
+            });
+            await new Promise((resolve) => {
+                server.listen(port, () => {
+                    console.log(`Server running at http://localhost:${port}`);
+                    resolve();
+                });
+            });
+            await use({
+                port,
+                baseURL: `http://localhost:${port}`,
+            });
+            server.close();
+        },
+        { scope: "worker" },
+    ],
+});

package/bundled/test-gen/agent/master/browser-tests/index.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.spec.d.ts.map

package/bundled/test-gen/agent/master/browser-tests/index.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.spec.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/browser-tests/index.spec.ts"],"names":[],"mappings":""}

package/bundled/test-gen/agent/master/browser-tests/index.spec.js

@@ -0,0 +1,113 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const utils_1 = require("../../browsing/utils");
+const element_annotation_1 = require("../element-annotation");
+const run_1 = require("../run");
+const fixtures_1 = require("./fixtures");
+(0, fixtures_1.test)("able to scroll and interact with elements", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `fill test@test.com into the email field and click the submit`,
+        page,
+        options: {},
+    });
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBe(0);
+    (0, fixtures_1.expect)(response.code).toContain("await page.getByPlaceholder('Enter your email').fill(\"test@test.com\")");
+    (0, fixtures_1.expect)(response.code).toContain("await page.getByRole('button', { name: 'Subscribe' }).click()");
+});
+(0, fixtures_1.test)("scroll when element does not exist", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `click search button`,
+        page,
+        options: {},
+    });
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBe(0);
+    (0, fixtures_1.expect)(response.code.length).toBe(0);
+});
+(0, fixtures_1.test)("scroll and click inside div elements", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/dropdown-scrolls.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `click on x-3 inside bmw dropdown, and then,
+click on maverick inside ford dropdown`,
+        page,
+        options: {},
+    });
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBe(0);
+    (0, fixtures_1.expect)(response.code.length).toBeGreaterThan(0);
+    const lines = response.code.split("\n");
+    (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page.+BMW.+.click/))).toBeTruthy();
+    (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page.+X3.+.click/))).toBeTruthy();
+    (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page.+Ford.+.click/))).toBeTruthy();
+    (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page.+Maverick.+.click/))).toBeTruthy();
+});
+(0, fixtures_1.test)("agent can click icons accurately", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/icons-navbar.html`);
+    await (0, fixtures_1.expect)(page.getByText("select an icon")).toBeVisible();
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `click on the gear icon`,
+        page,
+        options: {},
+    });
+    // Validate code generated and action performed
+    await (0, fixtures_1.expect)(page.getByText("you clicked Gear")).toBeVisible();
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBe(0);
+    (0, fixtures_1.expect)(response.code).toContain("page.locator");
+    (0, fixtures_1.expect)(response.code).toContain("click()");
+    // Validate icons registry
+    const iconsRegistryFile = path_1.default.join(process.cwd(), ".empiricalrun", "icons.json");
+    const icons = JSON.parse(fs_1.default.readFileSync(iconsRegistryFile, "utf-8"));
+    (0, fixtures_1.expect)(icons.length).toBeGreaterThan(0);
+    // Commenting out this check since with parallel test executions,
+    // we can have icons from other tests in the same file
+    // expect(icons.length).toBe(4); // 1 for each unique icon
+    fs_1.default.unlinkSync(iconsRegistryFile);
+});
+(0, fixtures_1.test)("annotate and enrich annotations correctly", async ({ page, server }) => {
+    await (0, utils_1.injectPwLocatorGenerator)(page);
+    await page.goto(`${server.baseURL}/iframe-elements.html`);
+    const { annotationKeys: keys } = await (0, element_annotation_1.getAnnotationKeys)({
+        page,
+        preference: {
+            actionType: "all",
+        },
+    });
+    console.log(keys);
+    (0, fixtures_1.expect)(keys.length).toBe(6);
+    // 2 icons: 1 in main frame, 1 in iframe
+    (0, fixtures_1.expect)(keys.filter((k) => k.text.includes("icon") && k.text.includes("close"))
+        .length).toBe(2);
+    // 2 text inputs: 1 in main frame, 1 in iframe
+    (0, fixtures_1.expect)(keys.filter((k) => k.text.includes("Enter your name")).length).toBe(2);
+    // 2 clickable divs: 1 in main frame, 1 in iframe
+    (0, fixtures_1.expect)(keys.filter((k) => k.text.includes("Lorem Ipsum")).length).toBe(2);
+});
+(0, fixtures_1.test)("fill action with multiple pages", async ({ context }) => {
+    const page1 = await context.newPage();
+    const page2 = await context.newPage();
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `goto empirical.run on page1 and goto github.com/search in page2. Enter text empirical on page2 and click on search.`,
+        page: page2,
+        options: {},
+        scopeVars: {
+            context,
+            page1,
+            page2,
+        },
+    });
+    (0, fixtures_1.expect)(response.code).toBeTruthy();
+    console.log(response.code);
+    const lines = response.code.split("\n");
+    (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page1\.goto(.+)empirical\.run/))).toBeTruthy();
+    (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page2\.goto(.+)github\.com\/search/))).toBeTruthy();
+    (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page2(.+)fill(.+)empirical/))).toBeTruthy();
+});

package/bundled/test-gen/agent/master/browser-tests/skills.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=skills.spec.d.ts.map

package/bundled/test-gen/agent/master/browser-tests/skills.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skills.spec.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/browser-tests/skills.spec.ts"],"names":[],"mappings":""}

package/bundled/test-gen/agent/master/browser-tests/skills.spec.js

@@ -0,0 +1,109 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const run_1 = require("../run");
+const fixtures_1 = require("./fixtures");
+fixtures_1.test.describe.configure({ mode: "default" });
+fixtures_1.test.beforeEach(async () => {
+    // Setup repo with POM file
+    fs_1.default.mkdirSync("pages");
+    const pomContent = `
+import type { Page } from "@playwright/test";
+
+export async function subscribeToBlog({ page, email }: { page: Page; email: string }) {
+  await page.getByPlaceholder("Enter your email").fill(email);
+  await page.getByRole("button", { name: "Subscribe" }).click();
+}
+
+export async function extractTitleForPost({ page, nth }: { page: Page; nth: number }) {
+  // nth is zero-indexed
+  const title = await page.locator("h2").nth(nth).textContent();
+  return title;
+}
+`;
+    fs_1.default.writeFileSync("pages/blog.ts", pomContent);
+});
+fixtures_1.test.afterEach(async () => {
+    // Clean up the repo in cwd
+    fs_1.default.rmSync("pages", { recursive: true });
+});
+(0, fixtures_1.test)("use skills to subscribe to blog", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `subscribe as user@example.com`,
+        page,
+        testCase: {
+            id: 1,
+            name: "subscribe to blog",
+            steps: ["subscribe as user@example.com"],
+            filePath: "blog.spec.ts",
+            suites: [],
+        },
+        options: {},
+    });
+    // Assert execution was successful
+    await (0, fixtures_1.expect)(page.getByText("Thanks for subscribing")).toBeVisible();
+    // Assert code generated is correct
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.importPaths[0]).toBe("pages/blog.ts");
+    (0, fixtures_1.expect)(response.code.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.code).toMatch(/await.*subscribeToBlog.*page.*email.*user@example\.com/);
+});
+(0, fixtures_1.test)("use skills to extract blog post title", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        // TODO: Extend this to click on "read more" and verify generated code
+        // uses the state variable created by the skill
+        task: `extract title of 2nd blog post`,
+        page,
+        testCase: {
+            id: 1,
+            name: "extract blog post title",
+            steps: ["extract title of 2nd blog post"],
+            filePath: "blog.spec.ts",
+            suites: [],
+        },
+        options: {},
+    });
+    // Assert code generated is correct
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.importPaths[0]).toBe("pages/blog.ts");
+    (0, fixtures_1.expect)(response.code.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.code).toMatch(/^const.*=.*await extractTitleForPost.*page.*nth:.*1/);
+});
+(0, fixtures_1.test)("use skills to subscribe with multiple pages", async ({ page, server, }) => {
+    await page.goto(`${server.baseURL}/icons-navbar.html`);
+    const blogPage = await page.context().newPage();
+    await blogPage.goto(`${server.baseURL}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `we have 2 pages open inside a web browser, and your task is to subscribe as user@example.com on the blogPage`,
+        page,
+        testCase: {
+            id: 1,
+            name: "subscribe to blog",
+            steps: [
+                "we have 2 pages open inside a web browser, and your task is to subscribe as user@example.com on the blogPage",
+            ],
+            filePath: "blog.spec.ts",
+            suites: [],
+        },
+        scopeVars: {
+            page,
+            blogPage,
+        },
+        options: {},
+    });
+    // Assert execution was successful
+    await (0, fixtures_1.expect)(blogPage.getByText("Thanks for subscribing")).toBeVisible();
+    // Assert code generated is correct
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.importPaths[0]).toBe("pages/blog.ts");
+    (0, fixtures_1.expect)(response.code.length).toBeGreaterThan(0);
+    (0, fixtures_1.expect)(response.code).toMatch(/await.*subscribeToBlog.*blogPage.*email.*user@example\.com/);
+});

package/bundled/test-gen/agent/master/element-annotation.d.ts

@@ -0,0 +1,30 @@
+import { LLM, TraceClient } from "@empiricalrun/llm";
+import { TestGenConfigOptions } from "@empiricalrun/shared-types";
+import { Page } from "playwright";
+import { ActionType } from "./action-tool-calls";
+export declare function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, preference, }: {
+    elementDescription: string;
+    annotations: string;
+    annotatedScreenshot: string;
+    trace?: TraceClient;
+    llm?: LLM;
+    options?: TestGenConfigOptions;
+    preference: AnnotationPreference;
+}): Promise<string | undefined>;
+export type AnnotationPreference = {
+    actionType: "all" | ActionType.FILL | ActionType.ASSERT_TEXT | ActionType.SCROLL;
+    assertionText?: string | undefined;
+};
+export declare function getAnnotationKeys({ page, preference, trace, }: {
+    page: Page;
+    preference: AnnotationPreference;
+    trace?: TraceClient;
+}): Promise<{
+    annotationKeys: {
+        elementID: string;
+        text: string;
+    }[];
+    annotationBuffer: Buffer;
+    annotatedPageScreenshot: string;
+}>;
+//# sourceMappingURL=element-annotation.d.ts.map

package/bundled/test-gen/agent/master/element-annotation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA2DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAgBF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAoFD"}