@empiricalrun/test-gen 0.8.2 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @empiricalrun/test-gen
 
+## 0.9.0
+
+### Minor Changes
+
+- a6067de: feat: generate better playwright locators using playwright
+
 ## 0.8.2
 
 ### Patch Changes

package/dist/actions/assertTextVisibility.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"assertTextVisibility.d.ts","sourceRoot":"","sources":["../../src/actions/assertTextVisibility.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBA0CjD,CAAC"}
+{"version":3,"file":"assertTextVisibility.d.ts","sourceRoot":"","sources":["../../src/actions/assertTextVisibility.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBA8DjD,CAAC"}

package/dist/actions/assertTextVisibility.js

@@ -1,18 +1,36 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.assertTextVisibilityActionGenerator = exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = void 0;
+const utils_1 = require("./utils");
 exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = "assert_text_visibility";
 const assertTextVisibilityActionGenerator = (page) => {
     return {
         execute: async (args) => {
+            const css = args.css_selector;
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(args.css_selector, page, {
+                hasText: args.text,
+            });
             await page
-                .locator(args.css_selector)
+                .locator(css)
                 .filter({ hasText: args.text })
                 .first()
                 .isVisible({ timeout: 3000 });
+            return {
+                locator,
+            };
         },
         // TODO: args transformer to be kept at a single place
-        template: (args) => `await expect(page.locator("${args.css_selector}").filter({ hasText: "${args.text}"}).first()).toBeVisible();`,
+        template: (args, options) => {
+            const css = args.css_selector;
+            const assertStr = `await expect(page.locator("${css}").filter({ hasText: "${args.text}"}).first()).toBeVisible();`;
+            if (options?.locator) {
+                return `
+        // ${assertStr}
+        await expect(page.${options.locator}).toBeVisible();
+        `;
+            }
+            return assertStr;
+        },
         name: exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME,
         schema: {
             type: "function",

package/dist/actions/click.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAoClC,CAAC"}
+{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAkDlC,CAAC"}

package/dist/actions/click.js

@@ -1,18 +1,31 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.clickActionGenerator = exports.PLAYWRIGHT_CLICK_ACTION_NAME = void 0;
+const utils_1 = require("./utils");
 exports.PLAYWRIGHT_CLICK_ACTION_NAME = "click_element";
 const clickActionGenerator = (page) => {
     return {
         execute: async (args) => {
-            await page
-                .locator(args.css_selector.split(" ")[0].replaceAll("\\", ""))
-                .first()
-                .click({ timeout: 3000 });
+            const selector = args.css_selector;
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(selector, page);
+            await page.locator(selector).first().click({ timeout: 3000 });
             await page.waitForTimeout(3000);
+            return {
+                locator,
+            };
         },
         // TODO: args transformer to be kept at a single place
-        template: (args) => `await page.locator("${args.css_selector.split(" ")[0].replaceAll("\\", "")}").first().click();`,
+        template: (args, options) => {
+            const css = args.css_selector;
+            const templ = `await page.locator("${css}").first().click();`;
+            if (options?.locator) {
+                return `
+        // ${templ}
+        await page.${options.locator}.click();
+        `;
+            }
+            return templ;
+        },
         name: exports.PLAYWRIGHT_CLICK_ACTION_NAME,
         schema: {
             type: "function",

package/dist/actions/fill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAEhE,eAAO,MAAM,mBAAmB,EAAE,yBAsCjC,CAAC"}
+{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAEhE,eAAO,MAAM,mBAAmB,EAAE,yBAkDjC,CAAC"}

package/dist/actions/fill.js

@@ -1,16 +1,30 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fillActionGenerator = exports.PLAYWRIGHT_FILL_ACTION_NAME = void 0;
+const utils_1 = require("./utils");
 exports.PLAYWRIGHT_FILL_ACTION_NAME = "fill_input_element";
 const fillActionGenerator = (page) => {
     return {
         execute: async (args) => {
-            await page
-                .locator(args.css_selector.split(" ")[0])
-                .fill(args.text, { timeout: 3000 });
+            const css = args.css_selector;
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(css, page);
+            await page.locator(css).fill(args.text, { timeout: 3000 });
+            return {
+                locator,
+            };
         },
         // TODO: args transformer to be kept at a single place
-        template: (args) => `await page.locator("${args.css_selector.split(" ")[0]}").fill("${args.text}");`,
+        template: (args, options) => {
+            const css = args.css_selector;
+            const templ = `await page.locator("${css}").fill("${args.text}");`;
+            if (options?.locator) {
+                return `
+        // ${templ}
+        await page.${options?.locator}.fill("${args.text}");
+        `;
+            }
+            return templ;
+        },
         name: exports.PLAYWRIGHT_FILL_ACTION_NAME,
         schema: {
             type: "function",

package/dist/actions/index.js

@@ -24,9 +24,9 @@ class PlaywrightActions {
         }
         try {
             console.log("executing:", name, "\nreason:", args.reason);
-            await action.execute(args);
+            const templateOptions = (await action.execute(args)) || undefined;
             // record successful actions
-            const code = action.template(args);
+            const code = action.template(args, templateOptions);
             this.recordedActions.push({ name, code });
             console.log(`code: ${code}`, "\n\n");
         }

package/dist/actions/utils/index.d.ts

@@ -0,0 +1,8 @@
+import { Page } from "playwright";
+declare global {
+    interface Window {
+        playwright: any;
+    }
+}
+export declare function getPlaywrightLocatorUsingCssSelector(cssSelector: string, page: Page, filter?: Record<string, any>): Promise<any>;
+//# sourceMappingURL=index.d.ts.map

package/dist/actions/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;KACjB;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,IAAI,EACV,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,gBAsB7B"}

package/dist/actions/utils/index.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPlaywrightLocatorUsingCssSelector = void 0;
+async function getPlaywrightLocatorUsingCssSelector(cssSelector, page, filter) {
+    return await page.evaluate((locator) => {
+        const elements = document.querySelectorAll(locator.css);
+        let element = elements[0];
+        if (locator.filter?.hasText) {
+            element = Array.from(elements).find((el) => el.textContent?.trim().replaceAll(" ", "") ===
+                locator.filter?.hasText.trim().replaceAll(" ", ""));
+        }
+        if (!element) {
+            throw Error(`Unable to find element, css: ${locator.css}, filter ${JSON.stringify(locator.filter)}`);
+        }
+        return window.playwright.generateLocator(element);
+    }, { css: cssSelector, filter });
+}
+exports.getPlaywrightLocatorUsingCssSelector = getPlaywrightLocatorUsingCssSelector;

package/dist/agent/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,GAAE;IACP,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACE,mBAsDP"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,GAAE;IACP,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACE,mBAgEP"}

package/dist/agent/index.js

@@ -15,6 +15,13 @@ async function browsingAgent(task, page, options = {}) {
     const tools = actions.getActionSchemas();
     let isTaskDone = false;
     const previousActions = [];
+    const resp = await fetch("https://assets-test.empirical.run/pw-selector.js");
+    const script = await resp.text();
+    await page.addScriptTag({
+        content: script,
+    });
+    let lastActionErrors = [];
+    // await page.waitForTimeout(3_00_000)
     while (!isTaskDone) {
         trace.update({ input: { task } });
         const pageContentSpan = trace.startSpan("page-content");
@@ -28,8 +35,10 @@ async function browsingAgent(task, page, options = {}) {
             pageSnapshot,
             previousActions,
             task,
+            lastActionErrors,
         });
         promptSpan.end({ output: { messages } });
+        lastActionErrors = [];
         const completion = await (0, ai_1.getLLMResult)({
             messages,
             tools,
@@ -44,6 +53,7 @@ async function browsingAgent(task, page, options = {}) {
             }
             catch (e) {
                 // TODO: implement feedback loop to llm
+                lastActionErrors.push(e.message);
                 console.error(e);
             }
         }

package/dist/bin/ai/prompts/provider/index.d.ts

@@ -1,8 +1,9 @@
 import OpenAI from "openai";
 export declare function getPrompt(name: string, vars: any): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
-export declare function getPromptForNextAction({ pageSnapshot, task, previousActions, }: {
+export declare function getPromptForNextAction({ pageSnapshot, task, previousActions, lastActionErrors, }: {
     pageSnapshot: string;
     task: string;
     previousActions: string[];
+    lastActionErrors: string[];
 }): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/ai/prompts/provider/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/ai/prompts/provider/index.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,GAAG,GACR,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC,CAI/D;AAGD,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,GACrB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B,iEAQA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/ai/prompts/provider/index.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,GAAG,GACR,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC,CAI/D;AAGD,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,GACtB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,iEASA"}

package/dist/bin/ai/prompts/provider/index.js

@@ -15,12 +15,13 @@ async function getPrompt(name, vars) {
 }
 exports.getPrompt = getPrompt;
 // TODO: move this prompt to langfuse
-async function getPromptForNextAction({ pageSnapshot = "", task = "", previousActions = [], }) {
+async function getPromptForNextAction({ pageSnapshot = "", task = "", previousActions = [], lastActionErrors = [], }) {
     const previousActionsStr = previousActions.join("\n\n ---- \n\n");
     const prompt = await getPrompt("browsing-agent-next-action", {
         pageSnapshot,
         previousActionsStr,
         task,
+        lastActionErrors,
     });
     return prompt;
 }

package/dist/types/index.d.ts

@@ -19,7 +19,11 @@ export type ActionSchema = OpenAI.Chat.Completions.ChatCompletionTool;
 export type Action = {
     name: string;
     schema: ActionSchema;
-    execute: (args: Record<string, any>) => Promise<void>;
-    template: (args: Record<string, any>) => string;
+    execute: (args: Record<string, any>) => Promise<{
+        locator: string;
+    } | void | undefined>;
+    template: (args: Record<string, any>, options?: {
+        locator: string;
+    }) => string;
 };
 //# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,MAAM,CAAC;CACjD,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CACP,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACtB,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;IACrD,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAC1B,MAAM,CAAC;CACb,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"