@empiricalrun/test-gen 0.45.0 → 0.46.0

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @empiricalrun/test-gen
 
+## 0.46.0
+
+### Minor Changes
+
+- b6dfc91: feat: create icon registry for test repositories
+
+### Patch Changes
+
+- b6dfc91: feat: cache icon description generated through LLM
+
+## 0.45.1
+
+### Patch Changes
+
+- 04db1a6: chore: better types for playwright action generators
+- 1b1815d: chore: move runtime planner to handlebars
+- 62800d7: fix: prevent sending test run events to sentry
+
 ## 0.45.0
 
 ### Minor Changes

package/dist/actions/assert.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { AssertTextVisibilityActionArgs, PlaywrightActionGenerator } from "../types";
 export declare const PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = "assert_text_visibility";
-export declare const assertTextVisibilityActionGenerator: PlaywrightActionGenerator;
+export declare const assertTextVisibilityActionGenerator: PlaywrightActionGenerator<AssertTextVisibilityActionArgs>;
 //# sourceMappingURL=assert.d.ts.map

package/dist/actions/assert.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"assert.d.ts","sourceRoot":"","sources":["../../src/actions/assert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBAuDjD,CAAC"}
+{"version":3,"file":"assert.d.ts","sourceRoot":"","sources":["../../src/actions/assert.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,8BAA8B,EAC9B,yBAAyB,EAC1B,MAAM,UAAU,CAAC;AAIlB,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBAAyB,CACzE,8BAA8B,CAqD/B,CAAC"}

package/dist/actions/assert.js

@@ -14,8 +14,7 @@ const assertTextVisibilityActionGenerator = (page) => {
                 locator,
             };
         },
-        // TODO: args transformer to be kept at a single place
-        template: (_, options) => {
+        template: (args, options) => {
             return {
                 code: `await expect(${page.name}.${options.locator}).toBeVisible();`,
             };

package/dist/actions/click.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { ClickActionArgs, PlaywrightActionGenerator } from "../types";
 export declare const PLAYWRIGHT_CLICK_ACTION_NAME = "click_element";
-export declare const clickActionGenerator: PlaywrightActionGenerator;
+export declare const clickActionGenerator: PlaywrightActionGenerator<ClickActionArgs>;
 //# sourceMappingURL=click.d.ts.map

package/dist/actions/click.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAsDlC,CAAC"}
+{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAItE,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAAyB,CAC1D,eAAe,CAsDhB,CAAC"}

package/dist/actions/click.js

@@ -15,8 +15,7 @@ const clickActionGenerator = (page) => {
                 locator,
             };
         },
-        // TODO: args transformer to be kept at a single place
-        template: (_, options) => {
+        template: (args, options) => {
             return {
                 code: `await ${page.name}.${options.locator}.click();`,
             };

package/dist/actions/done.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { BaseActionArgs, PlaywrightActionGenerator } from "../types";
 export declare const PLAYWRIGHT_DONE_ACTION_NAME = "task_done";
-export declare const doneActionGenerator: PlaywrightActionGenerator;
+export declare const doneActionGenerator: PlaywrightActionGenerator<BaseActionArgs>;
 //# sourceMappingURL=done.d.ts.map

package/dist/actions/done.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"done.d.ts","sourceRoot":"","sources":["../../src/actions/done.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAuBjC,CAAC"}
+{"version":3,"file":"done.d.ts","sourceRoot":"","sources":["../../src/actions/done.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrE,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAAyB,CACzD,cAAc,CAwBf,CAAC"}

package/dist/actions/fill.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { FillActionArgs, PlaywrightActionGenerator } from "../types";
 export declare const PLAYWRIGHT_FILL_ACTION_NAME = "fill_input_element";
-export declare const fillActionGenerator: PlaywrightActionGenerator;
+export declare const fillActionGenerator: PlaywrightActionGenerator<FillActionArgs>;
 //# sourceMappingURL=fill.d.ts.map

package/dist/actions/fill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAGhE,eAAO,MAAM,mBAAmB,EAAE,yBAgGjC,CAAC"}
+{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrE,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAGhE,eAAO,MAAM,mBAAmB,EAAE,yBAAyB,CAAC,cAAc,CA+FzE,CAAC"}

package/dist/actions/fill.js

@@ -25,8 +25,7 @@ const fillActionGenerator = (page, options) => {
                 testCaseLine,
             };
         },
-        // TODO: args transformer to be kept at a single place
-        template: (_, options) => {
+        template: (args, options) => {
             return {
                 code: `await ${page.name}.${options.testCaseLine}`,
             };

package/dist/actions/goto.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { GotoActionArgs, PlaywrightActionGenerator } from "../types";
 export declare const PLAYWRIGHT_GOTO_ACTION_NAME = "page_goto";
-export declare const gotoActionGenerator: PlaywrightActionGenerator;
+export declare const gotoActionGenerator: PlaywrightActionGenerator<GotoActionArgs>;
 //# sourceMappingURL=goto.d.ts.map

package/dist/actions/goto.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAuCjC,CAAC"}
+{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrE,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAAyB,CAAC,cAAc,CAyCzE,CAAC"}

package/dist/actions/hover.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { HoverActionArgs, PlaywrightActionGenerator } from "../types";
 export declare const PLAYWRIGHT_HOVER_ACTION_NAME = "hover_element";
-export declare const hoverActionGenerator: PlaywrightActionGenerator;
+export declare const hoverActionGenerator: PlaywrightActionGenerator<HoverActionArgs>;
 //# sourceMappingURL=hover.d.ts.map

package/dist/actions/hover.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hover.d.ts","sourceRoot":"","sources":["../../src/actions/hover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAqDlC,CAAC"}
+{"version":3,"file":"hover.d.ts","sourceRoot":"","sources":["../../src/actions/hover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAItE,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAAyB,CAC1D,eAAe,CAsDhB,CAAC"}

package/dist/actions/hover.js

@@ -11,14 +11,14 @@ const hoverActionGenerator = (page) => {
             const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(selector, args.xpath, page.pwPageInstance, args?.elementAnnotation);
             const exec = new Function("page", `return page.${locator}.hover({ timeout: 3000 })`);
             await exec(page.pwPageInstance);
-            await page.pwPageInstance.waitForTimeout(3000);
             return {
                 locator,
             };
         },
-        // TODO: args transformer to be kept at a single place
         template: (args, options) => {
-            return { code: `await ${page.name}.${options.locator}.hover();` };
+            return {
+                code: `await ${page.name}.${options.locator}.hover();`,
+            };
         },
         name: exports.PLAYWRIGHT_HOVER_ACTION_NAME,
         schema: {

package/dist/actions/index.d.ts

@@ -1,13 +1,13 @@
 import { TraceClient } from "@empiricalrun/llm";
 import { TestGenPage } from "../page";
-import { ActionSchema } from "../types";
+import { ActionArgs, ActionSchema } from "../types";
 export declare class PlaywrightActions {
     private page;
     private stateVariables;
     private actionGenerators;
     private recordedActions;
     constructor(page: TestGenPage, stateVariables?: Record<string, any>);
-    executeAction(name: string | undefined, args: Record<string, any>, trace?: TraceClient): Promise<string | undefined>;
+    executeAction(name: string | undefined, args: ActionArgs, trace?: TraceClient): Promise<string | undefined>;
     getBrowsingActionSchemas(): ActionSchema[];
     getMasterActionSchemas(): ActionSchema[];
     generateCode(): {

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IA0D9B,wBAAwB,IAAI,YAAY,EAAE;IAoB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAEL,UAAU,EACV,YAAY,EAEb,MAAM,UAAU,CAAC;AAWlB,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,UAAU,EAChB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IA0D9B,wBAAwB,IAAI,YAAY,EAAE;IAoB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}

package/dist/actions/press.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { PlaywrightActionGenerator, PressActionArgs } from "../types";
 export declare const PLAYWRIGHT_PRESS_ACTION_NAME = "keyboard_press_on_element";
-export declare const pressActionGenerator: PlaywrightActionGenerator;
+export declare const pressActionGenerator: PlaywrightActionGenerator<PressActionArgs>;
 //# sourceMappingURL=press.d.ts.map

package/dist/actions/press.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"press.d.ts","sourceRoot":"","sources":["../../src/actions/press.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,8BAA8B,CAAC;AAExE,eAAO,MAAM,oBAAoB,EAAE,yBA4DlC,CAAC"}
+{"version":3,"file":"press.d.ts","sourceRoot":"","sources":["../../src/actions/press.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAItE,eAAO,MAAM,4BAA4B,8BAA8B,CAAC;AAExE,eAAO,MAAM,oBAAoB,EAAE,yBAAyB,CAC1D,eAAe,CA6DhB,CAAC"}

package/dist/actions/skill.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
+import { PlaywrightActionGenerator, SkillActionArgs } from "../types";
 export declare const SKILL_USAGE = "skill_usage";
 export type Skill = {
     testStep: string;
@@ -16,6 +16,6 @@ export declare const testCaseSkills: TestCaseSkills;
 export declare function extractSkillFromArgs(args: Record<string, any>): {
     skillDetails: Skill | undefined;
 };
-export declare const skillActionGenerator: PlaywrightActionGenerator;
+export declare const skillActionGenerator: PlaywrightActionGenerator<SkillActionArgs>;
 export {};
 //# sourceMappingURL=skill.d.ts.map

package/dist/actions/skill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,MAAM,MAAM,KAAK,GAAG;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;;EAO7D;AAED,eAAO,MAAM,oBAAoB,EAAE,yBAmHlC,CAAC"}
+{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAGtE,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,MAAM,MAAM,KAAK,GAAG;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;;EAO7D;AAED,eAAO,MAAM,oBAAoB,EAAE,yBAAyB,CAC1D,eAAe,CAkHhB,CAAC"}

package/dist/actions/skill.js

@@ -35,9 +35,9 @@ const skillActionGenerator = (page, options) => {
     return {
         execute: async ({ args, trace }) => {
             const { skill, action } = args;
-            const { skillDetails } = extractSkillFromArgs(args);
+            const [skillDetails] = availableSkills.filter((skillDetails) => skillDetails.testStep === skill);
             if (!skillDetails) {
-                throw new Error(`No skill found for skill: ${skill}`);
+                throw new Error(`No skill found for ${skill}`);
             }
             const skillFilePath = skillDetails.filePath;
             // This assumes repoDir is process.cwd()
@@ -86,7 +86,6 @@ const skillActionGenerator = (page, options) => {
             };
         },
         template: (args, options) => {
-            // TODO: support for import statement
             const code = `${options.locator}`;
             const skill = args.skill;
             const [skillDetails] = availableSkills.filter((skillDetails) => skillDetails.testStep === skill);

package/dist/actions/text-content.d.ts

@@ -1,4 +1,4 @@
-import { PlaywrightActionGenerator } from "../types";
-export declare const PLAYWRIGHT_GOTO_ACTION_NAME = "text_content";
-export declare const textContentActionGenerator: PlaywrightActionGenerator;
+import { PlaywrightActionGenerator, TextContentActionArgs } from "../types";
+export declare const PLAYWRIGHT_TEXT_CONTENT_ACTION_NAME = "text_content";
+export declare const textContentActionGenerator: PlaywrightActionGenerator<TextContentActionArgs>;
 //# sourceMappingURL=text-content.d.ts.map

package/dist/actions/text-content.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"text-content.d.ts","sourceRoot":"","sources":["../../src/actions/text-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,iBAAiB,CAAC;AAE1D,eAAO,MAAM,0BAA0B,EAAE,yBA8DxC,CAAC"}
+{"version":3,"file":"text-content.d.ts","sourceRoot":"","sources":["../../src/actions/text-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAI5E,eAAO,MAAM,mCAAmC,iBAAiB,CAAC;AAElE,eAAO,MAAM,0BAA0B,EAAE,yBAAyB,CAChE,qBAAqB,CA2DtB,CAAC"}

package/dist/actions/text-content.js

@@ -1,34 +1,33 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.textContentActionGenerator = exports.PLAYWRIGHT_GOTO_ACTION_NAME = void 0;
+exports.textContentActionGenerator = exports.PLAYWRIGHT_TEXT_CONTENT_ACTION_NAME = void 0;
 const constants_1 = require("./constants");
 const utils_1 = require("./utils");
-exports.PLAYWRIGHT_GOTO_ACTION_NAME = "text_content";
+exports.PLAYWRIGHT_TEXT_CONTENT_ACTION_NAME = "text_content";
 const textContentActionGenerator = (page, options) => {
     return {
-        execute: async (args) => {
-            const css = args.css_selector;
-            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(css, args.xpath, page.pwPageInstance, args?.elementAnnotation);
+        execute: async ({ args }) => {
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(args.css_selector, args.xpath, page.pwPageInstance, args?.elementAnnotation);
             const exec = new Function("page", `return page.${locator}.textContent()`);
-            const value = (await exec(page.pwPageInstance));
-            if (options) {
-                options.stateVariables[args.variable_name] = value;
-            }
+            const textContent = await exec(page.pwPageInstance);
+            options.setStateVariables({
+                ...options.stateVariables,
+                [args.variable_name]: textContent,
+            });
             return {
                 locator,
             };
         },
         template: (args, options) => {
-            const code = `const ${args.variable_name} = await ${page.name}.${options.locator}.textContent();`;
             return {
-                code,
+                code: `const ${args.variable_name} = await ${page.name}.${options.locator}.textContent();`,
             };
         },
-        name: exports.PLAYWRIGHT_GOTO_ACTION_NAME,
+        name: exports.PLAYWRIGHT_TEXT_CONTENT_ACTION_NAME,
         schema: {
             type: "function",
             function: {
-                name: exports.PLAYWRIGHT_GOTO_ACTION_NAME,
+                name: exports.PLAYWRIGHT_TEXT_CONTENT_ACTION_NAME,
                 description: "extract text content of the element",
                 parameters: {
                     type: "object",

package/dist/agent/master/browser-tests/index.spec.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const test_1 = require("@playwright/test");
+const fs_1 = __importDefault(require("fs"));
 const http_server_1 = __importDefault(require("http-server"));
 const path_1 = __importDefault(require("path"));
 const run_1 = require("../run");
@@ -57,9 +58,15 @@ test_1.test.afterAll(() => {
         page,
         options: {},
     });
+    // Validate code generated and action performed
     await (0, test_1.expect)(page.getByText("you clicked Gear")).toBeVisible();
     console.log(response);
     (0, test_1.expect)(response.importPaths.length).toBe(0);
-    (0, test_1.expect)(response.code).toContain("page.getByRole");
+    (0, test_1.expect)(response.code).toContain("page.locator");
     (0, test_1.expect)(response.code).toContain("click()");
+    // Validate icons registry
+    const iconsRegistryFile = path_1.default.join(process.cwd(), "icons.json");
+    const icons = JSON.parse(fs_1.default.readFileSync(iconsRegistryFile, "utf-8"));
+    (0, test_1.expect)(icons.length).toBe(4); // 1 for each unique icon
+    fs_1.default.unlinkSync(iconsRegistryFile);
 });

package/dist/agent/master/element-annotation.d.ts

@@ -15,10 +15,11 @@ export type AnnotationPreference = {
     actionType: "all" | ActionType.FILL | ActionType.ASSERT_TEXT | ActionType.SCROLL;
     assertionText?: string | undefined;
 };
-export declare function getAnnotationKeys({ page, preference, options, }: {
+export declare function getAnnotationKeys({ page, preference, options, trace, }: {
     page: Page;
     preference: AnnotationPreference;
     options: BrowsingAgentOptions;
+    trace?: TraceClient;
 }): Promise<{
     annotationKeys: {
         elementID: string;

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAoDD"}
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA2DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAgBF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAuED"}

package/dist/agent/master/element-annotation.js

@@ -5,6 +5,7 @@ const llm_1 = require("@empiricalrun/llm");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements\npresent in the image.\n\nYou will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation\nis done by drawing a red box around the element and a small yellow box on it which contains unique element id.\n\nYou are given a Annotations which contains list of unique element id and description of the element separated by \":\".\n\nYou are also given the description of the element on which the action needs to be taken. The description includes\ninformation about how the element looks, it's position etc.\n\nYour task is to provide the annotation of the element on which the action needs to be performed based on the element\ndescription.\n\nFollow steps to fulfil your task:\n- Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each\nelement.\n- For describing each element Id\n-- iterate over each element Id in annotation list\n-- check if the description is already present for the element Id in the Annotation provided to you. If present skip\ndescribing it and use it as is.\n-- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or\nicon enclosed in the element.\n- Respond with the mapped element Ids as \"enriched_annotations\"\n- Based on the description provided to you and the enriched annotations, first identify the element Id whose description\nmatches the task provided\n\nNote:\n- Ensure providing the description of all the elements in the list.\n- Don't update the description if its already present in the given annotations\n- Replace all the \"NA\" with description of the element. Its position, how does it look like etc.\n- There should be no \"NA\" present in any of the element description\n{{/section}}\n\n{{#section \"user\"}}\nElement description:\n{{elementDescription}}\n\nAnnotations:\n{{annotations}}\n\n{{image annotatedScreenshot}}\n{{/section}}";
 const utils_1 = require("../utils");
+const icon_descriptor_1 = require("./icon-descriptor");
 const annotationToolAction = {
     name: "element_annotation",
     schema: {
@@ -99,8 +100,8 @@ async function getElementAnnotation({ elementDescription, annotations, annotated
     return;
 }
 exports.getElementAnnotation = getElementAnnotation;
-async function getAnnotationKeys({ page, preference, options, }) {
-    const annotationKeys = await page.evaluate(({ preference, options }) => {
+async function getAnnotationKeys({ page, preference, options, trace, }) {
+    const annotatedElements = await page.evaluate(({ preference, options }) => {
         // @ts-ignore
         // eslint-disable-next-line no-undef
         window.annotationInstance = annotateElementsWithPreference({
@@ -109,17 +110,22 @@ async function getAnnotationKeys({ page, preference, options, }) {
         });
         const annotations = Object.entries(
         // @ts-ignore
-        window.annotationInstance.annotations).map(([key, value]) => ({
-            elementID: key,
-            text: 
-            //@ts-ignore
-            value.node.innerText?.trim() ||
-                //@ts-ignore
-                value.node.placeholder?.trim() ||
-                "NA",
-        }));
-        return annotations;
+        window.annotationInstance.annotations).map(([key, value]) => {
+            return {
+                elementID: key,
+                innerText: value.node instanceof HTMLElement ? value.node.innerText : null,
+                innerHTML: value.node.innerHTML,
+                outerHTML: value.node.outerHTML,
+                placeholder: value.node instanceof HTMLInputElement ||
+                    value.node instanceof HTMLTextAreaElement
+                    ? value.node.placeholder
+                    : null,
+            };
+        });
+        const fullPageHTML = document.documentElement.outerHTML;
+        return { annotations, fullPageHTML };
     }, { preference, options });
+    const annotationKeys = await enrichAnnotations(annotatedElements.annotations, annotatedElements.fullPageHTML, trace);
     const annotationBuffer = await page.screenshot({
     // path: `screenshots/screenshot-${screenshotIndex++}.png`,
     });
@@ -145,3 +151,30 @@ async function getAnnotationKeys({ page, preference, options, }) {
     };
 }
 exports.getAnnotationKeys = getAnnotationKeys;
+async function enrichAnnotations(annotatedElements, pageHtml, trace) {
+    // create icon descriptor span
+    const iconDescriptionSpan = trace?.span({
+        name: "describe-icons",
+    });
+    const results = [];
+    for (const element of annotatedElements) {
+        const text = element.innerText?.trim() || element.placeholder?.trim();
+        if (text) {
+            results.push({
+                elementID: element.elementID,
+                text,
+            });
+            continue;
+        }
+        const description = await (0, icon_descriptor_1.getIconDescription)({
+            htmlString: element.outerHTML,
+            pageHtml: pageHtml,
+            trace: iconDescriptionSpan,
+        });
+        results.push({
+            elementID: element.elementID,
+            text: description || "NA",
+        });
+    }
+    return results;
+}

package/dist/agent/master/icon-descriptor/index.d.ts

@@ -0,0 +1,22 @@
+import { TraceClient } from "@empiricalrun/llm";
+type IconKnowledge = {
+    description: string;
+    key: string;
+    normalized: string;
+    createdAt: Date;
+};
+export declare function loadIconsKnowledge(): Array<IconKnowledge>;
+export declare function saveIconsKnowledge(iconsData: Array<IconKnowledge>): Promise<void>;
+export declare function generateKey(htmlString: string): string;
+export declare function reverseKey(hash: string): string;
+export declare function createNodeFromHTML(htmlString: string): {
+    node: Element | null;
+    children: Element[];
+};
+export declare function getIconDescription({ htmlString, pageHtml, trace, }: {
+    htmlString: string;
+    trace?: TraceClient;
+    pageHtml: string;
+}): Promise<string | undefined>;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/master/icon-descriptor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/icon-descriptor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAWrD,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,IAAI,CAAC;CACjB,CAAC;AAEF,wBAAgB,kBAAkB,IAAI,KAAK,CAAC,aAAa,CAAC,CAczD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,GAC9B,OAAO,CAAC,IAAI,CAAC,CAGf;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG/C;AAED,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM;;;EAUpD;AAqKD,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,QAAQ,EACR,KAAK,GACN,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAkC9B"}