@empiricalrun/test-gen 0.42.4 → 0.42.8

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @empiricalrun/test-gen
 
+## 0.42.8
+
+### Patch Changes
+
+- 18444bc: fix: pass action to BA in case of no annotations
+
+## 0.42.7
+
+### Patch Changes
+
+- b6879bb: test: added assert scenario for annotations
+
+## 0.42.6
+
+### Patch Changes
+
+- b0b578c: fix: added page for token debugging
+- 25ae4b6: fix: added preference in trace for annotations
+- 9247f19: fix: assertion capability with selector hints
+
+## 0.42.5
+
+### Patch Changes
+
+- 8f5a315: fix: trace url is clickable
+- 02784a8: chore: move browser-injected-scripts to enable lint
+- Updated dependencies [4df759f]
+- Updated dependencies [1749342]
+  - @empiricalrun/llm@0.9.30
+
 ## 0.42.4
 
 ### Patch Changes

package/dist/agent/codegen/repo-edit.js

@@ -125,7 +125,7 @@ exports.generateCodeUsingRepoAgent = generateCodeUsingRepoAgent;
 async function repoEditAgent({ trace, task, logger, }) {
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     void testgenUpdatesReporter.sendMessage(`Updating test code as per the task. \n View [trace](${trace?.getTraceUrl()})`);
-    logger?.log(`Starting repo agent: ${trace?.getTraceUrl()}`);
+    logger?.log(`Starting repo agent: [trace](${trace?.getTraceUrl()})`);
     const { prompt: repoFiles } = await (0, context_1.generateTxtForRepository)();
     const repoAgentOutput = await generateCodeUsingRepoAgent({
         task,

package/dist/agent/master/action-tool-calls.d.ts

@@ -0,0 +1,35 @@
+export declare enum ActionType {
+    FILL = "fill",
+    PAGE_GOTO = "page_goto",
+    CLICK = "click",
+    PRESS_ACTION = "keyboard_press_on_element",
+    ASSERT_TEXT = "assert_text",
+    HOVER = "hover_element"
+}
+export declare function isValidActionType(value: string): value is ActionType;
+export declare function getActionToolCalls(): {
+    type: string;
+    function: {
+        name: string;
+        description: string;
+        parameters: {
+            type: string;
+            properties: {
+                reason: {
+                    type: string;
+                    description: string;
+                };
+                action: {
+                    type: string;
+                    description: string;
+                };
+                element_description: {
+                    type: string;
+                    description: string;
+                };
+            };
+            required: string[];
+        };
+    };
+}[];
+//# sourceMappingURL=action-tool-calls.d.ts.map

package/dist/agent/master/action-tool-calls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"action-tool-calls.d.ts","sourceRoot":"","sources":["../../../src/agent/master/action-tool-calls.ts"],"names":[],"mappings":"AAAA,oBAAY,UAAU;IACpB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,YAAY,8BAA8B;IAC1C,WAAW,gBAAgB;IAC3B,KAAK,kBAAkB;CACxB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,UAAU,CAEpE;AAkDD,wBAAgB,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;IAyBjC"}

package/dist/agent/master/action-tool-calls.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getActionToolCalls = exports.isValidActionType = exports.ActionType = void 0;
+var ActionType;
+(function (ActionType) {
+    ActionType["FILL"] = "fill";
+    ActionType["PAGE_GOTO"] = "page_goto";
+    ActionType["CLICK"] = "click";
+    ActionType["PRESS_ACTION"] = "keyboard_press_on_element";
+    ActionType["ASSERT_TEXT"] = "assert_text";
+    ActionType["HOVER"] = "hover_element";
+})(ActionType || (exports.ActionType = ActionType = {}));
+function isValidActionType(value) {
+    return Object.values(ActionType).includes(value);
+}
+exports.isValidActionType = isValidActionType;
+const createActionCall = (name, description, additionalProperties = {}) => ({
+    name: "next_task",
+    description: "take the next action based on the provided task",
+    schema: {
+        type: "function",
+        function: {
+            name,
+            description,
+            parameters: {
+                type: "object",
+                properties: {
+                    reason: {
+                        type: "string",
+                        description: "explain how this action will help to complete the task. the reason should align with the task provided",
+                    },
+                    action: {
+                        type: "string",
+                        description: `explain the next action in natural language.
+                The next action should be as atomic as possible, precise and should contain enough details about the action to be performed.
+                E.g. each click, key press, input, assert should be a separate action.
+                Each action should take the task to completion, if not the action is invalid.`,
+                    },
+                    element_description: {
+                        type: "string",
+                        description: "The description of the element on which action needs to be taken, including its position, appearance, etc.",
+                    },
+                    ...additionalProperties,
+                },
+                required: [
+                    "reason",
+                    "action",
+                    "element_description",
+                    ...Object.keys(additionalProperties),
+                ],
+            },
+        },
+    },
+});
+function getActionToolCalls() {
+    return [
+        createActionCall(ActionType.ASSERT_TEXT, "assert whether the given element on the page is visible", {
+            assertion_text: {
+                type: "string",
+                description: "The text for which the visibility needs to be asserted.",
+            },
+        }),
+        createActionCall(ActionType.FILL, "fill the input element with a particular text"),
+        createActionCall(ActionType.CLICK, "click an element on the page"),
+        createActionCall(ActionType.PAGE_GOTO, "navigate the page to a new URL"),
+        createActionCall(ActionType.PRESS_ACTION, "Focuses the matching element and presses a combination of the keys"),
+        createActionCall(ActionType.HOVER, "hover over an element on the page"),
+    ].map((call) => call.schema);
+}
+exports.getActionToolCalls = getActionToolCalls;

package/dist/agent/master/element-annotation.d.ts

@@ -1,15 +1,20 @@
 import { LLM, TraceClient } from "@empiricalrun/llm";
 import { Page } from "playwright";
 import { BrowsingAgentOptions } from "../browsing";
-export declare function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, }: {
+import { ActionType } from "./action-tool-calls";
+export declare function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, preference, }: {
     elementDescription: string;
     annotations: string;
     annotatedScreenshot: string;
     trace?: TraceClient;
     llm?: LLM;
     options?: BrowsingAgentOptions;
+    preference: AnnotationPreference;
 }): Promise<string | undefined>;
-export type AnnotationPreference = "all" | "fill";
+export type AnnotationPreference = {
+    actionType: "all" | ActionType.FILL | ActionType.ASSERT_TEXT;
+    assertionText?: string | undefined;
+};
 export declare function getAnnotationKeys({ page, preference, options, }: {
     page: Page;
     preference: AnnotationPreference;

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAgDnD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,GACR,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA4F9B;AAED,MAAM,MAAM,oBAAoB,GAAG,KAAK,GAAG,MAAM,CAAC;AAElD,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAgDjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA6F9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,KAAK,GAAG,UAAU,CAAC,IAAI,GAAG,UAAU,CAAC,WAAW,CAAC;IAC7D,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}

package/dist/agent/master/element-annotation.js

@@ -46,13 +46,14 @@ const annotationToolAction = {
         },
     },
 };
-async function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, }) {
+async function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, preference, }) {
     const annotationsSpan = trace?.span({
         name: "get-element-annotation",
         input: {
             elementDescription,
             annotations,
             annotatedScreenshot,
+            preference,
         },
     });
     const systemMessage = {

package/dist/agent/master/next-action.d.ts

@@ -1,16 +1,8 @@
 import { LLM, TraceClient } from "@empiricalrun/llm";
 import { PlaywrightActions } from "../../actions";
 import { BrowsingAgentOptions } from "../browsing";
-export declare enum actionTypes {
-    PLAYWRIGHT_FILL_ACTION_NAME = "fill_input_element",
-    PLAYWRIGHT_GOTO_ACTION_NAME = "page_goto",
-    PLAYWRIGHT_CLICK_ACTION_NAME = "click_element",
-    PLAYWRIGHT_PRESS_ACTION_NAME = "keyboard_press_on_element",
-    PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = "assert_text_visibility",
-    PLAYWRIGHT_HOVER_ACTION_NAME = "hover_element"
-}
-export declare function isValidActionType(value: string): value is actionTypes;
-export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, disableSkills, }: {
+import { ActionType } from "./action-tool-calls";
+export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, actionTypes, disableSkills, }: {
     task: string;
     executedActions: string[];
     failedActions: any[];
@@ -21,6 +13,7 @@ export declare function getNextAction({ task, executedActions, failedActions, pa
     pageScreenshot: string;
     annotatedPageScreenshot?: string;
     actions: PlaywrightActions;
+    actionTypes: typeof ActionType;
     disableSkills: boolean;
     annotations?: string[];
 }): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;

package/dist/agent/master/next-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,oBAAY,WAAW;IACrB,2BAA2B,uBAAuB;IAClD,2BAA2B,cAAc;IACzC,4BAA4B,kBAAkB;IAC9C,4BAA4B,8BAA8B;IAC1D,6CAA6C,2BAA2B;IACxE,4BAA4B,kBAAkB;CAC/C;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,WAAW,CAErE;AAED,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,aAAa,GACd,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FA0KA"}
+{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,UAAU,EAAsB,MAAM,qBAAqB,CAAC;AAErE,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,WAAW,EACX,aAAa,GACd,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,EAAE,OAAO,UAAU,CAAC;IAC/B,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FAmIA"}

package/dist/agent/master/next-action.js

@@ -1,24 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getNextAction = exports.isValidActionType = exports.actionTypes = void 0;
+exports.getNextAction = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
 const skill_1 = require("../../actions/skill");
 const constants_1 = require("../../constants");
-var actionTypes;
-(function (actionTypes) {
-    actionTypes["PLAYWRIGHT_FILL_ACTION_NAME"] = "fill_input_element";
-    actionTypes["PLAYWRIGHT_GOTO_ACTION_NAME"] = "page_goto";
-    actionTypes["PLAYWRIGHT_CLICK_ACTION_NAME"] = "click_element";
-    actionTypes["PLAYWRIGHT_PRESS_ACTION_NAME"] = "keyboard_press_on_element";
-    actionTypes["PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME"] = "assert_text_visibility";
-    actionTypes["PLAYWRIGHT_HOVER_ACTION_NAME"] = "hover_element";
-})(actionTypes || (exports.actionTypes = actionTypes = {}));
-function isValidActionType(value) {
-    return Object.values(actionTypes).includes(value);
-}
-exports.isValidActionType = isValidActionType;
-async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, disableSkills, }) {
+const action_tool_calls_1 = require("./action-tool-calls");
+async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, actionTypes, disableSkills, }) {
     const nextActionSpan = trace?.span({
         name: "master-agent-next-action",
         input: {
@@ -117,44 +105,7 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
     const actionSchemas = disableSkills || skill_1.testCaseSkills.getAvailableSkills().length === 0
         ? []
         : actions.getMasterActionSchemas();
-    const actionToolCall = {
-        name: "next_task",
-        schema: {
-            type: "function",
-            function: {
-                name: "next_task",
-                description: "take the next action base on the provided task",
-                parameters: {
-                    type: "object",
-                    properties: {
-                        reason: {
-                            type: "string",
-                            description: "explain how this action will help to complete the task. the reason should align with the task provided",
-                        },
-                        action: {
-                            type: "string",
-                            description: `explain the next action in natural language.
-    The next action should be as atomic as possible, precise and should contain enough details about the action to be performed.
-    E.g. each click, key press, input, assert should be a separate action.
-    Each action should take the task to completion, if not the action is invalid.`,
-                        },
-                        action_type: {
-                            type: "string",
-                            enum: Object.values(actionTypes),
-                            description: `type of the action that needs to be taken.
-              Any other action type than the provided action type is invalid.`,
-                        },
-                        element_description: {
-                            type: "string",
-                            description: "The description of the element on which action needs to be taken, including its position, appearance, etc.",
-                        },
-                    },
-                    required: ["reason", "action", "element_description", "action_type"],
-                },
-            },
-        },
-    };
-    const tools = [actionToolCall.schema, ...actionSchemas];
+    const tools = [...(0, action_tool_calls_1.getActionToolCalls)(), ...actionSchemas];
     promptSpan?.end({ output: { messages } });
     llm =
         llm ||

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA0BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAmUA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA2BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAiVA"}

package/dist/agent/master/run.js

@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTestUsingMasterAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const actions_1 = require("../../actions");
-const fill_1 = require("../../actions/fill");
 const skill_1 = require("../../actions/skill");
 const utils_1 = require("../../actions/utils");
 const logger_1 = require("../../bin/logger");
@@ -15,6 +14,7 @@ const browsing_1 = require("../browsing");
 const utils_2 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
 const run_time_planner_1 = require("../planner/run-time-planner");
+const action_tool_calls_1 = require("./action-tool-calls");
 const element_annotation_1 = require("./element-annotation");
 const next_action_1 = require("./next-action");
 const with_hints_1 = require("./with-hints");
@@ -136,6 +136,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
             options,
             pageScreenshot,
             actions,
+            actionTypes: action_tool_calls_1.ActionType,
             disableSkills,
         });
         if (await (0, session_1.shouldStopSession)()) {
@@ -165,15 +166,28 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                     let hintsExecutionCompletion;
                     let elementAnnotation;
                     await page.waitForTimeout(2000);
-                    const actionType = JSON.parse(toolCall.function.arguments).action_type;
-                    let preference = "all";
-                    if (useActionSpecificAnnotations && (0, next_action_1.isValidActionType)(actionType)) {
+                    const nextActionToolCall = JSON.parse(toolCall.function.arguments);
+                    const actionType = toolCall.function.name;
+                    let preference = {
+                        actionType: "all",
+                    };
+                    if (useActionSpecificAnnotations && (0, action_tool_calls_1.isValidActionType)(actionType)) {
                         switch (actionType) {
-                            case fill_1.PLAYWRIGHT_FILL_ACTION_NAME:
-                                preference = "fill";
+                            case action_tool_calls_1.ActionType.FILL:
+                                preference = {
+                                    actionType: action_tool_calls_1.ActionType.FILL,
+                                };
+                                break;
+                            case action_tool_calls_1.ActionType.ASSERT_TEXT:
+                                preference = {
+                                    actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
+                                    assertionText: nextActionToolCall.assertion_text,
+                                };
                                 break;
                             default:
-                                preference = "all";
+                                preference = {
+                                    actionType: "all",
+                                };
                         }
                     }
                     let { annotationKeys, annotatedPageScreenshot, annotationBuffer } = await (0, element_annotation_1.getAnnotationKeys)({
@@ -181,42 +195,45 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                         preference,
                         options,
                     });
-                    const annotationMapString = annotationKeys
-                        ?.map((a) => `${a.elementID}:${a.text}`)
-                        .join("\n");
-                    // Provides the annotations for all the element present on screen
-                    // Also provides the annotation of element on which the action needs to be taken
-                    elementAnnotation = await (0, element_annotation_1.getElementAnnotation)({
-                        elementDescription: JSON.parse(toolCall.function.arguments)
-                            .element_description,
-                        annotations: annotationMapString,
-                        annotatedScreenshot: annotatedPageScreenshot,
-                        trace: masterAgentActionSpan,
-                        llm,
-                    });
-                    output.elementAnnotation = elementAnnotation;
-                    console.log("Output: ", output);
-                    await testGenReporter.sendCurrentView(annotationBuffer);
-                    const triggerHintsFlowSpan = masterAgentActionSpan?.span({
-                        name: "trigger-hints-flow",
-                        input: {
+                    if (annotationKeys.length > 0) {
+                        const annotationMapString = annotationKeys
+                            ?.map((a) => `${a.elementID}:${a.text}`)
+                            .join("\n");
+                        // Provides the annotations for all the element present on screen
+                        // Also provides the annotation of element on which the action needs to be taken
+                        elementAnnotation = await (0, element_annotation_1.getElementAnnotation)({
+                            elementDescription: JSON.parse(toolCall.function.arguments)
+                                .element_description,
+                            annotations: annotationMapString,
+                            annotatedScreenshot: annotatedPageScreenshot,
+                            trace: masterAgentActionSpan,
+                            llm,
+                            preference,
+                        });
+                        output.elementAnnotation = elementAnnotation;
+                        console.log("Output: ", output);
+                        await testGenReporter.sendCurrentView(annotationBuffer);
+                        const triggerHintsFlowSpan = masterAgentActionSpan?.span({
+                            name: "trigger-hints-flow",
+                            input: {
+                                outputFromGetNextAction: output,
+                                generatedAnnotations: annotationKeys,
+                            },
+                        });
+                        // Provides the action whether its a click, fill etc.
+                        const result = await (0, with_hints_1.triggerHintsFlow)({
                             outputFromGetNextAction: output,
                             generatedAnnotations: annotationKeys,
-                        },
-                    });
-                    // Provides the action whether its a click, fill etc.
-                    const result = await (0, with_hints_1.triggerHintsFlow)({
-                        outputFromGetNextAction: output,
-                        generatedAnnotations: annotationKeys,
-                        actions,
-                        llm,
-                        trace: triggerHintsFlowSpan,
-                    });
-                    shouldTriggerHintsFlow = result.shouldTriggerHintsFlow;
-                    hintsExecutionCompletion = result.hintsExecutionCompletion;
-                    triggerHintsFlowSpan?.end({
-                        output: result,
-                    });
+                            actions,
+                            llm,
+                            trace: triggerHintsFlowSpan,
+                        });
+                        shouldTriggerHintsFlow = result.shouldTriggerHintsFlow;
+                        hintsExecutionCompletion = result.hintsExecutionCompletion;
+                        triggerHintsFlowSpan?.end({
+                            output: result,
+                        });
+                    }
                     if (shouldTriggerHintsFlow && hintsExecutionCompletion) {
                         const toolCalls = hintsExecutionCompletion?.tool_calls || [];
                         for (const i in toolCalls) {

package/dist/agent/master/with-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAQlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;aAChC,iBAAiB;SACrB,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAuGA,CAAC"}
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AASlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;aAChC,iBAAiB;SACrB,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAyGA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.triggerHintsFlow = exports.getUserMessageWithForHints = void 0;
 const vision_1 = require("@empiricalrun/llm/vision");
+const assert_1 = require("../../actions/assert");
 const click_1 = require("../../actions/click");
 const fill_1 = require("../../actions/fill");
 const hover_1 = require("../../actions/hover");
@@ -92,7 +93,9 @@ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations,
         });
         const canTriggerHintsFlow = completion?.tool_calls?.some((currentToolCall) => currentToolCall.function.name === fill_1.PLAYWRIGHT_FILL_ACTION_NAME ||
             currentToolCall.function.name === click_1.PLAYWRIGHT_CLICK_ACTION_NAME ||
-            currentToolCall.function.name === hover_1.PLAYWRIGHT_HOVER_ACTION_NAME);
+            currentToolCall.function.name === hover_1.PLAYWRIGHT_HOVER_ACTION_NAME ||
+            currentToolCall.function.name ===
+                assert_1.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME);
         trace?.event({
             name: "can-trigger-hints-flow",
             output: {