@empiricalrun/test-gen 0.38.17 → 0.38.19

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.38.19
+
+### Patch Changes
+
+- ba654cf: fix: revert annotations
+
+## 0.38.18
+
+### Patch Changes
+
+- 683da08: fix: replaced master-agent-action with master-agent-execute-action in trace
+- ebf4861: fix: hover tool call
+
 ## 0.38.17
 
 ### Patch Changes

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAqD9B,wBAAwB,IAAI,YAAY,EAAE;IAmB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAqD9B,wBAAwB,IAAI,YAAY,EAAE;IAoB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}

package/dist/actions/index.js

@@ -93,6 +93,7 @@ class PlaywrightActions {
             done_1.doneActionGenerator,
             assert_1.assertTextVisibilityActionGenerator,
             text_content_1.textContentActionGenerator,
+            hover_1.hoverActionGenerator,
         ]
             .map((a) => a(this.page, {
             stateVariables: this.stateVariables,

package/dist/agent/master/run.d.ts

@@ -16,10 +16,7 @@ export declare function getNextAction({ task, executedActions, failedActions, pa
     actions: PlaywrightActions;
     disableSkills: boolean;
     useHints: boolean;
-    annotations?: {
-        elementID: string;
-        text: string;
-    }[];
+    annotations?: string[];
 }): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;
 export declare function createTestUsingMasterAgent({ task, page, testCase, options, scopeVars, }: {
     task: string;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAclD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAoBrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACrD,2FA6FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAiUA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAclD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAoBrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FA2FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAsTA"}

package/dist/agent/master/run.js

@@ -52,10 +52,8 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
         failedActions: failedActions.map((a) => a).join("\n"),
         executedActions: executedActions.map((a) => a).join("\n"),
         pageUrl,
-        annotations: annotations
-            ?.map((a) => `${a.elementID}:${a.text}`)
-            .join("\n"),
-    }, 27);
+        annotations,
+    }, 24);
     // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
     const userMessage = promptMessages.filter((m) => m.role === "user")[0];
     const systemMessage = promptMessages.filter((m) => m.role === "system")[0];
@@ -212,18 +210,8 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                 // @ts-ignore
                 // eslint-disable-next-line no-undef
                 window.annotationInstance = annotateClickableElements();
-                const annotations = Object.entries(
                 // @ts-ignore
-                window.annotationInstance.annotations).map(([key, value]) => ({
-                    elementID: key, // Assign the key to elementID
-                    text: 
-                    //@ts-ignore
-                    value.node.text?.trim() ||
-                        //@ts-ignore
-                        value.node.textContent?.trim() ||
-                        "<This is an icon or image. Check the screenshot>",
-                }));
-                return annotations;
+                return Object.keys(window.annotationInstance.annotations);
             });
             await page.waitForTimeout(2000);
             const annonationBuffer = await page.screenshot({
@@ -269,7 +257,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         if (toolCall) {
             const args = JSON.parse(toolCall.function.arguments);
             const masterAgentActionSpan = masterAgentSpan?.span({
-                name: "master-agent-action",
+                name: "master-agent-execute-action",
             });
             try {
                 output = {

package/dist/agent/master/with-hints.d.ts

@@ -13,10 +13,7 @@ export declare const triggerHintsFlow: ({ outputFromGetNextAction, generatedAnno
         action: string;
         elementAnnotation?: string;
     };
-    generatedAnnotations: {
-        elementID: string;
-        text: string;
-    }[];
+    generatedAnnotations: Record<string, any>;
     page: TestGenPage;
     llm: LLM;
     trace?: TraceClient | undefined;

package/dist/agent/master/with-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;UACrD,WAAW;SACZ,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAwGA,CAAC"}
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAO5B,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,WAAW;SACZ,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAwGA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.triggerHintsFlow = exports.getUserMessageWithForHints = void 0;
 const vision_1 = require("@empiricalrun/llm/vision");
 const actions_1 = require("../../actions");
+const click_1 = require("../../actions/click");
+const fill_1 = require("../../actions/fill");
+const hover_1 = require("../../actions/hover");
 const constants_1 = require("../../constants");
 const getUserMessageWithForHints = ({ userMessage, options, pageScreenshot, annotatedPageScreenshot, }) => {
     return [
@@ -37,7 +40,7 @@ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations,
     try {
         const hasElementAnnotation = outputFromGetNextAction?.elementAnnotation?.length &&
             outputFromGetNextAction?.elementAnnotation?.trim()?.length &&
-            generatedAnnotations.some((annotation) => annotation.elementID === outputFromGetNextAction?.elementAnnotation);
+            generatedAnnotations?.includes(outputFromGetNextAction?.elementAnnotation);
         trace?.event({
             name: "has-element-annotation",
             output: {
@@ -87,8 +90,9 @@ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations,
             },
             trace,
         });
-        const canTriggerHintsFlow = completion?.tool_calls?.some((currentToolCall) => currentToolCall.function.name === "fill_input_element" ||
-            currentToolCall.function.name === "click_element");
+        const canTriggerHintsFlow = completion?.tool_calls?.some((currentToolCall) => currentToolCall.function.name === fill_1.PLAYWRIGHT_FILL_ACTION_NAME ||
+            currentToolCall.function.name === click_1.PLAYWRIGHT_CLICK_ACTION_NAME ||
+            currentToolCall.function.name === hover_1.PLAYWRIGHT_HOVER_ACTION_NAME);
         trace?.event({
             name: "can-trigger-hints-flow",
             output: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.38.17",
+  "version": "0.38.19",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"