@empiricalrun/test-gen 0.38.27 → 0.38.29

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @empiricalrun/test-gen
 
+## 0.38.29
+
+### Patch Changes
+
+- 32ef92e: fix: removed master agent v1
+
+## 0.38.28
+
+### Patch Changes
+
+- dc8d538: fix: ordering of npm install
+
 ## 0.38.27
 
 ### Patch Changes

package/dist/agent/master/element-annotation.d.ts

@@ -0,0 +1,11 @@
+import { LLM, TraceClient } from "@empiricalrun/llm";
+import { BrowsingAgentOptions } from "../browsing";
+export declare function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, }: {
+    elementDescription: string;
+    annotations: string;
+    annotatedScreenshot: string;
+    trace?: TraceClient;
+    llm?: LLM;
+    options?: BrowsingAgentOptions;
+}): Promise<string | undefined>;
+//# sourceMappingURL=element-annotation.d.ts.map

package/dist/agent/master/element-annotation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AASrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAgDnD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,GACR,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA4F9B"}

package/dist/agent/master/element-annotation.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getElementAnnotation = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const vision_1 = require("@empiricalrun/llm/vision");
+const constants_1 = require("../../constants");
+const annotationToolAction = {
+    name: "element_annotation",
+    schema: {
+        type: "function",
+        function: {
+            name: "element_annotation",
+            description: "Handles annotations for elements and enriched annotations.",
+            parameters: {
+                type: "object",
+                properties: {
+                    enriched_annotations: {
+                        type: "array",
+                        description: "List of enriched annotations for elements.",
+                        items: {
+                            type: "object",
+                            properties: {
+                                element_id: {
+                                    type: "string",
+                                    description: "Unique element ID for the element.",
+                                },
+                                description: {
+                                    type: "string",
+                                    description: "Detailed description for the element. Must not be NA",
+                                },
+                            },
+                            required: ["element_id", "description"],
+                        },
+                    },
+                    reason: {
+                        type: "string",
+                        description: "Explain why this element is selected. The reason should be clear and align with the task or purpose.",
+                    },
+                    element_annotation: {
+                        type: "string",
+                        description: "Return the unique element ID for the element on which the action needs to be performed.",
+                    },
+                },
+                required: ["enriched_annotations", "reason", "element_annotation"],
+            },
+        },
+    },
+};
+async function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, }) {
+    const annotationsSpan = trace?.span({
+        name: "get-element-annotation",
+        input: {
+            elementDescription,
+            annotations,
+            annotatedScreenshot,
+        },
+    });
+    const systemMessage = {
+        role: "system",
+        content: `You are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements present in the image.
+  
+  You will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation is done by drawing a red box around the element and a small yellow box on it which contains unique element id.
+  
+  You are given a Annotations which contains list of  unique element id  and description of the element separated by ":". 
+  
+  You are also given the description of the element on which the action needs to be taken. The description includes information about how the element looks, it's position etc.
+  
+  Your task is to provide the annotation of the element on which the action needs to be performed based on the element description.
+  
+  Follow steps to fulfil your task:
+  - Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each element. 
+  - For describing each element Id
+  -- iterate over each element Id in annotation list
+  -- check if the description is already present for the element Id in the Annotation provided to you. If present skip describing it and use it as is.
+  -- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or icon enclosed in the element. 
+  - Respond with the mapped element Ids as "enriched_annotations"
+  - Based on the description provided to you and the enriched annotations, first identify the element Id whose description matches the task provided
+  
+  Note:
+  - Ensure providing the description of all the elements in the list.
+  - Don't update the description if its already present in the given annotations
+  - Replace all the "NA" with description of the element. Its position, how does it look like etc. 
+  - There should be no "NA" present in any of the element description
+  `,
+    };
+    const userMessage = {
+        role: "user",
+        content: [
+            {
+                type: "text",
+                text: `
+  Element description:
+  ${elementDescription}
+  
+  Annotations:
+  ${annotations}`,
+            },
+            {
+                type: "image_url",
+                image_url: {
+                    url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, annotatedScreenshot),
+                },
+            },
+        ],
+    };
+    const messages = [
+        systemMessage,
+        userMessage,
+    ];
+    llm =
+        llm ||
+            new llm_1.LLM({
+                provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+                defaultModel: options?.model || constants_1.DEFAULT_MODEL,
+            });
+    const completion = await llm.createChatCompletion({
+        messages,
+        modelParameters: {
+            ...constants_1.DEFAULT_MODEL_PARAMETERS,
+            ...options?.modelParameters,
+            tool_choice: "required",
+            temperature: 1,
+        },
+        trace: annotationsSpan,
+        traceName: "get-element-from-action",
+        //@ts-ignore
+        tools: [annotationToolAction.schema],
+    });
+    const toolCall = completion?.tool_calls?.[0];
+    annotationsSpan?.end({ output: toolCall });
+    const args = JSON.parse(toolCall.function.arguments);
+    return args.element_annotation;
+}
+exports.getElementAnnotation = getElementAnnotation;

package/dist/agent/master/next-action.d.ts

@@ -0,0 +1,19 @@
+import { LLM, TraceClient } from "@empiricalrun/llm";
+import { PlaywrightActions } from "../../actions";
+import { BrowsingAgentOptions } from "../browsing";
+export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, disableSkills, useHints, }: {
+    task: string;
+    executedActions: string[];
+    failedActions: any[];
+    pageUrl: string;
+    trace?: TraceClient;
+    llm?: LLM;
+    options?: BrowsingAgentOptions;
+    pageScreenshot: string;
+    annotatedPageScreenshot?: string;
+    actions: PlaywrightActions;
+    disableSkills: boolean;
+    useHints: boolean;
+    annotations?: string[];
+}): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;
+//# sourceMappingURL=next-action.d.ts.map

package/dist/agent/master/next-action.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FAkKA"}

package/dist/agent/master/next-action.js

@@ -0,0 +1,161 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getNextAction = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const vision_1 = require("@empiricalrun/llm/vision");
+const skill_1 = require("../../actions/skill");
+const constants_1 = require("../../constants");
+async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, disableSkills, useHints = false, }) {
+    const nextActionSpan = trace?.span({
+        name: "master-agent-next-action",
+        input: {
+            task,
+            executedActions,
+            failedActions,
+            pageUrl,
+            options,
+            pageScreenshot,
+            disableSkills,
+            useHints,
+            skills: skill_1.testCaseSkills.getAvailableSkills(),
+        },
+    });
+    const promptSpan = nextActionSpan?.span({
+        name: "master-agent-prompt",
+    });
+    const systemMessage = {
+        role: "system",
+        content: `You are an web automation tool which is given a task to complete. You need to execute the task provided to you with the help of web page screenshot, a browser automation tool or skills which are learnt while writing previous tests. 
+  
+  Browser automation tool is a tool which uses Playwright and browser to execute action using next_action tool call.
+  Skill usage is a tool which helps to execute previously known pieces of code to achieve a task.
+  
+  You will be provided with a screenshot of the webpage which you will use to extract the action that needs to be taken.
+  
+  You will be provided with previously executed actions by the browser automation tool and based on the current screenshot and previously executed actions, you need to predict the next action to be taken.
+  
+  You will also be provided with failed next action predicted by you, so that you can avoid suggesting the same action again - which failed.
+  
+  The next action should be as atomic as possible.
+  e.g: click on an element, fill an input element, assert, extract text from an element are valid next action as they are atomic in nature.
+  
+  You will also be provided with skill usage tool which you can use to execute action. These skills are compound functions which helps you to complete your action.
+  
+  You need to respond with either:
+  - Next action to be taken by a browser automation tool 
+  - Use previously learnt skills in the form of tool call.
+  
+  You need to make a decision whether the given skill can be reused if "YES" respond with the skill else respond with the next action.`,
+    };
+    const userMessage = {
+        role: "user",
+        content: [
+            {
+                type: "text",
+                text: `Task:
+          ${task}
+          
+          ----
+          
+          Previous executed actions:
+          ${executedActions.map((a) => a).join("\n")}
+          
+          ----
+          
+          Previous failed actions:
+          ${failedActions.map((a) => a).join("\n")}
+          
+          ----
+          You are also provided with a page screenshot for you to decide the next action.
+          
+          Current page URL: ${pageUrl}
+          
+          
+          Follow the instructions before responding:
+          - Divide the task into sub tasks
+          - Using previously executed actions, identify tasks are complete and which tasks needs to be executed next.
+          - You will be provided a skill usage action, if the testStep matches the next action then respond with the skill usage.
+          - If responding with next action, ensure next action to be detailed and explicit about what action needs to be done. Provide all the information which can be extracted from the screenshot as a part of next action.
+          - Mark task as complete only when executed actions provided to you indicates that the task is done.
+          - Refer to the text and references available in the screenshot to create the next action.
+          - Do not take any extra actions which are not required for the execution of the task
+          - If there are no further actions required based on the task, then respond with task as done.
+          - Do not recommend actions which are not available in the screenshot
+          `,
+            },
+            {
+                type: "text",
+                text: "Screenshot in normal mode 👇",
+            },
+            {
+                type: "image_url",
+                image_url: {
+                    url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
+                },
+            },
+        ],
+    };
+    const messages = [
+        systemMessage,
+        userMessage,
+    ];
+    const actionSchemas = disableSkills || skill_1.testCaseSkills.getAvailableSkills().length === 0
+        ? []
+        : actions.getMasterActionSchemas();
+    const actionToolCall = {
+        name: "next_task",
+        schema: {
+            type: "function",
+            function: {
+                name: "next_task",
+                description: "take the next action base on the provided task",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        reason: {
+                            type: "string",
+                            description: "explain how this action will help to complete the task. the reason should align with the task provided",
+                        },
+                        action: {
+                            type: "string",
+                            description: `explain the next action in natural language.
+    The next action should be as atomic as possible, precise and should contain enough details about the action to be performed.
+    E.g. each click, key press, input, assert should be a separate action.
+    Each action should take the task to completion, if not the action is invalid.`,
+                        },
+                        element_description: {
+                            type: "string",
+                            description: "The description of the element on which action needs to be taken, including its position, appearance, etc.",
+                        },
+                    },
+                    required: ["reason", "action", "element_description"],
+                },
+            },
+        },
+    };
+    const tools = [actionToolCall.schema, ...actionSchemas];
+    promptSpan?.end({ output: { messages } });
+    llm =
+        llm ||
+            new llm_1.LLM({
+                provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+                defaultModel: options?.model || constants_1.DEFAULT_MODEL,
+            });
+    const completion = await llm.createChatCompletion({
+        messages,
+        modelParameters: {
+            ...constants_1.DEFAULT_MODEL_PARAMETERS,
+            ...options?.modelParameters,
+            tool_choice: "required",
+            temperature: 1,
+        },
+        trace: nextActionSpan,
+        traceName: "master-agent-llm",
+        // @ts-ignore
+        tools,
+    });
+    const toolCall = completion?.tool_calls?.[0];
+    nextActionSpan?.end({ output: toolCall });
+    return toolCall;
+}
+exports.getNextAction = getNextAction;

package/dist/agent/master/run.d.ts

@@ -1,23 +1,6 @@
-import { LLM, TraceClient } from "@empiricalrun/llm";
 import { Page } from "playwright";
-import { PlaywrightActions } from "../../actions";
 import { ScopeVars, TestCase } from "../../types";
 import { BrowsingAgentOptions } from "../browsing";
-export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints, annotations, }: {
-    task: string;
-    executedActions: string[];
-    failedActions: any[];
-    pageUrl: string;
-    trace?: TraceClient;
-    llm?: LLM;
-    options?: BrowsingAgentOptions;
-    pageScreenshot: string;
-    annotatedPageScreenshot?: string;
-    actions: PlaywrightActions;
-    disableSkills: boolean;
-    useHints: boolean;
-    annotations?: string[];
-}): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;
 export declare function createTestUsingMasterAgent({ task, page, testCase, options, scopeVars, }: {
     task: string;
     page: Page;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAclD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAoBrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FA2FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAuTA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAsBrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA6VA"}