@empiricalrun/test-gen 0.38.29 → 0.38.30

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @empiricalrun/test-gen
 
+## 0.38.30
+
+### Patch Changes
+
+- dbe713b: fix: auto dismiss popup at test runtime
+
 ## 0.38.29
 
 ### Patch Changes

package/dist/actions/skill.d.ts

@@ -1,6 +1,6 @@
 import { PlaywrightActionGenerator } from "../types";
 export declare const SKILL_USAGE = "skill_usage";
-type Skill = {
+export type Skill = {
     testStep: string;
     filePath: string;
     usageExample: string;

package/dist/actions/skill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,KAAK,KAAK,GAAG;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,eAAO,MAAM,oBAAoB,EAAE,yBAmHlC,CAAC"}
+{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,MAAM,MAAM,KAAK,GAAG;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,eAAO,MAAM,oBAAoB,EAAE,yBAmHlC,CAAC"}

package/dist/agent/master/planner.d.ts

@@ -0,0 +1,15 @@
+import { TraceClient } from "@empiricalrun/llm";
+import { Page } from "playwright/test";
+export declare function runtimePlannerWithScreenshot({ trace, task, conversation, pages, page, currentPage, }: {
+    trace?: TraceClient;
+    conversation: string[];
+    task: string;
+    pages?: Record<string, any>;
+    page: Page;
+    currentPage?: string;
+}): Promise<{
+    pageName: string;
+    isDone: boolean;
+    reason: string;
+}>;
+//# sourceMappingURL=planner.d.ts.map

package/dist/agent/master/planner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../../../src/agent/master/planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAIvC,wBAAsB,4BAA4B,CAAC,EACjD,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GAwIA"}

package/dist/agent/master/planner.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runtimePlannerWithScreenshot = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const vision_1 = require("@empiricalrun/llm/vision");
+const constants_1 = require("../../constants");
+async function runtimePlannerWithScreenshot({ trace, task, conversation, pages, page, currentPage, }) {
+    const buffer = await page.screenshot({
+    //This is done to improve element annotation accuracy, anyways it doesn't annotate elements which are out of viewport
+    // fullPage: true,
+    // path: `screenshots/screenshot-${screenshotIndex++}.png`, // enable this and screenshotIndex var for local debugging
+    });
+    const pageBuffer = buffer.toString("base64");
+    const runTimePlannerSpan = trace?.span({
+        name: "popup-verifier",
+        input: {
+            task,
+            conversation,
+        },
+    });
+    const llm = new llm_1.LLM({ provider: "openai" });
+    const prompt = [
+        {
+            role: "system",
+            content: `
+  Given a successfully executed actions that lists only the actions that were successfully executed and a task comprising multiple actions, your goal is to analyse the list and determine if the entire task is completed.
+  These actions are executed by AI agents using Playwright on a browser. These agents already have access to browser tabs to execute actions. The successfully executed actions on browser post browser has opened, is provided to you as successfully executed actions.
+
+  You are also given a screenshot of the current screen which you can also use to determine whether the entire task is completed or not.
+  
+  If the task is not fully completed, identify which specific actions are missing and suggest next steps to complete the task. Assume that the conversation provided is entirely truthful and no additional actions were performed beyond those listed.
+  
+  To fulfil your goal, follow these steps:
+  - Divide the task into individual actions.
+  - Compare each task action against the actions listed in the successfully executed actions list.
+  - Identify which actions have been executed and which have not.
+  - If all actions are executed, respond with the task as done.
+  - If any actions are missing, respond with the task as not done, listing all actions and specifying which are complete and which are missing.
+  - If provided with list of pages, based on the next pending action and previously executed action, identify the page on which next action needs to be taken
+      `,
+        },
+        {
+            role: "user",
+            content: [
+                {
+                    type: "text",
+                    text: `
+  Task: ${task}
+  
+  ----
+  
+  Following are successfully executed actions:
+  ${conversation.join("\n")}
+  
+  ----
+  
+  Current page:
+  ${currentPage}
+          `,
+                },
+                {
+                    type: "text",
+                    text: "Page Screenshot",
+                },
+                {
+                    type: "image_url",
+                    image_url: {
+                        url: (0, vision_1.imageFormatForProvider)(constants_1.DEFAULT_MODEL_PROVIDER, pageBuffer),
+                    },
+                },
+            ],
+        },
+    ];
+    const response = await llm.createChatCompletion({
+        trace: runTimePlannerSpan,
+        traceName: "runtime-planner-llm",
+        model: "gpt-4o",
+        messages: prompt,
+        tools: [
+            {
+                type: "function",
+                function: {
+                    name: "task_done",
+                    description: "end the task by calling this method",
+                    parameters: {
+                        type: "object",
+                        properties: {
+                            actions: {
+                                type: "string",
+                                description: "actions extracted from task",
+                            },
+                            successful_actions: {
+                                type: "string",
+                                description: "successful actions mentioned in the conversation",
+                            },
+                            reason: {
+                                type: "string",
+                                description: "reasoning for identification of task status",
+                            },
+                            isDone: {
+                                type: "boolean",
+                                description: "whether the task is done",
+                            },
+                            pageName: {
+                                type: "string",
+                                enum: pages ? Object.keys(pages) : [],
+                                description: "page name for the next action.",
+                            },
+                        },
+                        required: ["isDone", "reason", "pageName"],
+                    },
+                },
+            },
+        ],
+        modelParameters: {
+            tool_choice: "required",
+            temperature: 0.5,
+        },
+    });
+    const toolCallResp = (response?.tool_calls || [])[0];
+    if (toolCallResp) {
+        const toolCall = JSON.parse(toolCallResp.function.arguments);
+        const output = {
+            pageName: toolCall.pageName,
+            isDone: toolCall.isDone,
+            reason: toolCall.reason,
+        };
+        runTimePlannerSpan?.end({
+            output,
+        });
+        return output;
+    }
+    const output = {
+        pageName: "",
+        isDone: false,
+        reason: "LLM failed to generate a valid response",
+    };
+    runTimePlannerSpan?.end({
+        output,
+    });
+    return output;
+}
+exports.runtimePlannerWithScreenshot = runtimePlannerWithScreenshot;

package/dist/agent/master/run.d.ts

@@ -4,7 +4,7 @@ import { BrowsingAgentOptions } from "../browsing";
 export declare function createTestUsingMasterAgent({ task, page, testCase, options, scopeVars, }: {
     task: string;
     page: Page;
-    testCase: TestCase;
+    testCase?: TestCase;
     options: BrowsingAgentOptions;
     scopeVars?: ScopeVars;
 }): Promise<{

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAsBrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA6VA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAsBrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAkWA"}

package/dist/agent/master/run.js

@@ -62,13 +62,16 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         // we will be using google model for larger context window, in such cases 1 million tokens is not enough
         maxTokens: options.modelProvider === "google" ? 3000000 : 1000000,
     });
-    //Fetching available skills
-    const skills = await (0, skills_retriever_1.getAppropriateSkills)({
-        testCase,
-        trace,
-        //@ts-ignore
-        options,
-    });
+    let skills = [];
+    if (testCase) {
+        //Fetching available skills
+        skills = await (0, skills_retriever_1.getAppropriateSkills)({
+            testCase,
+            trace,
+            //@ts-ignore
+            options,
+        });
+    }
     skill_1.testCaseSkills.updateSkills(skills);
     const actions = new actions_1.PlaywrightActions(testGenPage, scopeVars);
     await (0, utils_2.injectPwLocatorGenerator)(page);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.38.29",
+  "version": "0.38.30",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -18,6 +18,14 @@
       "types": "./dist/types/index.d.ts",
       "default": "./dist/types/index.js"
     },
+    "./agent/master/run": {
+      "types": "./dist/agent/master/run.d.ts",
+      "default": "./dist/agent/master/run.js"
+    },
+    "./agent/master/planner": {
+      "types": "./dist/agent/master/planner.d.ts",
+      "default": "./dist/agent/master/planner.js"
+    },
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"