@empiricalrun/test-gen 0.46.8 → 0.46.9

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @empiricalrun/test-gen
 
+## 0.46.9
+
+### Patch Changes
+
+- 6914e32: fix: use o3-mini to improve browsing agent accuracy
+- 22ef805: fix: improve accuracy for multiple pages in runtime planner
+
 ## 0.46.8
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts

@@ -9,7 +9,7 @@ export declare function executeTaskUsingBrowsingAgent({ action, page, actions, l
     action: string;
     page: Page;
     actions: PlaywrightActions;
-    llm: LLM;
+    llm?: LLM;
     trace?: TraceClient;
 }): Promise<BrowserAgentAction | undefined>;
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAQlD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAkE1C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CA6D1C"}

package/dist/agent/browsing/index.js

@@ -7,7 +7,6 @@ const promptTemplate_0 = "{{#section \"system\"}}\nYou are a browser automation
 const reporter_1 = require("../../reporter");
 const html_1 = require("../../utils/html");
 const utils_1 = require("../utils");
-const o1_completion_1 = require("./o1-completion");
 async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace, }) {
     let generatedCodeSteps;
     const tools = actions.getBrowsingActionSchemas();
@@ -31,24 +30,20 @@ async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace
     const promptSpan = browsingAgentSpan?.span({ name: "page-prompt" });
     const messages = (0, llm_1.compilePrompt)(promptTemplate_0, { pageSnapshot, task: action });
     promptSpan?.end({ output: { messages } });
-    let completion;
-    completion = await (0, o1_completion_1.getO1Completion)({
+    llm =
+        llm ||
+            new llm_1.LLM({
+                provider: constants_1.DEFAULT_MODEL_PROVIDER,
+            });
+    const completion = await llm.createChatCompletion({
+        model: "o3-mini-2025-01-31",
         messages,
         tools,
         trace: browsingAgentSpan,
+        modelParameters: {
+            tool_choice: "required",
+        },
     });
-    // If O1 completion fails due to any reason, resort to old flow
-    if (!completion) {
-        completion = await llm.createChatCompletion({
-            messages,
-            tools,
-            trace: browsingAgentSpan,
-            modelParameters: {
-                ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                tool_choice: "required",
-            },
-        });
-    }
     const toolCalls = completion?.tool_calls || [];
     const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
     for (const i in toolCalls) {

package/dist/agent/master/browser-tests/skills.spec.js

@@ -81,12 +81,14 @@ fixtures_1.test.afterEach(async () => {
     const blogPage = await page.context().newPage();
     await blogPage.goto(`${server.baseURL}/blog-page.html`);
     const response = await (0, run_1.createTestUsingMasterAgent)({
-        task: `subscribe as user@example.com on the blog page`,
+        task: `we have 2 pages open inside a web browser, and your task is to subscribe as user@example.com on the blogPage`,
         page,
         testCase: {
             id: 1,
             name: "subscribe to blog",
-            steps: ["subscribe as user@example.com on the blog page"],
+            steps: [
+                "we have 2 pages open inside a web browser, and your task is to subscribe as user@example.com on the blogPage",
+            ],
             filePath: "blog.spec.ts",
             suites: [],
         },

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAyBxC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAoRA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAgCxC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAoRA"}

package/dist/agent/master/run.js

@@ -23,7 +23,10 @@ const next_action_1 = require("./next-action");
 const MAX_ERROR_COUNT = 2;
 function getPageVariables(stateVariables) {
     const keys = Object.keys(stateVariables);
-    const pageVariables = keys.filter((key) => key.endsWith("Page") || key.endsWith("page"));
+    // This checks for whether page.url() exists, which is true for all pages
+    // created by playwright actions.
+    const pageVariables = keys.filter((key) => typeof stateVariables[key] === "object" &&
+        typeof stateVariables[key]?.url === "function");
     const pages = pageVariables.reduce((acc, key) => {
         acc[key] = stateVariables[key];
         return acc;

package/dist/agent/planner/run-time-planner.d.ts

@@ -1,9 +1,10 @@
 import { TraceClient } from "@empiricalrun/llm";
+import type { Page } from "playwright";
 export declare function runtimePlanner({ trace, task, successfulActions, pages, currentPage, }: {
     trace?: TraceClient;
     successfulActions: string[];
     task: string;
-    pages?: Record<string, any>;
+    pages?: Record<string, Page>;
     currentPage?: string;
 }): Promise<{
     pageName: string;

package/dist/agent/planner/run-time-planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAKpE,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GAoFA"}
+{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAKvC,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GA4FA"}

package/dist/agent/planner/run-time-planner.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runtimePlanner = void 0;
 const llm_1 = require("@empiricalrun/llm");
-const promptTemplate_0 = "{{#section \"system\"}}\nGiven a successfully executed actions that lists only the actions that were\nsuccessfully executed and a task comprising multiple actions, your goal is to\nanalyse the list and determine if the entire task is completed.\n\nThese actions are executed by AI agents using Playwright on a browser. These agents\nalready have access to browser tabs to execute actions. The successfully executed\nactions on browser post browser has opened, is provided to you as successfully\nexecuted actions.\n\nIf the task is not fully completed, identify which specific actions are missing\nand suggest next steps to complete the task. Assume that the conversation provided\nis entirely truthful and no additional actions were performed beyond those listed.\n\nTo fulfil your goal, follow these steps:\n- Divide the task into individual actions.\n- Compare each task action against the actions listed in the successfully executed actions list.\n- Identify which actions have been executed and which have not.\n- If all actions are executed, respond with the task as done.\n- If any actions are missing, respond with the task as not done, listing all actions\n  and specifying which are complete and which are missing.\n- If provided with list of pages, based on the next pending action and previously executed\n  action, identify the page on which next action needs to be taken    \n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\n----\n\nFollowing are successfully executed actions:\n{{successfulActions}}\n\n----\n\nCurrent page:\n{{currentPage}}\n\n{{/section}}\n";
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are given a list of successfully executed actions that are done towards completing a task (which\nis also provided to you). Your goal is to analyse the list and determine if the task is completed.\n\nIf the task is not fully completed, identify which specific actions are missing\nand suggest next steps to complete the task. Assume that the conversation provided\nis entirely truthful and no additional actions were performed beyond those listed.\n\nThese actions were executed by AI agents using Playwright on a browser. These agents\nalready have access to browser tabs to execute actions. If there is a pending action,\none of the agents will execute it in the browser. However, they need your help to\nchoose which browser tab (= page) to use for the next action.\n\nTo fulfil your goal, follow these steps:\n- Divide the task into individual actions.\n- Compare each task action against the actions listed in the successfully executed actions list.\n- Identify which actions have been executed and which have not.\n- If all actions are executed, respond with the task as done.\n- If any actions are missing, respond with the task as not done, listing all actions\n  and specifying which are complete and which are missing.\n- If provided with list of pages, based on the next pending action and previously executed\n  action, identify the page on which next action needs to be taken\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\n----\n\nSuccessfully executed actions:\n{{successfulActions}}\n\n----\n\nList of pages with their current URLs:\n{{pagesSummary}}\n\n\n{{/section}}\n";
 const utils_1 = require("../utils");
 async function runtimePlanner({ trace, task, successfulActions, pages, currentPage, }) {
     const runTimePlannerSpan = trace?.span({
@@ -14,6 +14,9 @@ async function runtimePlanner({ trace, task, successfulActions, pages, currentPa
         },
     });
     const llm = new llm_1.LLM({ provider: "openai" });
+    const pagesSummary = pages
+        ? Object.entries(pages).map(([pageName, page]) => `${pageName}: Currently on ${page.url()}`)
+        : [];
     const response = await llm.createChatCompletion({
         trace: runTimePlannerSpan,
         traceName: "runtime-planner-llm",
@@ -21,7 +24,7 @@ async function runtimePlanner({ trace, task, successfulActions, pages, currentPa
         messages: (0, llm_1.compilePrompt)(promptTemplate_0, {
             task,
             successfulActions: successfulActions.join("\n"),
-            currentPage,
+            pagesSummary: pagesSummary.join("\n"),
         }),
         tools: [
             {
@@ -44,17 +47,21 @@ async function runtimePlanner({ trace, task, successfulActions, pages, currentPa
                                 type: "string",
                                 description: "reasoning for identification of task status",
                             },
-                            isDone: {
-                                type: "boolean",
-                                description: "whether the task is done",
+                            nextAction: {
+                                type: "string",
+                                description: "next action to be taken",
                             },
                             pageName: {
                                 type: "string",
                                 enum: pages ? Object.keys(pages) : [],
                                 description: "page name for the next action.",
                             },
+                            isDone: {
+                                type: "boolean",
+                                description: "whether the task is done",
+                            },
                         },
-                        required: ["isDone", "reason", "pageName"],
+                        required: ["isDone", "reason", "pageName", "nextAction"],
                     },
                 },
             },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.46.8",
+  "version": "0.46.9",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"

package/dist/agent/browsing/o1-completion.d.ts

@@ -1,8 +0,0 @@
-import { TraceClient } from "@empiricalrun/llm";
-import { ChatCompletionMessage, ChatCompletionMessageParam, ChatCompletionTool } from "openai/resources/index.mjs";
-export declare function getO1Completion({ messages, tools, trace, }: {
-    messages: ChatCompletionMessageParam[];
-    tools: ChatCompletionTool[];
-    trace?: TraceClient;
-}): Promise<ChatCompletionMessage | undefined>;
-//# sourceMappingURL=o1-completion.d.ts.map

package/dist/agent/browsing/o1-completion.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"o1-completion.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/o1-completion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAMpC,wBAAsB,eAAe,CAAC,EACpC,QAAQ,EACR,KAAK,EACL,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CA8D7C"}

package/dist/agent/browsing/o1-completion.js

@@ -1,72 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getO1Completion = void 0;
-const llm_1 = require("@empiricalrun/llm");
-const remove_markdown_1 = __importDefault(require("remove-markdown"));
-const constants_1 = require("../../constants");
-const utils_1 = require("../utils");
-async function getO1Completion({ messages, tools, trace, }) {
-    let completion;
-    try {
-        const o1Span = trace?.span({ name: "o1-response-span" });
-        const llm = new llm_1.LLM({
-            trace: o1Span,
-            provider: "openai",
-            defaultModel: "o1-mini",
-            providerApiKey: constants_1.MODEL_API_KEYS["openai"],
-        });
-        const [userInstruction] = messages.filter((s) => s.role === "user");
-        const [systemInstruction] = messages.filter((s) => s.role === "system");
-        userInstruction.content = `${systemInstruction?.content} 
-  
-    ${userInstruction?.content}
-
-    You need to respond with one of the following tool call with provided schema:
-    ${tools.map((tool) => JSON.stringify(tool, null, 2)).join("\n --- \n")}
- 
-    ------
-
-    Before responding, ensure the following:
-    - Do not respond with markdown, respond only with the JSON object.
-    - Do not respond with any backticks.
-    - The reason for action should also include what was been executed in the action.
-  `;
-        o1Span?.update({ input: [userInstruction] });
-        const response = (await llm.createChatCompletion({
-            messages: [userInstruction],
-            modelParameters: {
-                ...constants_1.DEFAULT_O1_MODEL_PARAMETERS,
-            },
-        }));
-        o1Span?.end({ output: response });
-        const toolResponseStr = (0, remove_markdown_1.default)(response.content);
-        const toolRespJSON = (0, utils_1.parseJson)(toolResponseStr);
-        const parameters = toolRespJSON.function.parameters || toolRespJSON.function.arguments;
-        if (!parameters) {
-            throw new Error("No parameters found in tool response");
-        }
-        const tool = {
-            type: "function",
-            function: {
-                name: toolRespJSON.function.name,
-                arguments: typeof parameters === "string"
-                    ? parameters
-                    : JSON.stringify(parameters, null, 2),
-            },
-        };
-        completion = {
-            role: "assistant",
-            content: response.content,
-            tool_calls: [tool],
-        };
-    }
-    catch (e) {
-        console.error("O1 response error", e);
-        return undefined;
-    }
-    return completion;
-}
-exports.getO1Completion = getO1Completion;