@empiricalrun/test-gen 0.80.3 → 0.81.1

package/dist/agent/master/planner.js

@@ -1,142 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runtimePlannerWithScreenshot = runtimePlannerWithScreenshot;
-const llm_1 = require("@empiricalrun/llm");
-const vision_1 = require("@empiricalrun/llm/vision");
-const constants_1 = require("../../constants");
-async function runtimePlannerWithScreenshot({ trace, task, conversation, pages, page, currentPage, }) {
-    const buffer = await page.screenshot({
-    //This is done to improve element annotation accuracy, anyways it doesn't annotate elements which are out of viewport
-    // fullPage: true,
-    // path: `screenshots/screenshot-${screenshotIndex++}.png`, // enable this and screenshotIndex var for local debugging
-    });
-    const pageBuffer = buffer.toString("base64");
-    const runTimePlannerSpan = trace?.span({
-        name: "popup-verifier",
-        input: {
-            task,
-            conversation,
-        },
-    });
-    const llm = new llm_1.LLM({ provider: "openai" });
-    const prompt = [
-        {
-            role: "system",
-            content: `
-  Given a successfully executed actions that lists only the actions that were successfully executed and a task comprising multiple actions, your goal is to analyse the list and determine if the entire task is completed.
-  These actions are executed by AI agents using Playwright on a browser. These agents already have access to browser tabs to execute actions. The successfully executed actions on browser post browser has opened, is provided to you as successfully executed actions.
-
-  You are also given a screenshot of the current screen which you can also use to determine whether the entire task is completed or not.
-  
-  If the task is not fully completed, identify which specific actions are missing and suggest next steps to complete the task. Assume that the conversation provided is entirely truthful and no additional actions were performed beyond those listed.
-  
-  To fulfil your goal, follow these steps:
-  - Divide the task into individual actions.
-  - Compare each task action against the actions listed in the successfully executed actions list.
-  - Identify which actions have been executed and which have not.
-  - If all actions are executed, respond with the task as done.
-  - If any actions are missing, respond with the task as not done, listing all actions and specifying which are complete and which are missing.
-  - If provided with list of pages, based on the next pending action and previously executed action, identify the page on which next action needs to be taken
-      `,
-        },
-        {
-            role: "user",
-            content: [
-                {
-                    type: "text",
-                    text: `
-  Task: ${task}
-  
-  ----
-  
-  Following are successfully executed actions:
-  ${conversation.join("\n")}
-  
-  ----
-  
-  Current page:
-  ${currentPage}
-          `,
-                },
-                {
-                    type: "text",
-                    text: "Page Screenshot",
-                },
-                {
-                    type: "image_url",
-                    image_url: {
-                        url: (0, vision_1.imageFormatForProvider)(constants_1.DEFAULT_MODEL_PROVIDER, pageBuffer),
-                    },
-                },
-            ],
-        },
-    ];
-    const response = await llm.createChatCompletion({
-        trace: runTimePlannerSpan,
-        traceName: "runtime-planner-llm",
-        model: "gpt-4o",
-        messages: prompt,
-        tools: [
-            {
-                type: "function",
-                function: {
-                    name: "task_done",
-                    description: "end the task by calling this method",
-                    parameters: {
-                        type: "object",
-                        properties: {
-                            actions: {
-                                type: "string",
-                                description: "actions extracted from task",
-                            },
-                            successful_actions: {
-                                type: "string",
-                                description: "successful actions mentioned in the conversation",
-                            },
-                            reason: {
-                                type: "string",
-                                description: "reasoning for identification of task status",
-                            },
-                            isDone: {
-                                type: "boolean",
-                                description: "whether the task is done",
-                            },
-                            pageName: {
-                                type: "string",
-                                enum: pages ? Object.keys(pages) : [],
-                                description: "page name for the next action.",
-                            },
-                        },
-                        required: ["isDone", "reason", "pageName"],
-                    },
-                },
-            },
-        ],
-        modelParameters: {
-            tool_choice: "required",
-            temperature: 0.5,
-        },
-    });
-    const toolCallResp = (response?.tool_calls || [])[0];
-    if (toolCallResp && "function" in toolCallResp) {
-        const toolCall = JSON.parse(toolCallResp.function.arguments);
-        const output = {
-            pageName: toolCall.pageName,
-            isDone: toolCall.isDone,
-            reason: toolCall.reason,
-        };
-        runTimePlannerSpan?.end({
-            output,
-        });
-        return output;
-    }
-    const output = {
-        pageName: "",
-        isDone: false,
-        reason: "LLM failed to generate a valid response",
-    };
-    runTimePlannerSpan?.end({
-        output,
-    });
-    return output;
-}

package/dist/agent/master/run.d.ts

@@ -1,17 +0,0 @@
-import type { TestGenConfigOptions } from "@empiricalrun/shared-types/agent-workflow";
-import type { TestCaseWithSuitesAsArray } from "@empiricalrun/shared-types/api/test-cases";
-import { Page } from "playwright";
-import { ScopeVars } from "../../types";
-export declare const IS_ALLOWED_TO_USE_SKILLS = false;
-export declare function createTestUsingMasterAgent({ task, page, testCase, specPath, options, scopeVars, }: {
-    task: string;
-    page: Page;
-    testCase?: TestCaseWithSuitesAsArray;
-    specPath?: string;
-    options: Partial<TestGenConfigOptions>;
-    scopeVars?: ScopeVars;
-}): Promise<{
-    code: string;
-    importPaths: string[];
-}>;
-//# sourceMappingURL=run.d.ts.map

package/dist/agent/master/run.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACtF,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,2CAA2C,CAAC;AAC3F,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAWxC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAqB9C,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,yBAAyB,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwJA"}

package/dist/agent/master/run.js

@@ -1,156 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.IS_ALLOWED_TO_USE_SKILLS = void 0;
-exports.createTestUsingMasterAgent = createTestUsingMasterAgent;
-const llm_1 = require("@empiricalrun/llm");
-const actions_1 = require("../../actions");
-const utils_1 = require("../../actions/utils");
-const logger_1 = require("../../bin/logger");
-const constants_1 = require("../../constants");
-const errors_1 = require("../../errors");
-const page_1 = require("../../page");
-const utils_2 = require("../browsing/utils");
-const run_1 = require("../planner/run");
-const run_time_planner_1 = require("../planner/run-time-planner");
-const action_tool_calls_1 = require("./action-tool-calls");
-const execute_browser_action_1 = require("./execute-browser-action");
-const next_action_1 = require("./next-action");
-const MAX_ERROR_COUNT = 2;
-// Disabling skills as we're seeing false usage with chat agent
-exports.IS_ALLOWED_TO_USE_SKILLS = false;
-function getPageVariables(stateVariables) {
-    const keys = Object.keys(stateVariables);
-    // This checks for whether page.url() exists, which is true for all pages
-    // created by playwright actions.
-    const pageVariables = keys.filter((key) => typeof stateVariables[key] === "object" &&
-        typeof stateVariables[key]?.url === "function");
-    const pages = pageVariables.reduce((acc, key) => {
-        acc[key] = stateVariables[key];
-        return acc;
-    }, {});
-    return pages;
-}
-async function createTestUsingMasterAgent({ task, page, testCase, specPath, options, scopeVars, }) {
-    const useActionSpecificAnnotations = options?.useActionSpecificAnnotations || false;
-    const usePlannerInMaster = options?.usePlannerInMaster || false;
-    const logger = new logger_1.CustomLogger({ useReporter: false });
-    const testGenPage = new page_1.TestGenPage(page, (0, utils_1.getPageVarName)());
-    const llm = new llm_1.LLM({
-        provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-        defaultModel: options.model || constants_1.DEFAULT_MODEL,
-        providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
-        // we will be using google model for larger context window, in such cases 1 million tokens is not enough
-        maxTokens: options.modelProvider === "google" ? 3_000_000 : 1_000_000,
-    });
-    if (usePlannerInMaster && testCase && specPath) {
-        logger.log(`Planner is working on task: ${task}`);
-        const plan = await (0, run_1.planTask)({
-            task,
-            specPath,
-        });
-        logger.log(`Here is the plan:\n${plan}`);
-        // Will assume this is the task hereon
-        task = plan;
-    }
-    const actions = new actions_1.PlaywrightActions(testGenPage, scopeVars);
-    await (0, utils_2.injectPwLocatorGenerator)(page);
-    let isGivenTaskDone = false;
-    const masterAgentActions = [];
-    let failedActions = [];
-    let disableSkills = false;
-    // Run the loop until task is done or we have reached max retry limit
-    while (!isGivenTaskDone) {
-        const plannerResp = await (0, run_time_planner_1.runtimePlanner)({
-            task,
-            successfulActions: [...masterAgentActions],
-            pages: getPageVariables(actions.getStateVariables()),
-            currentPage: testGenPage,
-        });
-        isGivenTaskDone = plannerResp.isDone;
-        if (isGivenTaskDone) {
-            break;
-        }
-        if (actions.getStateVariables()[plannerResp.pageName]) {
-            // update page for the master agent
-            page = actions.getStateVariables()[plannerResp.pageName];
-            // update page in actions
-            testGenPage.updatePage({ page, name: plannerResp.pageName });
-        }
-        // inject scripts in the page
-        await (0, utils_2.injectPwLocatorGenerator)(testGenPage.pwPageInstance);
-        const buffer = await page.screenshot();
-        const pageScreenshot = buffer.toString("base64");
-        let output;
-        let generatedCodeSteps = [];
-        // Provides next action that needs to be taken
-        const nextAction = await (0, next_action_1.getNextAction)({
-            page,
-            pageScreenshot: [pageScreenshot],
-            task,
-            executedActions: masterAgentActions,
-            failedActions,
-            llm,
-            options,
-            actions,
-            disableSkills,
-            logger,
-        });
-        if (nextAction) {
-            if (nextAction.actionType === action_tool_calls_1.ActionType.UNKNOWN) {
-                logger.error("Agent is not able to figure out next action since element is not visible on screen.");
-                break;
-            }
-            const args = JSON.parse(nextAction.toolCallArgs);
-            output = {
-                action: args.action || args.skill,
-                reason: args.reason,
-            };
-            try {
-                logger.log(`Next Action: ${output.action}`);
-                switch (nextAction.actionType) {
-                    case action_tool_calls_1.ActionType.OBSERVATION: {
-                        output.action = args.observation;
-                        logger.log("Observation: ", output.action);
-                        break;
-                    }
-                    default: {
-                        const result = await (0, execute_browser_action_1.executeBrowserAction)({
-                            page,
-                            nextAction,
-                            flags: {
-                                useActionSpecificAnnotations,
-                            },
-                            actions,
-                            llm,
-                        });
-                        const { generatedCodeSteps: codeFromExecuteAction, output: outputFromExecuteAction, } = result;
-                        generatedCodeSteps.push(...codeFromExecuteAction);
-                        output = outputFromExecuteAction;
-                    }
-                }
-                // resetting error count as there is a successful action
-                failedActions = [];
-                masterAgentActions.push(output.action);
-                // enable skills after success
-                disableSkills = false;
-            }
-            catch (e) {
-                if (!(e instanceof errors_1.HumanApprovalDenied)) {
-                    logger.error("Failed to run master agent's next task", JSON.stringify(nextAction, null, 2), e);
-                    failedActions.push(output.action);
-                    if (failedActions.length >= MAX_ERROR_COUNT) {
-                        const error = "Agent is not able to figure out next action, marking task as done";
-                        logger.error(error);
-                        break;
-                    }
-                }
-            }
-        }
-    }
-    const { code, importPaths } = actions.generateCode();
-    logger.success("Successfully generated code for the given task");
-    return {
-        code,
-        importPaths,
-    };
-}

package/dist/agent/master/scroller.d.ts

@@ -1,15 +0,0 @@
-import { TraceClient } from "@empiricalrun/llm";
-import type { Page } from "@playwright/test";
-import { CustomLogger } from "../../bin/logger";
-export type FrameReference = {
-    scrollPosition: number;
-    frameScreenshot: string;
-};
-export declare function scroller({ elementDescription, page, trace, frameReference, logger, }: {
-    elementDescription: string;
-    page: Page;
-    trace?: TraceClient;
-    frameReference?: FrameReference;
-    logger?: CustomLogger;
-}): Promise<FrameReference[]>;
-//# sourceMappingURL=scroller.d.ts.map

package/dist/agent/master/scroller.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAG7C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAYhD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAoZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}