@empiricalrun/test-gen 0.46.7 → 0.46.8

package/dist/agent/master/run.js

@@ -8,21 +8,18 @@ const utils_1 = require("../../actions/utils");
 const logger_1 = require("../../bin/logger");
 const constants_1 = require("../../constants");
 const errors_1 = require("../../errors");
-const human_in_the_loop_1 = require("../../human-in-the-loop");
 const page_1 = require("../../page");
 const reporter_1 = require("../../reporter");
 const session_1 = require("../../session");
-const env_1 = require("../../utils/env");
-const browsing_1 = require("../browsing");
 const utils_2 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
 const run_1 = require("../planner/run");
 const run_time_planner_1 = require("../planner/run-time-planner");
 const utils_3 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
-const element_annotation_1 = require("./element-annotation");
+const execute_browser_action_1 = require("./execute-browser-action");
+const execute_skill_action_1 = require("./execute-skill-action");
 const next_action_1 = require("./next-action");
-const with_hints_1 = require("./with-hints");
 const MAX_ERROR_COUNT = 2;
 function getPageVariables(stateVariables) {
     const keys = Object.keys(stateVariables);
@@ -110,13 +107,12 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                 failedActions,
             },
         });
-        // Provides a plan to master agent to execute, also verifies whether task is done or not
         const plannerResp = await (0, run_time_planner_1.runtimePlanner)({
             trace: masterAgentSpan,
             task,
             successfulActions: [...masterAgentActions],
             pages: getPageVariables(actions.getStateVariables()),
-            currentPage: (0, utils_1.getPageVarName)(),
+            currentPage: testGenPage.name,
         });
         isGivenTaskDone = plannerResp.isDone;
         if (isGivenTaskDone) {
@@ -131,9 +127,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
         }
         // inject scripts in the page
         await (0, utils_2.injectPwLocatorGenerator)(testGenPage.pwPageInstance);
-        const buffer = await page.screenshot({
-        // path: `screenshots/screenshot-${screenshotIndex++}.png`, // enable this and screenshotIndex var for local debugging
-        });
+        const buffer = await page.screenshot();
         const testGenReporter = new reporter_1.TestGenUpdatesReporter();
         await testGenReporter.sendCurrentView(buffer);
         const pageScreenshot = buffer.toString("base64");
@@ -143,31 +137,29 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
             break;
         }
         // Provides next action that needs to be taken
-        const toolCall = await (0, next_action_1.getNextAction)({
+        const nextAction = await (0, next_action_1.getNextAction)({
+            page,
+            pageScreenshot: [pageScreenshot],
             task,
             executedActions: masterAgentActions,
             failedActions,
-            pageUrl: page.url(),
             trace: masterAgentSpan,
             llm,
             options,
-            pageScreenshot: [pageScreenshot],
             actions,
-            actionTypes: action_tool_calls_1.ActionType,
             disableSkills,
-            page,
             logger,
         });
         if (await (0, session_1.shouldStopSession)()) {
             break;
         }
-        if (toolCall) {
-            if (toolCall.actionType === "unknown") {
+        if (nextAction) {
+            if (nextAction.actionType === action_tool_calls_1.ActionType.UNKNOWN) {
                 logger.error("Agent is not able to figure out next action since element is not visible on screen.");
                 await testgenUpdatesReporter.sendMessage("Agent is not able to figure out next action since element is not visible on screen.");
                 break;
             }
-            const args = (0, utils_3.parseJson)(toolCall.toolCallArgs);
+            const args = (0, utils_3.parseJson)(nextAction.toolCallArgs);
             const masterAgentActionSpan = masterAgentSpan?.span({
                 name: "master-agent-execute-action",
             });
@@ -178,124 +170,35 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
             try {
                 void testGenReporter.sendMessage(output.reason);
                 logger.log(`Next Action: ${output.action}`);
-                if (toolCall.actionType === skill_1.SKILL_USAGE) {
-                    if (!(0, env_1.isRunningOnCloud)()) {
-                        const { skillDetails } = (0, skill_1.extractSkillFromArgs)(args);
-                        const feedback = await human_in_the_loop_1.humanLoop.getFeedback({
-                            message: `Should I use this skill?\n\n${skillDetails?.usageExample}\n\nfrom: ${skillDetails?.filePath}\n\nAnswer with "yes" or "no"`,
+                switch (nextAction.actionType) {
+                    case skill_1.SKILL_USAGE: {
+                        const code = await (0, execute_skill_action_1.executeSkillAction)({
+                            actions,
+                            nextAction,
+                            trace: masterAgentActionSpan,
                         });
-                        if (feedback.toLowerCase().trim().includes("no")) {
-                            throw new errors_1.HumanApprovalDenied("Skill usage approval not provided");
+                        if (code) {
+                            generatedCodeSteps.push(code);
                         }
+                        break;
                     }
-                    const code = await actions.executeAction(toolCall.actionType, args, masterAgentActionSpan);
-                    if (code) {
-                        generatedCodeSteps.push(code);
-                    }
-                }
-                else {
-                    let shouldTriggerHintsFlow;
-                    let hintsExecutionCompletion;
-                    let elementAnnotation;
-                    const actionType = toolCall.actionType;
-                    let preference = {
-                        actionType: "all",
-                    };
-                    if (useActionSpecificAnnotations && (0, action_tool_calls_1.isValidActionType)(actionType)) {
-                        switch (actionType) {
-                            case action_tool_calls_1.ActionType.FILL:
-                                preference = {
-                                    actionType: action_tool_calls_1.ActionType.FILL,
-                                };
-                                break;
-                            case action_tool_calls_1.ActionType.ASSERT_TEXT:
-                                preference = {
-                                    actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
-                                    assertionText: args.assertion_text,
-                                };
-                                break;
-                            default:
-                                preference = {
-                                    actionType: "all",
-                                };
-                        }
+                    case action_tool_calls_1.ActionType.OBSERVATION: {
+                        output.action = args.observation;
+                        logger.log("Observation: ", output.action);
+                        break;
                     }
-                    let { annotationKeys, annotatedPageScreenshot, annotationBuffer } = await (0, element_annotation_1.getAnnotationKeys)({
-                        page,
-                        preference,
-                        options,
-                        trace: masterAgentActionSpan,
-                    });
-                    if (annotationKeys.length > 0) {
-                        // TODO: this string has newline characters that makes it harder to read
-                        const annotationMapString = annotationKeys
-                            ?.map((a) => `${a.elementID}: ${a.text}`)
-                            .join("\n");
-                        // Provides the annotations for all the element present on screen
-                        // Also provides the annotation of element on which the action needs to be taken
-                        elementAnnotation = await (0, element_annotation_1.getElementAnnotation)({
-                            elementDescription: args.element_description,
-                            annotations: annotationMapString,
-                            annotatedScreenshot: annotatedPageScreenshot,
-                            trace: masterAgentActionSpan,
-                            llm,
-                            preference,
-                        });
-                        output.elementAnnotation = elementAnnotation;
-                        console.log("Output: ", output);
-                        await testGenReporter.sendCurrentView(annotationBuffer);
-                        const triggerHintsFlowSpan = masterAgentActionSpan?.span({
-                            name: "trigger-hints-flow",
-                            input: {
-                                outputFromGetNextAction: output,
-                                generatedAnnotations: annotationKeys,
+                    default: {
+                        const { generatedCodeSteps: codeFromExecuteAction } = await (0, execute_browser_action_1.executeBrowserAction)({
+                            page,
+                            nextAction,
+                            flags: {
+                                useActionSpecificAnnotations,
                             },
-                        });
-                        // Provides the action whether its a click, fill etc.
-                        const result = await (0, with_hints_1.triggerHintsFlow)({
-                            outputFromGetNextAction: output,
-                            generatedAnnotations: annotationKeys,
                             actions,
                             llm,
-                            trace: triggerHintsFlowSpan,
-                        });
-                        shouldTriggerHintsFlow = result.shouldTriggerHintsFlow;
-                        hintsExecutionCompletion = result.hintsExecutionCompletion;
-                        triggerHintsFlowSpan?.end({
-                            output: result,
-                        });
-                    }
-                    if (shouldTriggerHintsFlow && hintsExecutionCompletion) {
-                        const toolCalls = hintsExecutionCompletion?.tool_calls || [];
-                        for (const i in toolCalls) {
-                            const currentToolCall = toolCalls[i];
-                            const code = await actions.executeAction(currentToolCall.function.name, {
-                                ...JSON.parse(currentToolCall.function.arguments),
-                                elementAnnotation,
-                            }, masterAgentActionSpan);
-                            if (code) {
-                                generatedCodeSteps.push(code);
-                            }
-                        }
-                        if (actions.isStuckInLoop()) {
-                            throw new Error("Agent is not able to figure out next action when using hints");
-                        }
-                    }
-                    else {
-                        const browserAction = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
                             trace: masterAgentActionSpan,
-                            action: output.action,
-                            page,
-                            options,
-                            llm,
-                            actions,
                         });
-                        if (browserAction) {
-                            output.action = browserAction.action;
-                            if (browserAction.code) {
-                                generatedCodeSteps.push(browserAction.code);
-                            }
-                        }
+                        generatedCodeSteps.push(...codeFromExecuteAction);
                     }
                 }
                 // resetting error count as there is a successful action
@@ -305,7 +208,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                     input: {
                         action: output.action,
                         reason: output.reason,
-                        type: toolCall.actionType,
+                        type: nextAction.actionType,
                     },
                     output: {
                         success: true,
@@ -315,7 +218,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                 disableSkills = false;
             }
             catch (e) {
-                if (toolCall.actionType === skill_1.SKILL_USAGE) {
+                if (nextAction.actionType === skill_1.SKILL_USAGE) {
                     logger.log("Disabling skill usage for next retry");
                     disableSkills = true;
                 }
@@ -323,7 +226,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                     input: {
                         action: output.action,
                         reason: output.reason,
-                        type: toolCall.actionType,
+                        type: nextAction.actionType,
                     },
                     output: {
                         error: true,
@@ -331,7 +234,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                     },
                 });
                 if (!(e instanceof errors_1.HumanApprovalDenied)) {
-                    logger.error("Failed to run master agent's next task", JSON.stringify(toolCall, null, 2), e);
+                    logger.error("Failed to run master agent's next task", JSON.stringify(nextAction, null, 2), e);
                     failedActions.push(output.action);
                     if (failedActions.length >= MAX_ERROR_COUNT) {
                         const error = "Agent is not able to figure out next action, marking task as done";

package/dist/agent/master/scroller.d.ts

@@ -1,5 +1,5 @@
 import { TraceClient } from "@empiricalrun/llm";
-import { Page } from "playwright";
+import type { Page } from "@playwright/test";
 import { CustomLogger } from "../../bin/logger";
 export type FrameReference = {
     scrollPosition: number;

package/dist/agent/master/scroller.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAchD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AA4ZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}
+{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAG7C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAchD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AA2ZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}

package/dist/agent/master/scroller.js

@@ -187,7 +187,6 @@ async function getDivAnnotationToScrollOn({ elementDescription, page, trace, log
         preference: {
             actionType: action_tool_calls_1.ActionType.SCROLL,
         },
-        options: {},
         trace,
     });
     // Remove the used annotations from the list

package/dist/evals/master-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UAwC1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
+{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UAoC1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/master-agent.evals.js

@@ -3,11 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.masterGetNextActionEvaluator = void 0;
 const actions_1 = require("../actions");
 const skill_1 = require("../actions/skill");
-const action_tool_calls_1 = require("../agent/master/action-tool-calls");
 const next_action_1 = require("../agent/master/next-action");
 const page_1 = require("../page");
 const masterGetNextActionEvaluator = async ({ item, trace, }) => {
-    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, disableSkills, skills = [], } = item.input;
+    const { task, executedActions, failedActions, options, pageScreenshot, disableSkills, skills = [], } = item.input;
     const page = {};
     skill_1.testCaseSkills.updateSkills(skills);
     const actions = new actions_1.PlaywrightActions(new page_1.TestGenPage(page, "page"));
@@ -15,12 +14,10 @@ const masterGetNextActionEvaluator = async ({ item, trace, }) => {
         task,
         executedActions,
         failedActions,
-        pageUrl,
         trace,
         options,
         pageScreenshot,
         actions,
-        actionTypes: action_tool_calls_1.ActionType,
         disableSkills,
         page,
     });

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAYpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAsC3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAYpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAiC3E"}

package/dist/index.js

@@ -40,11 +40,6 @@ async function createTest(task, page, scope) {
         page,
         task,
         options: {
-            htmlSanitize: {
-                disallowedStrings: [
-                    "v-data-table__td v-data-table-column--align-start",
-                ],
-            },
             ...testGenConfig.options,
         },
         scopeVars: scope,

package/dist/page/index.d.ts

@@ -1,4 +1,4 @@
-import { Page } from "playwright";
+import type { Page } from "playwright";
 export declare class TestGenPage {
     pwPageInstance: Page;
     name: string;

package/dist/page/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/page/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,qBAAa,WAAW;IAEb,cAAc,EAAE,IAAI;IACpB,IAAI,EAAE,MAAM;gBADZ,cAAc,EAAE,IAAI,EACpB,IAAI,EAAE,MAAM;IAGrB,UAAU,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;CAIxD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/page/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,qBAAa,WAAW;IAEb,cAAc,EAAE,IAAI;IACpB,IAAI,EAAE,MAAM;gBADZ,cAAc,EAAE,IAAI,EACpB,IAAI,EAAE,MAAM;IAGrB,UAAU,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;CAIxD"}

package/dist/utils/env.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"env.d.ts","sourceRoot":"","sources":["../../src/utils/env.ts"],"names":[],"mappings":"AAAA,wBAAgB,gBAAgB,YAE/B"}
+{"version":3,"file":"env.d.ts","sourceRoot":"","sources":["../../src/utils/env.ts"],"names":[],"mappings":"AAAA,wBAAgB,gBAAgB,YAM/B"}

package/dist/utils/env.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.isRunningOnCloud = void 0;
 function isRunningOnCloud() {
-    return process.env.CI === "true";
+    return (process.env.CI === "true" ||
+        // eslint-disable-next-line turbo/no-undeclared-env-vars
+        !!process.env.RUNNING_BROWSER_TESTS_FOR_TEST_GEN);
 }
 exports.isRunningOnCloud = isRunningOnCloud;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.46.7",
+  "version": "0.46.8",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"

package/playwright.config.ts

@@ -8,7 +8,7 @@ export default defineConfig({
   testDir: "./dist/.",
   testMatch: "**/*.spec.js",
   // Setting large timeout as master agent takes time to run.
-  timeout: 240_000,
+  timeout: 300_000,
   fullyParallel: true,
   workers: "50%",
   use: {