@empiricalrun/test-gen 0.20.4 → 0.21.0

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @empiricalrun/test-gen
 
+## 0.21.0
+
+### Minor Changes
+
+- 3c714f7: feat: add master agent support
+
+### Patch Changes
+
+- d7a24f0: fix: add lint rule for no floating promises
+- d1a1ff2: fix: restrict usage of llm basis token usage
+- Updated dependencies [3c714f7]
+- Updated dependencies [d1a1ff2]
+  - @empiricalrun/llm@0.4.3
+
+## 0.20.5
+
+### Patch Changes
+
+- Updated dependencies [a9423b7]
+  - @empiricalrun/llm@0.4.2
+
 ## 0.20.4
 
 ### Patch Changes

package/dist/actions/done.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"done.d.ts","sourceRoot":"","sources":["../../src/actions/done.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAyBjC,CAAC"}
+{"version":3,"file":"done.d.ts","sourceRoot":"","sources":["../../src/actions/done.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAuBjC,CAAC"}

package/dist/actions/done.js

@@ -2,11 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.doneActionGenerator = exports.PLAYWRIGHT_DONE_ACTION_NAME = void 0;
 exports.PLAYWRIGHT_DONE_ACTION_NAME = "task_done";
-const doneActionGenerator = (page) => {
+const doneActionGenerator = () => {
     return {
-        execute: async () => {
-            await page.close();
-        },
+        execute: async () => { },
         template: () => ``,
         name: exports.PLAYWRIGHT_DONE_ACTION_NAME,
         schema: {

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGlC,OAAO,EAAU,YAAY,EAAE,MAAM,UAAU,CAAC;AAQhD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAmC;gBAC9C,IAAI,EAAE,IAAI;IAYhB,aAAa,CAAC,IAAI,oBAAa,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAqBhE,gBAAgB,IAAI,YAAY,EAAE;IAIlC,YAAY;IAIZ,UAAU;CAMX"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGlC,OAAO,EAAU,YAAY,EAAE,MAAM,UAAU,CAAC;AAQhD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAmC;gBAC9C,IAAI,EAAE,IAAI;IAYhB,aAAa,CAAC,IAAI,oBAAa,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAqBhE,gBAAgB,IAAI,YAAY,EAAE;IAIlC,YAAY;IAIZ,UAAU;CASX"}

package/dist/actions/index.js

@@ -51,6 +51,7 @@ class PlaywrightActions {
     }
     isComplete() {
         const [doneAction] = this.recordedActions.filter((a) => a.name === done_1.PLAYWRIGHT_DONE_ACTION_NAME);
+        this.recordedActions = this.recordedActions.filter((a) => a.name !== done_1.PLAYWRIGHT_DONE_ACTION_NAME);
         return !!doneAction;
     }
 }

package/dist/actions/next-task.d.ts

@@ -0,0 +1,26 @@
+export declare const NEXT_TASK = "next_task";
+export declare const NextTaskAction: {
+    name: string;
+    schema: {
+        type: string;
+        function: {
+            name: string;
+            description: string;
+            parameters: {
+                type: string;
+                properties: {
+                    action: {
+                        type: string;
+                        description: string;
+                    };
+                    reason: {
+                        type: string;
+                        description: string;
+                    };
+                };
+                required: string[];
+            };
+        };
+    };
+};
+//# sourceMappingURL=next-task.d.ts.map

package/dist/actions/next-task.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"next-task.d.ts","sourceRoot":"","sources":["../../src/actions/next-task.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,SAAS,cAAc,CAAC;AAErC,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;CA0B1B,CAAC"}

package/dist/actions/next-task.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NextTaskAction = exports.NEXT_TASK = void 0;
+exports.NEXT_TASK = "next_task";
+exports.NextTaskAction = {
+    name: exports.NEXT_TASK,
+    schema: {
+        type: "function",
+        function: {
+            name: exports.NEXT_TASK,
+            description: "take the next action base on the provided task",
+            parameters: {
+                type: "object",
+                properties: {
+                    action: {
+                        type: "string",
+                        description: `explain the next action in natural language.
+The next action should be as atomic as possible. E.g. each click, key press, input should be a separate action. 
+Each action should take the task to completion, if not the action is invalid.`,
+                    },
+                    reason: {
+                        type: "string",
+                        description: "explain how this action will help to complete the task",
+                    },
+                },
+                required: ["action", "reason"],
+            },
+        },
+    },
+};

package/dist/actions/reload-page.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reload-page.d.ts","sourceRoot":"","sources":["../../src/actions/reload-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,6BAA6B,gBAAgB,CAAC;AAE3D,eAAO,MAAM,qBAAqB,EAAE,yBA8BnC,CAAC"}
+{"version":3,"file":"reload-page.d.ts","sourceRoot":"","sources":["../../src/actions/reload-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,6BAA6B,gBAAgB,CAAC;AAE3D,eAAO,MAAM,qBAAqB,EAAE,yBA+BnC,CAAC"}

package/dist/actions/reload-page.js

@@ -19,7 +19,7 @@ const reloadActionGenerator = (page) => {
             type: "function",
             function: {
                 name: exports.PLAYWRIGHT_RELOAD_ACTION_NAME,
-                description: "reload the page by calling this method",
+                description: "reload the page by calling this method. Call this method only when a page reload is requested in the task.",
                 parameters: {
                     type: "object",
                     properties: {

package/dist/actions/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,IAAI,gBAmBX"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,IAAI,gBAkBX"}

package/dist/actions/utils/index.js

@@ -4,7 +4,9 @@ exports.getPlaywrightLocatorUsingCssSelector = void 0;
 async function getPlaywrightLocatorUsingCssSelector(cssSelector, page) {
     // TODO: analyse other solutions than just css. Also find other solutions to support :has-text -> prompting ?
     // jquery doesnt support :has-text. neither css. Only playwright locator supports this selector though.
-    const sanitizedCssSelectorForJQuery = cssSelector.replaceAll(":has-text", ":contains");
+    const sanitizedCssSelectorForJQuery = cssSelector
+        .replaceAll(":has-text", ":contains")
+        .replaceAll(":text", ":contains");
     return await page.evaluate((locator) => {
         const elements = window.jQuery(locator.cssForJq);
         let selectedElem = elements[0];

package/dist/agent/browsing/index.d.ts

@@ -5,6 +5,7 @@ type BrowsingAgentOptions = Partial<TestGenConfigOptions> & {
         disallowedStrings?: string[];
     };
 };
+export declare function browsingAgentUsingMasterAgent(task: string, page: Page, options: BrowsingAgentOptions): Promise<string>;
 export declare function browsingAgent(task: string, page: Page, options: BrowsingAgentOptions): Promise<string>;
 export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAInD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAkG9B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAiK9B;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAqG9B"}

package/dist/agent/browsing/index.js

@@ -1,13 +1,159 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.browsingAgent = void 0;
+exports.browsingAgent = exports.browsingAgentUsingMasterAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const actions_1 = require("../../actions");
 const logger_1 = require("../../bin/logger");
 const constants_1 = require("../../constants");
 const session_1 = require("../../session");
 const html_1 = require("../../utils/html");
+const run_1 = require("../master/run");
+const verification_1 = require("../verification");
 const utils_1 = require("./utils");
+async function browsingAgentUsingMasterAgent(task, page, options) {
+    const logger = new logger_1.CustomLogger();
+    const trace = llm_1.langfuseInstance.trace({
+        name: "test-generator",
+        ...(0, session_1.getSessionDetails)(),
+        tags: [
+            options.metadata?.projectName,
+            options.metadata?.environment,
+        ].filter((s) => !!s),
+    });
+    const llm = new llm_1.LLM({
+        trace,
+        provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: options.model || constants_1.DEFAULT_MODEL,
+        providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+    });
+    const actions = new actions_1.PlaywrightActions(page);
+    const tools = actions.getActionSchemas();
+    await (0, utils_1.injectPwLocatorGenerator)(page);
+    trace.update({ input: { task } });
+    let lastActionExecTrace = "";
+    let isGivenTaskDone = false;
+    const masterAgentActions = [];
+    while (!isGivenTaskDone) {
+        const masterAgentSpan = trace.span({ name: "master-agent" });
+        if (masterAgentActions.length > 0) {
+            const verificationAgentResp = await (0, verification_1.verificationAgent)({
+                llm,
+                trace: masterAgentSpan,
+                task,
+                conversation: ["Successful actions", ...masterAgentActions],
+            });
+            isGivenTaskDone = verificationAgentResp.isDone;
+            if (isGivenTaskDone) {
+                logger.log(`Master agent task is done: ${verificationAgentResp.reason}`);
+                break;
+            }
+        }
+        const { action, reason } = await (0, run_1.masterAgent)(task, page, masterAgentActions, masterAgentSpan, llm, options);
+        logger.log(`Next action: ${action} \n reason: ${reason}`);
+        if (isGivenTaskDone) {
+            break;
+        }
+        let isTaskDone = false;
+        const executedActions = [];
+        while (!isTaskDone) {
+            const browsingAgentSpan = masterAgentSpan.span({
+                name: `browsing-agent`,
+            });
+            const pageContentSpan = browsingAgentSpan.span({ name: "page-content" });
+            const pageContent = await page.content();
+            pageContentSpan.end({ output: { pageContent } });
+            const sanitizationSpan = browsingAgentSpan.span({
+                name: "page-sanitization",
+            });
+            const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
+            sanitizationSpan.end({ output: { pageSnapshot } });
+            const promptSpan = browsingAgentSpan.span({ name: "page-prompt" });
+            // extract all successful actions
+            const successfulActions = executedActions
+                .filter((a) => !a.isError)
+                .map((a) => a.action);
+            if (successfulActions.length > 0) {
+                const verificationAgentResp = await (0, verification_1.verificationAgent)({
+                    llm,
+                    trace: browsingAgentSpan,
+                    task: action,
+                    conversation: ["Successful actions", ...successfulActions],
+                });
+                isTaskDone = verificationAgentResp.isDone;
+                if (isTaskDone) {
+                    browsingAgentSpan.event({ name: "task-done" });
+                    browsingAgentSpan.end({
+                        output: { taskDone: true, reason: verificationAgentResp.reason },
+                    });
+                    break;
+                }
+            }
+            const messages = await (0, utils_1.getPromptForNextAction)({
+                pageSnapshot,
+                previousActions: successfulActions,
+                task: action,
+                lastActionErrors: lastActionExecTrace ? [lastActionExecTrace] : [],
+                promptType: "browsing-agent-as-tool",
+            });
+            promptSpan.end({ output: { messages } });
+            const completion = await llm.createChatCompletion({
+                messages,
+                tools,
+                trace: browsingAgentSpan,
+                model: options.model || constants_1.DEFAULT_MODEL,
+                modelParameters: {
+                    ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                    ...options.modelParameters,
+                    tool_choice: "required",
+                },
+            });
+            const toolCalls = completion?.tool_calls || [];
+            const toolCallsSpan = browsingAgentSpan.span({ name: "tool-calls" });
+            for (const i in toolCalls) {
+                const toolCall = toolCalls[i];
+                try {
+                    await actions.executeAction(toolCall.function.name, JSON.parse(toolCall.function.arguments));
+                    executedActions.push({
+                        isError: false,
+                        action: JSON.stringify(toolCall),
+                    });
+                    lastActionExecTrace = "";
+                }
+                catch (e) {
+                    // TODO: implement feedback loop to llm
+                    executedActions.push({
+                        isError: true,
+                        action: JSON.stringify(toolCall.function.arguments)
+                            ?.reason,
+                    });
+                    lastActionExecTrace = e.message;
+                    logger.error(lastActionExecTrace, e);
+                }
+            }
+            toolCallsSpan.end({ output: { toolCalls } });
+            // mark task as done if llm is stuck in loop
+            if (executedActions.length > 4) {
+                const lastThreeActions = executedActions.slice(-4);
+                const lastThreeActionsFailed = lastThreeActions.every((a) => a.isError);
+                if (lastThreeActionsFailed) {
+                    // TODO: this should be sent to dashboard
+                    logger.error("Agent is not able to figure out next action, marking task as done");
+                    isTaskDone = true;
+                    break;
+                }
+            }
+        }
+        masterAgentSpan.end({ output: { action, reason } });
+        masterAgentActions.push(action);
+    }
+    await page.close();
+    const code = actions.generateCode();
+    trace.update({ input: { task }, output: { code } });
+    logger.success("Successfully generated code for the given task");
+    logger.log(`Trace: ${trace.getTraceUrl()}`);
+    return code;
+}
+exports.browsingAgentUsingMasterAgent = browsingAgentUsingMasterAgent;
 async function browsingAgent(task, page, options) {
     const logger = new logger_1.CustomLogger();
     const session = (0, session_1.getSessionDetails)();
@@ -27,6 +173,12 @@ async function browsingAgent(task, page, options) {
     await (0, utils_1.injectPwLocatorGenerator)(page);
     trace.update({ input: { task } });
     let lastActionExecTrace = "";
+    const llm = new llm_1.LLM({
+        trace,
+        provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: options.model || constants_1.DEFAULT_MODEL,
+        providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+    });
     while (!isTaskDone) {
         const pageContentSpan = trace.span({ name: "page-content" });
         const pageContent = await page.content();
@@ -46,13 +198,9 @@ async function browsingAgent(task, page, options) {
             lastActionErrors: lastActionExecTrace ? [lastActionExecTrace] : [],
         });
         promptSpan.end({ output: { messages } });
-        const completion = await (0, llm_1.getLLMResult)({
+        const completion = await llm.createChatCompletion({
             messages,
             tools,
-            trace,
-            model: options.model || constants_1.DEFAULT_MODEL,
-            provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-            providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
             modelParameters: {
                 ...constants_1.DEFAULT_MODEL_PARAMETERS,
                 ...options.modelParameters,
@@ -92,6 +240,7 @@ async function browsingAgent(task, page, options) {
             }
         }
     }
+    await page.close();
     const code = actions.generateCode();
     trace.update({ input: { task }, output: { code } });
     logger.success("Successfully generated code for the given task");

package/dist/agent/browsing/utils.d.ts

@@ -14,10 +14,11 @@ export declare function readPlaywrightConfig(): Promise<PlaywrightTestConfig>;
  * @returns
  */
 export declare function detectProjectName(testFilePath: string, playwrightConfig: PlaywrightTestConfig): Promise<string>;
-export declare function getPromptForNextAction({ pageSnapshot, task, previousActions, lastActionErrors, }: {
+export declare function getPromptForNextAction({ pageSnapshot, task, previousActions, lastActionErrors, promptType, }: {
     pageSnapshot: string;
     task: string;
     previousActions: string[];
     lastActionErrors: string[];
+    promptType?: string;
 }): Promise<import("openai/resources/index.mjs").ChatCompletionMessageParam[]>;
 //# sourceMappingURL=utils.d.ts.map

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AASvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAqCzE;AAiBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,GACtB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,8EASA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AASvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAqCzE;AAiBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}

package/dist/agent/browsing/utils.js

@@ -155,9 +155,9 @@ async function detectProjectName(testFilePath, playwrightConfig) {
     return project;
 }
 exports.detectProjectName = detectProjectName;
-async function getPromptForNextAction({ pageSnapshot = "", task = "", previousActions = [], lastActionErrors = [], }) {
+async function getPromptForNextAction({ pageSnapshot = "", task = "", previousActions = [], lastActionErrors = [], promptType = "browsing-agent-next-action", }) {
     const previousActionsStr = previousActions.join("\n\n ---- \n\n");
-    const prompt = await (0, llm_1.getPrompt)("browsing-agent-next-action", {
+    const prompt = await (0, llm_1.getPrompt)(promptType, {
         pageSnapshot,
         previousActionsStr,
         task,

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAmJrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgJrB"}

package/dist/agent/codegen/run.js

@@ -50,12 +50,14 @@ async function generateTest(testCase, file, options) {
         scenarioFile: file,
     });
     promptSpan.end({ output: { instruction } });
-    const firstShotMessage = await (0, llm_1.getLLMResult)({
-        messages: instruction,
+    const llm = new llm_1.LLM({
         trace,
-        model: options.model || constants_1.DEFAULT_MODEL,
         provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: options.model || constants_1.DEFAULT_MODEL,
         providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+    });
+    const firstShotMessage = await llm.createChatCompletion({
+        messages: instruction,
         modelParameters: {
             ...constants_1.DEFAULT_MODEL_PARAMETERS,
             ...options.modelParameters,
@@ -112,12 +114,8 @@ async function generateTest(testCase, file, options) {
             scenaioName: testCase.name,
         });
         promptSpan.end({ output: { instruction } });
-        const message = await (0, llm_1.getLLMResult)({
+        const message = await llm.createChatCompletion({
             messages: instruction,
-            trace,
-            model: options.model || constants_1.DEFAULT_MODEL,
-            provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-            providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
             modelParameters: {
                 ...constants_1.DEFAULT_MODEL_PARAMETERS,
                 ...options.modelParameters,

package/dist/agent/master/run.d.ts

@@ -0,0 +1,7 @@
+import { LLM, TraceClient } from "@empiricalrun/llm";
+import { Page } from "playwright";
+import { TestGenConfigOptions } from "../../types";
+type BrowsingAgentOptions = Partial<TestGenConfigOptions>;
+export declare function masterAgent(task: string, page: Page, executedActions: string[], trace: TraceClient, llm: LLM, options: BrowsingAgentOptions): Promise<any>;
+export {};
+//# sourceMappingURL=run.d.ts.map

package/dist/agent/master/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAKlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAE1D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,eAAe,EAAE,MAAM,EAAE,EACzB,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,oBAAoB,gBAgE9B"}

package/dist/agent/master/run.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.masterAgent = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const done_1 = require("../../actions/done");
+const next_task_1 = require("../../actions/next-task");
+const constants_1 = require("../../constants");
+async function masterAgent(task, page, executedActions, trace, llm, options) {
+    trace.update({ input: { task } });
+    const promptSpan = trace.span({ name: "page-prompt" });
+    const buffer = await page.screenshot({ fullPage: true });
+    const pageScreenshot = `data:image/png;base64,${buffer.toString("base64")}`;
+    const promptMessages = await (0, llm_1.getPrompt)("test-gen", {
+        task,
+        executedActions: executedActions.map((a) => a).join("\n"),
+        pageUrl: page.url(),
+    });
+    // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
+    const userMessage = promptMessages.filter((m) => m.role === "user")[0];
+    const systemMessage = promptMessages.filter((m) => m.role === "system")[0];
+    userMessage.content = [
+        {
+            type: "text",
+            text: userMessage.content,
+        },
+        {
+            type: "image_url",
+            image_url: {
+                url: pageScreenshot,
+            },
+        },
+    ];
+    const messages = [
+        systemMessage,
+        userMessage,
+    ];
+    const tools = [next_task_1.NextTaskAction.schema, (0, done_1.doneActionGenerator)(page).schema];
+    promptSpan.end({ output: { messages } });
+    const completion = await llm.createChatCompletion({
+        messages,
+        modelParameters: {
+            ...constants_1.DEFAULT_MODEL_PARAMETERS,
+            ...options.modelParameters,
+            tool_choice: "required",
+        },
+        trace,
+        // @ts-ignore
+        tools,
+    });
+    let output;
+    const toolCall = completion?.tool_calls?.[0];
+    if (toolCall) {
+        if (toolCall.function.name === "task_done") {
+            output = {
+                action: "",
+                isDone: true,
+                reason: JSON.parse(toolCall.function.arguments).reason,
+            };
+        }
+        else {
+            output = {
+                isDone: false,
+                action: JSON.parse(toolCall.function.arguments).action,
+                reason: JSON.parse(toolCall.function.arguments).reason,
+            };
+        }
+    }
+    trace.update({ input: { task }, output: { output } });
+    return output;
+}
+exports.masterAgent = masterAgent;

package/dist/agent/verification/index.d.ts

@@ -0,0 +1,14 @@
+import { LLM, TraceClient } from "@empiricalrun/llm";
+/**
+ * This agent is used to verify whether the task is done basis the conversation history
+ */
+export declare function verificationAgent({ llm, trace, task, conversation, }: {
+    llm: LLM;
+    trace?: TraceClient;
+    conversation: string[];
+    task: string;
+}): Promise<{
+    isDone: boolean;
+    reason: string;
+}>;
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/verification/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA4DA"}

package/dist/agent/verification/index.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.verificationAgent = void 0;
+/**
+ * This agent is used to verify whether the task is done basis the conversation history
+ */
+async function verificationAgent({ llm, trace, task, conversation, }) {
+    const response = await llm.createChatCompletion({
+        trace,
+        messages: [
+            {
+                role: "system",
+                content: "Given a conversation and a task, your task is to analyse the conversation and tell if the task is completed. If not, you need to tell what is not completed and suggest next steps to complete the task.",
+            },
+            {
+                role: "user",
+                content: `
+Task: ${task}
+
+Conversation:
+${conversation.join("\n")}
+        `,
+            },
+        ],
+        tools: [
+            {
+                type: "function",
+                function: {
+                    name: "task_done",
+                    description: "end the task by calling this method",
+                    parameters: {
+                        type: "object",
+                        properties: {
+                            isDone: {
+                                type: "boolean",
+                                description: "whether the task is done",
+                            },
+                            reason: {
+                                type: "string",
+                                description: "reason for declaring the task is complete",
+                            },
+                        },
+                        required: ["isDone", "reason"],
+                    },
+                },
+            },
+        ],
+        model: "gpt-4o",
+        modelParameters: {
+            tool_choice: "required",
+        },
+    });
+    const toolCallResp = (response?.tool_calls || [])[0];
+    if (toolCallResp) {
+        const toolCall = JSON.parse(toolCallResp.function.arguments);
+        return {
+            isDone: toolCall.isDone,
+            reason: toolCall.reason,
+        };
+    }
+    return {
+        isDone: false,
+        reason: "LLM failed to generate a valid response",
+    };
+}
+exports.verificationAgent = verificationAgent;

package/dist/bin/logger/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/logger/index.ts"],"names":[],"mappings":"AAKA,qBAAa,YAAY;IACvB,OAAO,CAAC,WAAW,CAAkB;gBACzB,EAAE,WAAkB,EAAE;;KAAK;IAIvC,OAAO,CAAC,aAAa;IAKrB,GAAG,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAK9C,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAK/C,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAKlD,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAKhD,YAAY;CAGb"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/logger/index.ts"],"names":[],"mappings":"AAKA,qBAAa,YAAY;IACvB,OAAO,CAAC,WAAW,CAAkB;gBACzB,EAAE,WAAkB,EAAE;;KAAK;IAIvC,OAAO,CAAC,aAAa;IASrB,GAAG,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAK9C,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAK/C,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAKlD,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAKhD,YAAY;CAGb"}

package/dist/bin/logger/index.js

@@ -11,7 +11,9 @@ class CustomLogger {
     }
     logToReporter(message) {
         if (this.useReporter) {
-            (0, reporter_2.getReporter)()?.report(new reporter_1.ProcessLogMessageBuilder({ message: message }));
+            (async () => {
+                await (0, reporter_2.getReporter)()?.report(new reporter_1.ProcessLogMessageBuilder({ message: message }));
+            })();
         }
     }
     log(message, ...optionalParams) {

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBAsBnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBA0BnE"}

package/dist/index.js

@@ -24,7 +24,10 @@ async function createTest(task, page, test) {
     });
     const fileService = new client_1.default(Number(port));
     test.setTimeout(900000);
-    const code = await (0, browsing_1.browsingAgent)(task, page, {
+    const agent = testGenConfig.options?.agent === "browser"
+        ? browsing_1.browsingAgent
+        : browsing_1.browsingAgentUsingMasterAgent;
+    const code = await agent(task, page, {
         htmlSanitize: {
             disallowedStrings: ["v-data-table__td v-data-table-column--align-start"],
         },

package/dist/types/index.d.ts

@@ -6,7 +6,7 @@ export type FileContent = {
     content: string;
 };
 export type TestGenConfigOptions = {
-    agent: "code" | "browser";
+    agent: "code" | "browser" | "master";
     model: LLMModel;
     modelProvider: LLMProvider;
     modelParameters?: ModelParameters;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;IACrC,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}

package/dist/utils/html.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../src/utils/html.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,YAAY,QAClB,MAAM,YACF;IACP,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B,WAuBF,CAAC"}
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../src/utils/html.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,YAAY,QAClB,MAAM,YACF;IACP,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B,WAwBF,CAAC"}

package/dist/utils/html.js

@@ -18,6 +18,7 @@ const sanitizeHtml = (str, options = {}) => {
             "h1",
             "h2",
             "h3",
+            "label",
         ]),
         allowedAttributes: false,
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.20.4",
+  "version": "0.21.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -40,7 +40,7 @@
     "slugify": "^1.6.6",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.4.1",
+    "@empiricalrun/llm": "^0.4.3",
     "@empiricalrun/reporter": "^0.14.1"
   },
   "devDependencies": {