@empiricalrun/test-gen 0.31.19 → 0.31.21

package/dist/agent/master/run.js

@@ -1,18 +1,28 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.masterAgent = exports.getNextAction = void 0;
+exports.createTestUsingMasterAgent = exports.getNextAction = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
+const actions_1 = require("../../actions");
 const next_task_1 = require("../../actions/next-task");
+const skill_1 = require("../../actions/skill");
+const logger_1 = require("../../bin/logger");
 const constants_1 = require("../../constants");
 const reporter_1 = require("../../reporter");
-async function getNextAction({ task, executedActions, page, trace, llm, options, pageScreenshot, }) {
-    const promptSpan = trace?.span({ name: "page-prompt" });
+const session_1 = require("../../session");
+const browsing_1 = require("../browsing");
+const utils_1 = require("../browsing/utils");
+const skills_retriever_1 = require("../codegen/skills-retriever");
+const verification_1 = require("../verification");
+const MAX_ERROR_COUNT = 2;
+async function getNextAction({ task, executedActions, failedActions, page, trace, llm, options, pageScreenshot, actions, disableSkills, }) {
+    const promptSpan = trace?.span({ name: "master-agent-prompt" });
     const promptMessages = await (0, llm_1.getPrompt)("test-gen", {
         task,
+        failedActions: failedActions.map((a) => a).join("\n"),
         executedActions: executedActions.map((a) => a).join("\n"),
         pageUrl: page.url(),
-    });
+    }, 14);
     // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
     const userMessage = promptMessages.filter((m) => m.role === "user")[0];
     const systemMessage = promptMessages.filter((m) => m.role === "system")[0];
@@ -32,7 +42,10 @@ async function getNextAction({ task, executedActions, page, trace, llm, options,
         systemMessage,
         userMessage,
     ];
-    const tools = [next_task_1.NextTaskAction.schema];
+    const actionSchemas = disableSkills || skill_1.testCaseSkills.getAvailableSkills().length === 0
+        ? []
+        : actions.getMasterActionSchemas();
+    const tools = [next_task_1.NextTaskAction.schema, ...actionSchemas];
     promptSpan?.end({ output: { messages } });
     const completion = await llm.createChatCompletion({
         messages,
@@ -43,6 +56,7 @@ async function getNextAction({ task, executedActions, page, trace, llm, options,
             temperature: 1,
         },
         trace,
+        traceName: "master-agent-llm",
         // @ts-ignore
         tools,
     });
@@ -50,43 +64,169 @@ async function getNextAction({ task, executedActions, page, trace, llm, options,
     return toolCall;
 }
 exports.getNextAction = getNextAction;
-async function masterAgent(task, page, executedActions, trace, llm, options) {
-    trace.update({ input: { task } });
-    const buffer = await page.screenshot({ fullPage: true });
-    const testGenReporter = new reporter_1.TestGenUpdatesReporter();
-    const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
-    const pageScreenshot = buffer.toString("base64");
-    let output;
-    const toolCall = await getNextAction({
-        task,
-        executedActions,
-        page,
+async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
+    const logger = new logger_1.CustomLogger({ useReporter: false });
+    const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
+    // add timeout for the page to settle in
+    await page.waitForTimeout(3000);
+    const trace = llm_1.langfuseInstance.trace({
+        name: "test-generator",
+        id: crypto.randomUUID(),
+        version: (0, session_1.getSessionDetails)().version,
+        metadata: {
+            generationId: (0, session_1.getSessionDetails)().generationId,
+            sessionId: (0, session_1.getSessionDetails)().sessionId,
+        },
+        tags: [
+            options.metadata?.projectName,
+            options.metadata?.environment,
+        ].filter((s) => !!s),
+    });
+    const llm = new llm_1.LLM({
         trace,
-        llm,
+        provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+        defaultModel: options.model || constants_1.DEFAULT_MODEL,
+        providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+        // we will be using google model for larger context window, in such cases 1 million tokens is not enough
+        maxTokens: options.modelProvider === "google" ? 3000000 : 1000000,
+    });
+    const skills = await (0, skills_retriever_1.getAppropriateSkills)({
+        testCase,
+        trace,
+        //@ts-ignore
         options,
-        pageScreenshot,
     });
-    if (toolCall) {
-        if (toolCall.function.name === "task_done") {
-            output = {
-                action: "",
-                isDone: true,
-                reason: JSON.parse(toolCall.function.arguments).reason,
-            };
+    skill_1.testCaseSkills.updateSkills(skills);
+    const actions = new actions_1.PlaywrightActions(page);
+    await (0, utils_1.injectPwLocatorGenerator)(page);
+    trace.update({ input: { task } });
+    let isGivenTaskDone = false;
+    const masterAgentActions = [];
+    let failedActions = [];
+    let disableSkills = false;
+    while (!isGivenTaskDone) {
+        const masterAgentSpan = trace.span({
+            name: "master-agent",
+            input: {
+                task,
+                executedActions: masterAgentActions,
+                failedActions,
+            },
+        });
+        if (masterAgentActions.length > 0) {
+            const verificationAgentResp = await (0, verification_1.verificationAgent)({
+                llm,
+                trace: masterAgentSpan,
+                task,
+                conversation: ["Successfully executed actions", ...masterAgentActions],
+            });
+            isGivenTaskDone = verificationAgentResp.isDone;
+            if (isGivenTaskDone) {
+                await testgenUpdatesReporter.sendMessage(`${verificationAgentResp.reason} Marking the task as done.`);
+                break;
+            }
         }
-        else {
-            output = {
-                isDone: false,
-                action: JSON.parse(toolCall.function.arguments).action,
-                reason: JSON.parse(toolCall.function.arguments).reason,
-            };
+        const buffer = await page.screenshot({ fullPage: true });
+        const testGenReporter = new reporter_1.TestGenUpdatesReporter();
+        const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
+        const pageScreenshot = buffer.toString("base64");
+        let output;
+        const toolCall = await getNextAction({
+            task,
+            executedActions: masterAgentActions,
+            failedActions,
+            page,
+            trace: masterAgentSpan,
+            llm,
+            options,
+            pageScreenshot,
+            actions,
+            disableSkills,
+        });
+        if (toolCall) {
+            const args = JSON.parse(toolCall.function.arguments);
+            const masterAgentActionSpan = masterAgentSpan.span({
+                name: "master-agent-action",
+            });
+            try {
+                output = {
+                    action: JSON.parse(toolCall.function.arguments).action ||
+                        JSON.parse(toolCall.function.arguments).skill,
+                    reason: JSON.parse(toolCall.function.arguments).reason,
+                };
+                void testGenReporter.sendMessage(output.reason);
+                logger.log(`Next Action: ${output.reason}`);
+                if (toolCall.function.name === skill_1.SKILL_USAGE) {
+                    await actions.executeAction(toolCall.function.name, args, masterAgentActionSpan);
+                }
+                else {
+                    await (0, browsing_1.executeTaskUsingBrowsingAgent)({
+                        trace: masterAgentActionSpan,
+                        action: output.action,
+                        logger,
+                        page,
+                        options,
+                        llm,
+                        actions,
+                    });
+                }
+                // resetting error count as there is a successful action
+                failedActions = [];
+                masterAgentActions.push(output.action);
+                masterAgentActionSpan.end({
+                    input: {
+                        action: output.action,
+                        reason: output.reason,
+                        type: toolCall.function.name,
+                    },
+                    output: {
+                        success: true,
+                    },
+                });
+                // enable skills after success
+                disableSkills = false;
+            }
+            catch (e) {
+                if (toolCall.function.name === skill_1.SKILL_USAGE) {
+                    logger.log("Disabling skill usage for next retry");
+                    disableSkills = true;
+                }
+                masterAgentActionSpan.end({
+                    input: {
+                        action: output.action,
+                        reason: output.reason,
+                        type: toolCall.function.name,
+                    },
+                    output: {
+                        error: true,
+                        errorSummary: e.message,
+                    },
+                });
+                logger.error("Failed to run master agent's next task", JSON.stringify(toolCall, null, 2), e);
+                failedActions.push(JSON.stringify(toolCall, null, 2));
+                if (failedActions.length >= MAX_ERROR_COUNT) {
+                    const error = "Agent is not able to figure out next action, marking task as done";
+                    logger.error(error);
+                    await testgenUpdatesReporter.sendMessage(error);
+                    break;
+                }
+            }
         }
+        trace.update({ input: { task }, output: { output } });
+        await testGenSnapshotUpdatePromise;
+        masterAgentSpan.end({
+            output: { action: output.action, reason: output.reason },
+        });
     }
-    trace.update({ input: { task }, output: { output } });
-    if (output.action) {
-        await testGenReporter.sendMessage(output.action);
-    }
-    await testGenSnapshotUpdatePromise;
-    return output;
+    await page.close();
+    const { code, importPaths } = actions.generateCode();
+    trace.update({ input: { task }, output: { code } });
+    logger.success("Successfully generated code for the given task");
+    await testgenUpdatesReporter.sendMessage(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
+    logger.log(`Trace: ${trace.getTraceUrl()}`);
+    return {
+        code,
+        importPaths,
+    };
 }
-exports.masterAgent = masterAgent;
+exports.createTestUsingMasterAgent = createTestUsingMasterAgent;

package/dist/agent/verification/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GAgDA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GAkDA"}

package/dist/agent/verification/index.js

@@ -12,6 +12,8 @@ async function verificationAgent({ llm, trace, task, conversation, }) {
     });
     const response = await llm.createChatCompletion({
         trace,
+        traceName: "verification-agent-llm",
+        model: "gpt-4o",
         messages,
         tools: [
             {

package/dist/bin/utils/context.d.ts

@@ -1,3 +1,4 @@
+export declare function createGitIgnoreFileFilter(): Promise<(pathname: string) => boolean>;
 export declare function contextForGeneration(file: string): Promise<{
     codePrompt: string | undefined;
     pomPrompt: string | undefined;

package/dist/bin/utils/context.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/context.ts"],"names":[],"mappings":"AAKA,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,MAAM;;;;GAatD"}
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/context.ts"],"names":[],"mappings":"AAKA,wBAAsB,yBAAyB,2CAS9C;AAED,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,MAAM;;;;GAOtD"}

package/dist/bin/utils/context.js

@@ -3,11 +3,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.contextForGeneration = void 0;
+exports.contextForGeneration = exports.createGitIgnoreFileFilter = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const ignore_1 = __importDefault(require("ignore"));
 const fs_1 = require("./fs");
-async function contextForGeneration(file) {
+async function createGitIgnoreFileFilter() {
     const ignoreFn = (0, ignore_1.default)();
     if (fs_extra_1.default.existsSync(".gitignore")) {
         // Not checking for nested gitignore
@@ -15,6 +15,11 @@ async function contextForGeneration(file) {
         ignoreFn.add(gitignore);
     }
     const filter = ignoreFn.createFilter();
+    return filter;
+}
+exports.createGitIgnoreFileFilter = createGitIgnoreFileFilter;
+async function contextForGeneration(file) {
+    const filter = await createGitIgnoreFileFilter();
     return {
         codePrompt: await (0, fs_1.generatePromptFromDirectory)("./tests", filter),
         pomPrompt: await (0, fs_1.generatePromptFromDirectory)("./pages", filter),

package/dist/bin/utils/platform/web/index.d.ts

@@ -54,4 +54,5 @@ export declare const injectCodeSnippetBySuiteChain: ({ testFileContent, codeSnip
     codeSnippet: string;
     suites: string[];
 }) => string;
+export declare const importAllExportsStmtFromFilePaths: (filePaths: string[], testFilePath: string) => Promise<string[]>;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAGL,IAAI,EAIL,MAAM,UAAU,CAAC;AAGlB;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAuCA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAGL,IAAI,EAIL,MAAM,UAAU,CAAC;AAGlB;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAuCA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC;AAEF,eAAO,MAAM,iCAAiC,cACjC,MAAM,EAAE,gBACL,MAAM,sBAyBrB,CAAC"}

package/dist/bin/utils/platform/web/index.js

@@ -3,10 +3,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.injectCodeSnippetBySuiteChain = exports.replaceCreateTestWithNewCode = exports.getPageVariableNameFromCreateTest = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.findFirstSerialDescribeBlock = exports.hasTopLevelDescribeConfigureWithSerialMode = exports.getTypescriptTestBlock = void 0;
+exports.importAllExportsStmtFromFilePaths = exports.injectCodeSnippetBySuiteChain = exports.replaceCreateTestWithNewCode = exports.getPageVariableNameFromCreateTest = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.findFirstSerialDescribeBlock = exports.hasTopLevelDescribeConfigureWithSerialMode = exports.getTypescriptTestBlock = void 0;
 const eslint_1 = require("eslint");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
+const path_1 = __importDefault(require("path"));
 const prettier_1 = __importDefault(require("prettier"));
 const ts_morph_1 = require("ts-morph");
 const typescript_1 = __importDefault(require("typescript"));
@@ -345,3 +346,29 @@ const injectCodeSnippetBySuiteChain = ({ testFileContent, codeSnippet, suites, }
     return sourceFile.getFullText();
 };
 exports.injectCodeSnippetBySuiteChain = injectCodeSnippetBySuiteChain;
+const importAllExportsStmtFromFilePaths = async (filePaths, testFilePath) => {
+    const statements = [];
+    for (const filePath of filePaths) {
+        const fullPath = path_1.default.resolve(process.cwd(), filePath);
+        let importPath = path_1.default.relative(path_1.default.dirname(testFilePath), filePath);
+        if (!importPath.startsWith(".")) {
+            importPath = "./" + importPath;
+        }
+        const file = await fs_extra_1.default.readFile(fullPath, "utf-8");
+        const project = new ts_morph_1.Project();
+        const sourceFile = project.createSourceFile("index.ts", file);
+        const exportedFunctions = sourceFile
+            .getFunctions()
+            .filter((fn) => fn.isExported())
+            .map((fn) => fn.getName());
+        const exportedVariables = sourceFile
+            .getVariableDeclarations()
+            .filter((v) => v.isExported())
+            .map((fn) => fn.getName());
+        const imports = [...exportedFunctions, ...exportedVariables];
+        const importStatement = `import { ${imports.join(",")} } from '${importPath.replace(".ts", "")}';`;
+        statements.push(importStatement);
+    }
+    return statements;
+};
+exports.importAllExportsStmtFromFilePaths = importAllExportsStmtFromFilePaths;

package/dist/file/client.d.ts

@@ -1,9 +1,10 @@
 declare class TestFileService {
     baseUrl: string;
     constructor(port: number);
-    updateTest({ generatedCode, task, }: {
+    updateTest({ generatedCode, task, importPaths, }: {
         generatedCode: string;
         task: string;
+        importPaths: string[];
     }): Promise<void>;
 }
 export default TestFileService;

package/dist/file/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/file/client.ts"],"names":[],"mappings":"AAAA,cAAM,eAAe;IACnB,OAAO,EAAE,MAAM,CAAC;gBACJ,IAAI,EAAE,MAAM;IAIlB,UAAU,CAAC,EACf,aAAa,EACb,IAAI,GACL,EAAE;QACD,aAAa,EAAE,MAAM,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC;KACd;CAeF;AAED,eAAe,eAAe,CAAC"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/file/client.ts"],"names":[],"mappings":"AAAA,cAAM,eAAe;IACnB,OAAO,EAAE,MAAM,CAAC;gBACJ,IAAI,EAAE,MAAM;IAIlB,UAAU,CAAC,EACf,aAAa,EACb,IAAI,EACJ,WAAW,GACZ,EAAE;QACD,aAAa,EAAE,MAAM,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;CAeF;AAED,eAAe,eAAe,CAAC"}

package/dist/file/client.js

@@ -5,13 +5,13 @@ class TestFileService {
     constructor(port) {
         this.baseUrl = `http://localhost:${port}`;
     }
-    async updateTest({ generatedCode, task, }) {
+    async updateTest({ generatedCode, task, importPaths, }) {
         const resp = await fetch(`${this.baseUrl}/test`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify({ generatedCode, task }),
+            body: JSON.stringify({ generatedCode, task, importPaths }),
         });
         if (!resp.ok) {
             throw new Error(resp.statusText);

package/dist/file/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AASA,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,IAAI,CAAa;gBACb,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE;IAGtC,WAAW,CAAC,QAAQ,EAAE,MAAM;IAGtB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CA8B1C;AAED,wBAAsB,gBAAgB,kBAAK"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAUA,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,IAAI,CAAa;gBACb,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE;IAGtC,WAAW,CAAC,QAAQ,EAAE,MAAM;IAGtB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAuC1C;AAED,wBAAsB,gBAAgB,kBAAK"}

package/dist/file/server.js

@@ -21,13 +21,14 @@ class FileService {
         const app = (0, express_1.default)();
         app.use(express_1.default.json());
         app.post("/test", async (req, res) => {
-            const { generatedCode } = req.body;
+            const { generatedCode, importPaths } = req.body;
             try {
                 const testFilePath = path_1.default.resolve(process.cwd(), this.filePath);
                 if (testFilePath) {
                     const testFile = fs_1.default.readFileSync(testFilePath, "utf-8");
                     const newContents = (0, web_1.replaceCreateTestWithNewCode)(testFilePath, testFile, generatedCode);
-                    fs_1.default.writeFileSync(testFilePath, newContents, "utf-8");
+                    const importStatements = await (0, web_1.importAllExportsStmtFromFilePaths)(importPaths, this.filePath);
+                    fs_1.default.writeFileSync(testFilePath, `${importStatements.join("\n")}\n${newContents}`, "utf-8");
                     await (0, web_1.lintErrors)(testFilePath);
                     return res.send({ success: true });
                 }

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAalC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,iBA2BxD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAalC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,iBAmCxD"}

package/dist/index.js

@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTest = void 0;
 const llm_1 = require("@empiricalrun/llm");
-const browsing_1 = require("./agent/browsing");
+const run_1 = require("./agent/master/run");
 const utils_1 = require("./bin/utils");
 const client_1 = __importDefault(require("./file/client"));
 const reporter_1 = require("./reporter");
@@ -31,15 +31,23 @@ async function createTest(task, page) {
         generationId: testGenConfig.options?.metadata.generationId,
     });
     const fileService = new client_1.default(Number(port));
-    const code = await (0, browsing_1.browsingAgentUsingMasterAgent)(task, page, {
-        htmlSanitize: {
-            disallowedStrings: ["v-data-table__td v-data-table-column--align-start"],
+    const { code, importPaths } = await (0, run_1.createTestUsingMasterAgent)({
+        testCase: testGenConfig.testCase,
+        page,
+        task,
+        options: {
+            htmlSanitize: {
+                disallowedStrings: [
+                    "v-data-table__td v-data-table-column--align-start",
+                ],
+            },
+            ...testGenConfig.options,
         },
-        ...testGenConfig.options,
     });
     await fileService.updateTest({
         task,
         generatedCode: code,
+        importPaths,
     });
 }
 exports.createTest = createTest;

package/dist/types/index.d.ts

@@ -1,4 +1,4 @@
-import { LLMModel, LLMProvider, ModelParameters } from "@empiricalrun/llm";
+import { LLMModel, LLMProvider, ModelParameters, TraceClient } from "@empiricalrun/llm";
 import OpenAI from "openai";
 import { Page } from "playwright";
 export type FileContent = {
@@ -35,18 +35,24 @@ export type TestCase = {
     group: string;
     suites: string[];
 };
-export type PlaywrightActionGenerator = (page: Page, options?: {
+export type PlaywrightActionGenerator = (page: Page, options: {
     stateVariables: Record<string, any>;
 }) => Action;
 export type ActionSchema = OpenAI.Chat.Completions.ChatCompletionTool;
 export type Action = {
     name: string;
     schema: ActionSchema;
-    execute: (args: Record<string, any>) => Promise<{
+    execute: (options: {
+        args: Record<string, any>;
+        trace?: TraceClient;
+    }) => Promise<{
         locator: string;
     } | void>;
     template: (args: Record<string, any>, options: {
         locator: string;
-    }) => string;
+    }) => {
+        code: string;
+        importPath?: string;
+    };
 };
 //# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,QAAQ,CAAC;IACzB,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,IAAI,EACV,OAAO,CAAC,EAAE;IACR,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACrC,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,QAAQ,CAAC;IACzB,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACrC,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KACzB;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.31.19",
+  "version": "0.31.21",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -44,9 +44,9 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.3",
+    "@empiricalrun/llm": "^0.9.4",
     "@empiricalrun/r2-uploader": "^0.3.3",
-    "@empiricalrun/reporter": "^0.20.2"
+    "@empiricalrun/reporter": "^0.20.3"
   },
   "devDependencies": {
     "@types/detect-port": "^1.3.5",