@empiricalrun/test-gen 0.34.4 → 0.35.0

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @empiricalrun/test-gen
 
+## 0.35.0
+
+### Minor Changes
+
+- 069347f: feat: add support for master agent evals
+- 11e4cbd: feat: add fetch skills agent evals
+
+### Patch Changes
+
+- 297508d: fix: langfuse key errors
+- Updated dependencies [069347f]
+- Updated dependencies [297508d]
+  - @empiricalrun/llm@0.9.21
+
+## 0.34.5
+
+### Patch Changes
+
+- dc5718a: feat: add support for evals
+- Updated dependencies [06cf0d8]
+  - @empiricalrun/llm@0.9.20
+
 ## 0.34.4
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts

@@ -10,7 +10,7 @@ export type BrowsingAgentOptions = Partial<TestGenConfigOptions> & {
 };
 export declare function executeTaskUsingBrowsingAgent({ trace, action, logger, page, options, llm, actions, }: {
     action: string;
-    trace: TraceClient;
+    trace?: TraceClient;
     logger: CustomLogger;
     page: Page;
     options: BrowsingAgentOptions;

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,WAAW,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,iBAyIA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,iBAwIA"}

package/dist/agent/browsing/index.js

@@ -15,30 +15,29 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
     const tools = actions.getBrowsingActionSchemas();
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     while (!isTaskDone) {
-        const browsingAgentSpan = trace.span({
+        const browsingAgentSpan = trace?.span({
             name: `browsing-agent`,
         });
         if (await (0, session_1.shouldStopSession)()) {
             break;
         }
-        const pageContentSpan = browsingAgentSpan.span({
+        const pageContentSpan = browsingAgentSpan?.span({
             name: "page-content",
         });
         const pageContent = await page.content();
-        pageContentSpan.end({ output: { pageContent } });
-        const sanitizationSpan = browsingAgentSpan.span({
+        pageContentSpan?.end({ output: { pageContent } });
+        const sanitizationSpan = browsingAgentSpan?.span({
             name: "page-sanitization",
         });
         const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
-        sanitizationSpan.end({ output: { pageSnapshot } });
-        const promptSpan = browsingAgentSpan.span({ name: "page-prompt" });
+        sanitizationSpan?.end({ output: { pageSnapshot } });
+        const promptSpan = browsingAgentSpan?.span({ name: "page-prompt" });
         // extract all successful actions
         const successfulActions = executedActions
             .filter((a) => !a.isError)
             .map((a) => a.action);
         if (successfulActions.length > 0) {
             const verificationAgentResp = await (0, verification_1.verificationAgent)({
-                llm,
                 trace: browsingAgentSpan,
                 task: action,
                 conversation: ["Successfully executed actions", ...successfulActions],
@@ -47,8 +46,8 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
             logger.log(`isTaskDone: ${isTaskDone}`);
             logger.log(`reason: ${verificationAgentResp.reason}`);
             if (isTaskDone) {
-                browsingAgentSpan.event({ name: "task-done" });
-                browsingAgentSpan.end({
+                browsingAgentSpan?.event({ name: "task-done" });
+                browsingAgentSpan?.end({
                     output: { taskDone: true, reason: verificationAgentResp.reason },
                 });
                 break;
@@ -61,7 +60,7 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
             lastActionErrors: lastActionExecTrace ? [lastActionExecTrace] : [],
             promptType: "browsing-agent-as-tool",
         });
-        promptSpan.end({ output: { messages } });
+        promptSpan?.end({ output: { messages } });
         let completion;
         completion = await (0, o1_completion_1.getO1Completion)({
             //@ts-ignore
@@ -92,7 +91,7 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
                 action: "",
             });
         }
-        const toolCallsSpan = browsingAgentSpan.span({ name: "tool-calls" });
+        const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
         for (const i in toolCalls) {
             const toolCall = toolCalls[i];
             if (await (0, session_1.shouldStopSession)()) {
@@ -117,7 +116,7 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
                 logger.error(lastActionExecTrace, e);
             }
         }
-        toolCallsSpan.end({ output: { toolCalls } });
+        toolCallsSpan?.end({ output: { toolCalls } });
         // mark task as done if llm is stuck in loop
         if (executedActions.length >= 3) {
             const lastThreeActions = executedActions.slice(-3);

package/dist/agent/codegen/create-test-block.js

@@ -15,7 +15,7 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
     const session = (0, session_1.getSessionDetails)();
     trace =
         trace ||
-            llm_1.langfuseInstance.trace({
+            llm_1.langfuseInstance?.trace({
                 name: "create-empty-test-block",
                 id: crypto.randomUUID(),
                 release: session.version,

package/dist/agent/codegen/run.d.ts

@@ -1,4 +1,4 @@
 import { TraceClient } from "@empiricalrun/llm";
 import { TestCase, TestGenConfigOptions } from "../../types";
-export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace: TraceClient): Promise<TestCase[]>;
+export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[]>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkC,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAkBhF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAwGrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkC,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAkBhF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,CAAC,CA0GrB"}

package/dist/agent/codegen/run.js

@@ -31,7 +31,7 @@ async function generateTest(testCase, file, options, trace) {
     }
     const generatedTestCases = [];
     logger.logEmptyLine();
-    const createTestSpan = trace.span({
+    const createTestSpan = trace?.span({
         name: "create-test",
         input: {
             testCase,
@@ -39,7 +39,7 @@ async function generateTest(testCase, file, options, trace) {
             options,
         },
     });
-    createTestSpan.event({
+    createTestSpan?.event({
         name: "collate-files-as-text",
         output: {
             codePrompt,
@@ -47,7 +47,7 @@ async function generateTest(testCase, file, options, trace) {
             testFileContent,
         },
     });
-    const promptSpan = createTestSpan.span({
+    const promptSpan = createTestSpan?.span({
         name: "add-scenario-prompt",
     });
     const instruction = await (0, llm_1.getPrompt)("add-scenario", {
@@ -57,7 +57,7 @@ async function generateTest(testCase, file, options, trace) {
         scenarioSteps: testCase.steps.join("\n"),
         scenarioFile: file,
     });
-    promptSpan.end({ output: { instruction } });
+    promptSpan?.end({ output: { instruction } });
     const llm = new llm_1.LLM({
         trace,
         provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
@@ -73,7 +73,7 @@ async function generateTest(testCase, file, options, trace) {
     });
     let response = firstShotMessage?.content || "";
     logger.success("Test generated successfully!");
-    const readWriteFileSpan = trace.span({ name: "write-to-file" });
+    const readWriteFileSpan = trace?.span({ name: "write-to-file" });
     let contents = fs_extra_1.default.readFileSync(file, "utf-8");
     const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response, testCase?.name);
     let updatedContent = prependContent +
@@ -83,9 +83,9 @@ async function generateTest(testCase, file, options, trace) {
             codeSnippet: `\n\n${strippedContent}`,
         });
     await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
-    readWriteFileSpan.end({ output: { updatedContent } });
+    readWriteFileSpan?.end({ output: { updatedContent } });
     logger.log("Linting generated code...");
-    createTestSpan.event({ name: "lint-file" });
+    createTestSpan?.event({ name: "lint-file" });
     await (0, web_1.lintErrors)(file);
     await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
         trace,
@@ -96,12 +96,14 @@ async function generateTest(testCase, file, options, trace) {
         testCase: testCase,
         options,
     });
-    createTestSpan.event({ name: "format-file" });
+    createTestSpan?.event({ name: "format-file" });
     await (0, web_1.formatCode)(file);
     logger.success("File formatted successfully!");
-    logger.log(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
+    if (trace) {
+        logger.log(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
+    }
     generatedTestCases.push(testCase);
-    createTestSpan.end({ output: { response } });
+    createTestSpan?.end({ output: { response } });
     await (0, llm_1.flushAllTraces)();
     return generatedTestCases;
 }

package/dist/agent/codegen/skills-retriever.d.ts

@@ -1,5 +1,16 @@
 import { TraceClient } from "@empiricalrun/llm";
 import { TestCase, TestGenConfigOptions } from "../../types";
+export declare const fetchPomSkills: ({ testCase, pomFiles, options, trace, }: {
+    testCase: TestCase;
+    pomFiles?: string | undefined;
+    trace?: TraceClient | undefined;
+    options?: TestGenConfigOptions | undefined;
+}) => Promise<{
+    testStep: string;
+    filePath: string;
+    usageExample: string;
+    reason: string;
+}[]>;
 export declare function getAppropriateSkills({ testCase, options, trace, }: {
     testCase: TestCase;
     options?: TestGenConfigOptions;

package/dist/agent/codegen/skills-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"skills-retriever.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/skills-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAYhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,oBAAoB,CAAC,EACzC,QAAQ,EACR,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;;;;;KAmDA"}
+{"version":3,"file":"skills-retriever.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/skills-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAYhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,eAAO,MAAM,cAAc;cAMf,QAAQ;;;;;;;;;IA0CnB,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,EACzC,QAAQ,EACR,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;;;;;KA6BA"}

package/dist/agent/codegen/skills-retriever.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAppropriateSkills = void 0;
+exports.getAppropriateSkills = exports.fetchPomSkills = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_1 = __importDefault(require("fs"));
 const logger_1 = require("../../bin/logger");
@@ -11,18 +11,15 @@ const context_1 = require("../../bin/utils/context");
 const fs_2 = require("../../bin/utils/fs");
 const constants_1 = require("../../constants");
 const utils_1 = require("./utils");
-async function getAppropriateSkills({ testCase, options, trace, }) {
-    const logger = new logger_1.CustomLogger({ useReporter: false });
-    logger.log("getting skill set for the repository");
-    const filter = await (0, context_1.createGitIgnoreFileFilter)();
-    const pomFiles = await (0, fs_2.generatePromptFromDirectory)("./pages", filter);
-    const fetchSkillsSpan = trace?.span({
+const fetchPomSkills = async ({ testCase, pomFiles, options, trace, }) => {
+    const fetchSkillsUsingPOMFilesSpan = trace?.span({
         name: "fetch-pom-skills",
         input: {
+            pomFiles,
             testCase,
         },
     });
-    const promptSpan = fetchSkillsSpan?.span({
+    const promptSpan = fetchSkillsUsingPOMFilesSpan?.span({
         name: "fetch-pom-skills-prompt",
     });
     const prompt = await (0, llm_1.getPrompt)("fetch-skills-prompt", {
@@ -32,7 +29,7 @@ async function getAppropriateSkills({ testCase, options, trace, }) {
     });
     promptSpan?.end({ output: { prompt } });
     const llm = new llm_1.LLM({
-        trace: fetchSkillsSpan,
+        trace: fetchSkillsUsingPOMFilesSpan,
         provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
         defaultModel: options?.model || constants_1.DEFAULT_MODEL,
         providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
@@ -47,6 +44,27 @@ async function getAppropriateSkills({ testCase, options, trace, }) {
     });
     let response = firstShotMessage?.content || "";
     const skills = (0, utils_1.extractTestStepsSuggestions)(response);
+    fetchSkillsUsingPOMFilesSpan?.end({ output: { skills } });
+    return skills;
+};
+exports.fetchPomSkills = fetchPomSkills;
+async function getAppropriateSkills({ testCase, options, trace, }) {
+    const logger = new logger_1.CustomLogger({ useReporter: false });
+    logger.log("getting skill set for the repository");
+    const filter = await (0, context_1.createGitIgnoreFileFilter)();
+    const pomFiles = await (0, fs_2.generatePromptFromDirectory)("./pages", filter);
+    const fetchSkillsSpan = trace?.span({
+        name: "get-appropriate-skills",
+        input: {
+            testCase,
+        },
+    });
+    const skills = await (0, exports.fetchPomSkills)({
+        testCase,
+        pomFiles,
+        trace: fetchSkillsSpan,
+        options,
+    });
     const validateSkillsSpan = fetchSkillsSpan?.span({
         name: "validate-skills",
         input: {

package/dist/agent/codegen/update-flow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAsB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CAoG5B;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA6E7B"}
+{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAsB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CAsG5B;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA+E7B"}

package/dist/agent/codegen/update-flow.js

@@ -29,7 +29,7 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
         if (testBlockUpdate) {
             // assuming the test case getting updated
             // maintaining the previous accuracy of the test case update
-            const readWriteFileSpan = trace.span({ name: "write-to-file" });
+            const readWriteFileSpan = trace?.span({ name: "write-to-file" });
             let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
             const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(fileChange.newCode, testCase?.name);
             let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
@@ -41,10 +41,10 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
             contents = contents.replace(testBlock, `\n\n${strippedContent}`);
             updatedContent = prependContent + contents;
             await fs_extra_1.default.writeFile(fileChange.filePath, updatedContent, "utf-8");
-            readWriteFileSpan.end({ output: { updatedContent } });
+            readWriteFileSpan?.end({ output: { updatedContent } });
         }
         else {
-            const readWriteFileSpan = trace.span({ name: "write-to-file" });
+            const readWriteFileSpan = trace?.span({ name: "write-to-file" });
             let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
             const project = new ts_morph_1.Project();
             const sourceFile = project.createSourceFile("updated-code.ts", fileChange.newCode);
@@ -84,7 +84,7 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
                 contents = contents.replace(fileChange.oldCode, `\n\n${fileChange.newCode}`);
             }
             await fs_extra_1.default.writeFile(fileChange.filePath, contents, "utf-8");
-            readWriteFileSpan.end({ output: { contents } });
+            readWriteFileSpan?.end({ output: { contents } });
         }
         // format and validate file change
         if (validateTypes) {
@@ -98,7 +98,7 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
                 options: testGenOptions,
             });
         }
-        trace.event({ name: "format-file" });
+        trace?.event({ name: "format-file" });
         await (0, web_1.formatCode)(fileChange.filePath);
         logger.success(`${fileChange.filePath} file formatted successfully!`);
     }));
@@ -112,7 +112,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
     const session = (0, session_1.getSessionDetails)();
     trace =
         trace ||
-            llm_1.langfuseInstance.trace({
+            llm_1.langfuseInstance?.trace({
                 name: "update-test",
                 id: crypto_1.default.randomUUID(),
                 release: session.version,
@@ -121,7 +121,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
                     options?.metadata.environment || "",
                 ].filter((s) => !!s),
             });
-    const updateTestSpan = trace.span({
+    const updateTestSpan = trace?.span({
         name: "update-test",
         input: {
             testCase,
@@ -129,7 +129,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
             options,
         },
     });
-    updateTestSpan.event({
+    updateTestSpan?.event({
         name: "collate-files-as-text",
         output: {
             codePrompt,
@@ -137,7 +137,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
             testFileContent,
         },
     });
-    const promptSpan = updateTestSpan.span({
+    const promptSpan = updateTestSpan?.span({
         name: "update-scenario-prompt",
     });
     const promptName = "update-scenario";
@@ -161,7 +161,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
         scenarioFile: file,
         currentScenarioCodeBlock,
     });
-    promptSpan.end({ output: { instruction } });
+    promptSpan?.end({ output: { instruction } });
     const llm = new llm_1.LLM({
         trace: updateTestSpan,
         provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
@@ -188,12 +188,14 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
         pomPrompt: pomPrompt,
         codePrompt: codePrompt,
     });
-    logger.log(`Trace: ${trace.getTraceUrl()}`);
+    if (trace) {
+        logger.log(`Trace: ${trace?.getTraceUrl()}`);
+    }
     generatedTestCases.push({
         ...testCase,
         updatedFiles: fileChanges.map((f) => f.filePath),
     });
-    updateTestSpan.end({ output: { response } });
+    updateTestSpan?.end({ output: { response } });
     await (0, llm_1.flushAllTraces)();
     return generatedTestCases;
 }
@@ -215,7 +217,7 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
     const session = (0, session_1.getSessionDetails)();
     trace =
         trace ||
-            llm_1.langfuseInstance.trace({
+            llm_1.langfuseInstance?.trace({
                 name: "append-create-test-block",
                 id: crypto_1.default.randomUUID(),
                 release: session.version,
@@ -225,7 +227,7 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
                 ].filter((s) => !!s),
             });
     const promptName = "append-create-test-block";
-    const promptSpan = trace.span({
+    const promptSpan = trace?.span({
         name: "append-create-test-block-prompt",
     });
     const instruction = await (0, llm_1.getPrompt)(promptName, {
@@ -235,7 +237,7 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
         scenarioSteps: testCase.steps.join("\n"),
         scenarioFile: file,
     });
-    promptSpan.end({ output: { instruction } });
+    promptSpan?.end({ output: { instruction } });
     const [userInstruction] = instruction.filter((s) => s.role === "user");
     const [systemInstruction] = instruction.filter((s) => s.role === "system");
     userInstruction.content = `${systemInstruction?.content} 
@@ -267,12 +269,14 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
         codePrompt: codePrompt,
         validateTypes,
     });
-    logger.log(`Trace: ${trace.getTraceUrl()}`);
+    if (trace) {
+        logger.log(`Trace: ${trace.getTraceUrl()}`);
+    }
     generatedTestCases.push({
         ...testCase,
         updatedFiles: fileChanges.map((f) => f.filePath),
     });
-    trace.update({ input: { testCase }, output: { response } });
+    trace?.update({ input: { testCase }, output: { response } });
     await (0, llm_1.flushAllTraces)();
     return generatedTestCases;
 }

package/dist/agent/infer-agent/index.d.ts

@@ -6,6 +6,5 @@ export declare function inferAgentBasedTask({ task, options, trace, }: {
     trace?: TraceClient;
 }): Promise<{
     response: Agent;
-    trace: TraceClient;
 }>;
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/infer-agent/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/infer-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG1D,wBAAsB,mBAAmB,CAAC,EACxC,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC,CA6EnD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/infer-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG1D,wBAAsB,mBAAmB,CAAC,EACxC,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,KAAK,CAAA;CAAE,CAAC,CA4E/B"}

package/dist/agent/infer-agent/index.js

@@ -8,19 +8,19 @@ const session = (0, session_1.getSessionDetails)();
 async function inferAgentBasedTask({ task, options, trace, }) {
     trace =
         trace ||
-            llm_1.langfuseInstance.trace({
+            llm_1.langfuseInstance?.trace({
                 name: "infer-agent-task",
                 id: crypto.randomUUID(),
                 release: session.version,
             });
-    const inferAgentSpan = trace.span({
+    const inferAgentSpan = trace?.span({
         name: "infer-agent",
         input: {
             task,
             options,
         },
     });
-    const promptSpan = inferAgentSpan.span({
+    const promptSpan = inferAgentSpan?.span({
         name: "infer-agent-prompt",
         input: {
             task,
@@ -65,7 +65,7 @@ async function inferAgentBasedTask({ task, options, trace, }) {
         },
     });
     const output = JSON.parse(firstShotMessage?.content || "{}");
-    inferAgentSpan.end({
+    inferAgentSpan?.end({
         output: {
             response: output.response,
             reason: output.reason,
@@ -73,7 +73,6 @@ async function inferAgentBasedTask({ task, options, trace, }) {
     });
     return {
         response: output.response,
-        trace: inferAgentSpan,
     };
 }
 exports.inferAgentBasedTask = inferAgentBasedTask;

package/dist/agent/master/run.d.ts

@@ -3,14 +3,14 @@ import { Page } from "playwright";
 import { PlaywrightActions } from "../../actions";
 import { TestCase } from "../../types";
 import { BrowsingAgentOptions } from "../browsing";
-export declare function getNextAction({ task, executedActions, failedActions, page, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints, }: {
+export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints, }: {
     task: string;
     executedActions: string[];
     failedActions: any[];
-    page: Page;
+    pageUrl: string;
     trace?: TraceClient;
-    llm: LLM;
-    options: BrowsingAgentOptions;
+    llm?: LLM;
+    options?: BrowsingAgentOptions;
     pageScreenshot: string;
     annotatedPageScreenshot?: string;
     actions: PlaywrightActions;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAgEA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GA8QA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAwFA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GAiRA"}

package/dist/agent/master/run.js

@@ -16,13 +16,30 @@ const skills_retriever_1 = require("../codegen/skills-retriever");
 const verification_1 = require("../verification");
 const with_hints_1 = require("./with-hints");
 const MAX_ERROR_COUNT = 2;
-async function getNextAction({ task, executedActions, failedActions, page, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints = false, }) {
-    const promptSpan = trace?.span({ name: "master-agent-prompt" });
+async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints = false, }) {
+    const nextActionSpan = trace?.span({
+        name: "master-agent-next-action",
+        input: {
+            task,
+            executedActions,
+            failedActions,
+            pageUrl,
+            options,
+            pageScreenshot,
+            annotatedPageScreenshot,
+            disableSkills,
+            useHints,
+            skills: skill_1.testCaseSkills.getAvailableSkills(),
+        },
+    });
+    const promptSpan = nextActionSpan?.span({
+        name: "master-agent-prompt",
+    });
     const promptMessages = await (0, llm_1.getPrompt)("test-gen", {
         task,
         failedActions: failedActions.map((a) => a).join("\n"),
         executedActions: executedActions.map((a) => a).join("\n"),
-        pageUrl: page.url(),
+        pageUrl,
     }, useHints ? 16 : 14);
     // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
     const userMessage = promptMessages.filter((m) => m.role === "user")[0];
@@ -43,7 +60,7 @@ async function getNextAction({ task, executedActions, failedActions, page, trace
                 {
                     type: "image_url",
                     image_url: {
-                        url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
+                        url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
                     },
                 },
             ];
@@ -56,20 +73,27 @@ async function getNextAction({ task, executedActions, failedActions, page, trace
         : actions.getMasterActionSchemas();
     const tools = [next_task_1.NextTaskAction.schema, ...actionSchemas];
     promptSpan?.end({ output: { messages } });
+    llm =
+        llm ||
+            new llm_1.LLM({
+                provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
+                defaultModel: options?.model || constants_1.DEFAULT_MODEL,
+            });
     const completion = await llm.createChatCompletion({
         messages,
         modelParameters: {
             ...constants_1.DEFAULT_MODEL_PARAMETERS,
-            ...options.modelParameters,
+            ...options?.modelParameters,
             tool_choice: "required",
             temperature: 1,
         },
-        trace,
+        trace: nextActionSpan,
         traceName: "master-agent-llm",
         // @ts-ignore
         tools,
     });
     const toolCall = completion?.tool_calls?.[0];
+    nextActionSpan?.end({ output: toolCall });
     return toolCall;
 }
 exports.getNextAction = getNextAction;
@@ -80,7 +104,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     // add timeout for the page to settle in
     await page.waitForTimeout(3000);
-    const trace = llm_1.langfuseInstance.trace({
+    const trace = llm_1.langfuseInstance?.trace({
         name: "test-generator",
         id: crypto.randomUUID(),
         version: (0, session_1.getSessionDetails)().version,
@@ -93,9 +117,11 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
             options.metadata?.environment,
         ].filter((s) => !!s),
     });
-    void testgenUpdatesReporter.sendMessage(`Starting master agent. [view trace](${trace.getTraceUrl()})`);
-    logger.log(`Starting master agent: ${trace.getTraceUrl()}`);
-    void testgenUpdatesReporter.sendAgentTraceUrl(trace.getTraceUrl());
+    if (trace) {
+        void testgenUpdatesReporter.sendMessage(`Starting master agent. [view trace](${trace?.getTraceUrl()})`);
+        logger.log(`Starting master agent: ${trace?.getTraceUrl()}`);
+        void testgenUpdatesReporter.sendAgentTraceUrl(trace.getTraceUrl());
+    }
     const llm = new llm_1.LLM({
         trace,
         provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
@@ -113,7 +139,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
     skill_1.testCaseSkills.updateSkills(skills);
     const actions = new actions_1.PlaywrightActions(page);
     await (0, utils_1.injectPwLocatorGenerator)(page);
-    trace.update({ input: { task } });
+    trace?.update({ input: { task } });
     let isGivenTaskDone = false;
     const masterAgentActions = [];
     let failedActions = [];
@@ -122,7 +148,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
         if (await (0, session_1.shouldStopSession)()) {
             break;
         }
-        const masterAgentSpan = trace.span({
+        const masterAgentSpan = trace?.span({
             name: "master-agent",
             input: {
                 task,
@@ -132,7 +158,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
         });
         if (masterAgentActions.length > 0) {
             const verificationAgentResp = await (0, verification_1.verificationAgent)({
-                llm,
                 trace: masterAgentSpan,
                 task,
                 conversation: ["Successfully executed actions", ...masterAgentActions],
@@ -190,7 +215,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
             task,
             executedActions: masterAgentActions,
             failedActions,
-            page,
+            pageUrl: page.url(),
             trace: masterAgentSpan,
             llm,
             options,
@@ -205,7 +230,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
         }
         if (toolCall) {
             const args = JSON.parse(toolCall.function.arguments);
-            const masterAgentActionSpan = masterAgentSpan.span({
+            const masterAgentActionSpan = masterAgentSpan?.span({
                 name: "master-agent-action",
             });
             try {
@@ -257,7 +282,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
                 // resetting error count as there is a successful action
                 failedActions = [];
                 masterAgentActions.push(output.action);
-                masterAgentActionSpan.end({
+                masterAgentActionSpan?.end({
                     input: {
                         action: output.action,
                         reason: output.reason,
@@ -275,7 +300,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
                     logger.log("Disabling skill usage for next retry");
                     disableSkills = true;
                 }
-                masterAgentActionSpan.end({
+                masterAgentActionSpan?.end({
                     input: {
                         action: output.action,
                         reason: output.reason,
@@ -296,19 +321,21 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
                 }
             }
         }
-        trace.update({ input: { task }, output: { output } });
+        trace?.update({ input: { task }, output: { output } });
         await testGenSnapshotUpdatePromise;
         if (testGenAnnotatedSnapshotUpdatePromise) {
             await testGenAnnotatedSnapshotUpdatePromise;
         }
-        masterAgentSpan.end({
+        masterAgentSpan?.end({
             output: { action: output.action, reason: output.reason },
         });
     }
     const { code, importPaths } = actions.generateCode();
-    trace.update({ input: { task }, output: { code } });
+    trace?.update({ input: { task }, output: { code } });
     logger.success("Successfully generated code for the given task");
-    await testgenUpdatesReporter.sendMessage(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
+    if (trace) {
+        await testgenUpdatesReporter.sendMessage(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
+    }
     return {
         code,
         importPaths,

package/dist/agent/master/with-hints.d.ts

@@ -4,7 +4,7 @@ import { Page } from "playwright";
 import { BrowsingAgentOptions } from "../browsing";
 export declare const getUserMessageWithForHints: ({ userMessage, options, pageScreenshot, annotatedPageScreenshot, }: {
     userMessage: OpenAI.ChatCompletionUserMessageParam;
-    options: BrowsingAgentOptions;
+    options?: BrowsingAgentOptions | undefined;
     pageScreenshot: string;
     annotatedPageScreenshot: string;
 }) => string | OpenAI.ChatCompletionContentPart[];

package/dist/agent/master/with-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAExC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;aACzC,oBAAoB;oBACb,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAMF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,IAAI;SACL,GAAG;MACN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAgFA,CAAC"}
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAExC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAMF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,IAAI;SACL,GAAG;MACN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAgFA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -17,7 +17,7 @@ const getUserMessageWithForHints = ({ userMessage, options, pageScreenshot, anno
         {
             type: "image_url",
             image_url: {
-                url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
+                url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
             },
         },
         {
@@ -27,7 +27,7 @@ const getUserMessageWithForHints = ({ userMessage, options, pageScreenshot, anno
         {
             type: "image_url",
             image_url: {
-                url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, annotatedPageScreenshot),
+                url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, annotatedPageScreenshot),
             },
         },
     ];

package/dist/agent/verification/index.d.ts

@@ -1,9 +1,8 @@
-import { LLM, TraceClient } from "@empiricalrun/llm";
+import { TraceClient } from "@empiricalrun/llm";
 /**
  * This agent is used to verify whether the task is done basis the conversation history
  */
-export declare function verificationAgent({ llm, trace, task, conversation, }: {
-    llm: LLM;
+export declare function verificationAgent({ trace, task, conversation, }: {
     trace?: TraceClient;
     conversation: string[];
     task: string;

package/dist/agent/verification/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GAkDA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA+EA"}

package/dist/agent/verification/index.js

@@ -5,13 +5,21 @@ const llm_1 = require("@empiricalrun/llm");
 /**
  * This agent is used to verify whether the task is done basis the conversation history
  */
-async function verificationAgent({ llm, trace, task, conversation, }) {
+async function verificationAgent({ trace, task, conversation, }) {
+    const verificationAgentSpan = trace?.span({
+        name: "verification-agent",
+        input: {
+            task,
+            conversation,
+        },
+    });
     const messages = await (0, llm_1.getPrompt)("agent-steps-verification", {
         task,
         conversation: conversation.join("\n"),
-    });
+    }, 4);
+    const llm = new llm_1.LLM({ provider: "openai" });
     const response = await llm.createChatCompletion({
-        trace,
+        trace: verificationAgentSpan,
         traceName: "verification-agent-llm",
         model: "gpt-4o",
         messages,
@@ -24,13 +32,21 @@ async function verificationAgent({ llm, trace, task, conversation, }) {
                     parameters: {
                         type: "object",
                         properties: {
-                            isDone: {
-                                type: "boolean",
-                                description: "whether the task is done",
+                            actions: {
+                                type: "string",
+                                description: "actions extracted from task",
+                            },
+                            successful_actions: {
+                                type: "string",
+                                description: "successful actions mentioned in the conversation",
                             },
                             reason: {
                                 type: "string",
-                                description: "reason for declaring the task is complete",
+                                description: "reasoning for identification of task status",
+                            },
+                            isDone: {
+                                type: "boolean",
+                                description: "whether the task is done",
                             },
                         },
                         required: ["isDone", "reason"],
@@ -40,19 +56,28 @@ async function verificationAgent({ llm, trace, task, conversation, }) {
         ],
         modelParameters: {
             tool_choice: "required",
+            temperature: 0.5,
         },
     });
     const toolCallResp = (response?.tool_calls || [])[0];
     if (toolCallResp) {
         const toolCall = JSON.parse(toolCallResp.function.arguments);
-        return {
+        const output = {
             isDone: toolCall.isDone,
             reason: toolCall.reason,
         };
+        verificationAgentSpan?.end({
+            output,
+        });
+        return output;
     }
-    return {
+    const output = {
         isDone: false,
         reason: "LLM failed to generate a valid response",
     };
+    verificationAgentSpan?.end({
+        output,
+    });
+    return output;
 }
 exports.verificationAgent = verificationAgent;

package/dist/bin/index.js

@@ -46,7 +46,7 @@ async function runAgent(testGenConfig) {
     }
     let agent = testGenConfig.options?.agent;
     const session = (0, session_1.getSessionDetails)();
-    const trace = llm_1.langfuseInstance.trace({
+    const trace = llm_1.langfuseInstance?.trace({
         name: "generate-test",
         id: crypto.randomUUID(),
         release: session.version,
@@ -55,11 +55,13 @@ async function runAgent(testGenConfig) {
             testGenConfig.options?.metadata.environment || "",
         ].filter((s) => !!s),
     });
-    try {
-        await new reporter_1.TestGenUpdatesReporter().sendAgentTraceUrl(trace.getTraceUrl());
-    }
-    catch (e) {
-        console.warn("Failed to send trace url as test gen update", e);
+    if (trace) {
+        try {
+            await new reporter_1.TestGenUpdatesReporter().sendAgentTraceUrl(trace.getTraceUrl());
+        }
+        catch (e) {
+            console.warn("Failed to send trace url as test gen update", e);
+        }
     }
     if (!agent || agent === "auto") {
         agent = await resolveAgentUsingTask({

package/dist/evals/fetch-pom-skills-agent.evals.d.ts

@@ -0,0 +1,4 @@
+import { EvaluateFn } from "./type";
+declare const fetchSkillsAgentEvaluator: EvaluateFn;
+export default fetchSkillsAgentEvaluator;
+//# sourceMappingURL=fetch-pom-skills-agent.evals.d.ts.map

package/dist/evals/fetch-pom-skills-agent.evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-pom-skills-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/fetch-pom-skills-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,QAAA,MAAM,yBAAyB,EAAE,UAiChC,CAAC;AAEF,eAAe,yBAAyB,CAAC"}

package/dist/evals/fetch-pom-skills-agent.evals.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const skills_retriever_1 = require("../agent/codegen/skills-retriever");
+const fetchSkillsAgentEvaluator = async ({ item, trace }) => {
+    const { testCase, pomFiles } = item.input;
+    const output = await (0, skills_retriever_1.fetchPomSkills)({
+        testCase,
+        pomFiles,
+        trace,
+    });
+    if (item.expectedOutput.length === 0 && output.length === 0) {
+        return {
+            scores: [
+                {
+                    name: "equality",
+                    value: 1,
+                },
+            ],
+            output,
+        };
+    }
+    return {
+        scores: [
+            {
+                name: "equality",
+                value: output.some((o) => {
+                    return item.expectedOutput.some((e) => e.usageExample === o.usageExample);
+                })
+                    ? 1
+                    : 0,
+            },
+        ],
+        output,
+    };
+};
+exports.default = fetchSkillsAgentEvaluator;

package/dist/evals/infer-master-or-code-agent.evals.d.ts

@@ -0,0 +1,4 @@
+import { EvaluateFn } from "./type";
+export declare const inferMasterOrCodeAgentEvaluate: EvaluateFn;
+export default inferMasterOrCodeAgentEvaluate;
+//# sourceMappingURL=infer-master-or-code-agent.evals.d.ts.map

package/dist/evals/infer-master-or-code-agent.evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"infer-master-or-code-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/infer-master-or-code-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,8BAA8B,EAAE,UAkB5C,CAAC;AAEF,eAAe,8BAA8B,CAAC"}

package/dist/evals/infer-master-or-code-agent.evals.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inferMasterOrCodeAgentEvaluate = void 0;
+const infer_agent_1 = require("../agent/infer-agent");
+const inferMasterOrCodeAgentEvaluate = async ({ item, trace, }) => {
+    const { task } = item.input;
+    const { response } = await (0, infer_agent_1.inferAgentBasedTask)({
+        task,
+        trace,
+    });
+    return {
+        scores: [
+            {
+                name: "equality",
+                value: item.expectedOutput === response ? 1 : 0,
+            },
+        ],
+        output: response,
+    };
+};
+exports.inferMasterOrCodeAgentEvaluate = inferMasterOrCodeAgentEvaluate;
+exports.default = exports.inferMasterOrCodeAgentEvaluate;

package/dist/evals/master-agent.evals.d.ts

@@ -0,0 +1,4 @@
+import { EvaluateFn } from "./type";
+export declare const masterGetNextActionEvaluator: EvaluateFn;
+export default masterGetNextActionEvaluator;
+//# sourceMappingURL=master-agent.evals.d.ts.map

package/dist/evals/master-agent.evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA0C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/master-agent.evals.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.masterGetNextActionEvaluator = void 0;
+const actions_1 = require("../actions");
+const skill_1 = require("../actions/skill");
+const run_1 = require("../agent/master/run");
+const masterGetNextActionEvaluator = async ({ item, trace, }) => {
+    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, annotatedPageScreenshot, disableSkills, useHints, skills = [], } = item.input;
+    const page = {};
+    skill_1.testCaseSkills.updateSkills(skills);
+    const actions = new actions_1.PlaywrightActions(page);
+    const output = await (0, run_1.getNextAction)({
+        task,
+        executedActions,
+        failedActions,
+        pageUrl,
+        trace,
+        options,
+        pageScreenshot,
+        annotatedPageScreenshot,
+        actions,
+        disableSkills,
+        useHints,
+    });
+    return {
+        scores: [
+            {
+                name: "action_correctness",
+                value: item.expectedOutput.function.name === output?.function.name ? 1 : 0,
+            },
+        ],
+        output,
+    };
+};
+exports.masterGetNextActionEvaluator = masterGetNextActionEvaluator;
+exports.default = exports.masterGetNextActionEvaluator;

package/dist/evals/type.d.ts

@@ -0,0 +1,12 @@
+import { TraceClient } from "@empiricalrun/llm";
+export type EvaluateFn = ({ trace, item, }: {
+    trace: TraceClient;
+    item: any;
+}) => Promise<{
+    output: any;
+    scores: {
+        name: string;
+        value: number;
+    }[];
+}>;
+//# sourceMappingURL=type.d.ts.map

package/dist/evals/type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"type.d.ts","sourceRoot":"","sources":["../../src/evals/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,MAAM,UAAU,GAAG,CAAC,EACxB,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,EAAE,WAAW,CAAC;IACnB,IAAI,EAAE,GAAG,CAAC;CACX,KAAK,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,CAAC;IACZ,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC3C,CAAC,CAAC"}

package/dist/evals/type.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/evals/verification-agent.evals.d.ts

@@ -0,0 +1,4 @@
+import { EvaluateFn } from "./type";
+export declare const verifierAgentEvaluate: EvaluateFn;
+export default verifierAgentEvaluate;
+//# sourceMappingURL=verification-agent.evals.d.ts.map

package/dist/evals/verification-agent.evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verification-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/verification-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,qBAAqB,EAAE,UAgBnC,CAAC;AAEF,eAAe,qBAAqB,CAAC"}

package/dist/evals/verification-agent.evals.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.verifierAgentEvaluate = void 0;
+const verification_1 = require("../agent/verification");
+const verifierAgentEvaluate = async ({ item, trace }) => {
+    const { conversation = [], task = "" } = item.input;
+    const output = await (0, verification_1.verificationAgent)({
+        conversation,
+        trace,
+        task,
+    });
+    return {
+        scores: [
+            {
+                name: "equality",
+                value: item.expectedOutput.isDone === output.isDone ? 1 : 0,
+            },
+        ],
+        output,
+    };
+};
+exports.verifierAgentEvaluate = verifierAgentEvaluate;
+exports.default = exports.verifierAgentEvaluate;

package/dist/session/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAeA,iBAAS,iBAAiB;;;;EAMzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB,QAGA;AAED,wBAAsB,iBAAiB,qBAGtC;AAED,wBAAsB,eAAe,0DAkBpC;AAED,wBAAsB,UAAU,kBAkB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAgBA,iBAAS,iBAAiB;;;;EAMzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB,QAGA;AAED,wBAAsB,iBAAiB,qBAGtC;AAED,wBAAsB,eAAe,0DAqBpC;AAED,wBAAsB,UAAU,kBAqB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/session/index.js

@@ -10,7 +10,8 @@ const sessionDetails = {
     version: package_json_1.default.version,
     generationId: undefined,
 };
-const DASHBOARD_DOMAIN = process.env.DASHBOARD_DOMAIN || "https://dash.empirical.run";
+const DASHBOARD_DOMAIN = process.env.DASHBOARD_DOMAIN ||
+    (process.env.CI === "true" ? "https://dash.empirical.run" : "");
 function getSessionDetails() {
     return {
         generationId: sessionDetails.generationId,
@@ -30,6 +31,9 @@ async function shouldStopSession() {
 }
 exports.shouldStopSession = shouldStopSession;
 async function getSessionState() {
+    if (!DASHBOARD_DOMAIN) {
+        return "started";
+    }
     const apiPath = `${DASHBOARD_DOMAIN}/api/sessions/${sessionDetails.sessionId}/generations/${sessionDetails.generationId}/state`;
     const response = await fetch(apiPath, {
         method: "GET",
@@ -44,6 +48,9 @@ async function getSessionState() {
 }
 exports.getSessionState = getSessionState;
 async function endSession() {
+    if (!DASHBOARD_DOMAIN) {
+        return;
+    }
     const apiPath = `${DASHBOARD_DOMAIN}/api/sessions/${sessionDetails.sessionId}/generations/${sessionDetails.generationId}/state`;
     try {
         await fetch(apiPath, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.34.4",
+  "version": "0.35.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -44,7 +44,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.19",
+    "@empiricalrun/llm": "^0.9.21",
     "@empiricalrun/r2-uploader": "^0.3.6",
     "@empiricalrun/reporter": "^0.21.2"
   },

package/dist/evals/infer-master-code.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=infer-master-code.d.ts.map

package/dist/evals/infer-master-code.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"infer-master-code.d.ts","sourceRoot":"","sources":["../../src/evals/infer-master-code.ts"],"names":[],"mappings":""}

package/dist/evals/infer-master-code.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const llm_1 = require("@empiricalrun/llm");
-const infer_agent_1 = require("../agent/infer-agent");
-const datasetName = "infer-master-or-code-agent";
-(async function main() {
-    const dataset = await llm_1.langfuseInstance.getDataset(datasetName);
-    const runName = `${datasetName}-${Date.now()}`;
-    for (const item of dataset.items) {
-        const { response, trace } = await (0, infer_agent_1.inferAgentBasedTask)(item.input);
-        await item.link(trace, runName, {});
-        trace?.score({
-            name: "equality",
-            value: item.expectedOutput === response ? 1 : 0, // score value
-        });
-    }
-    await llm_1.langfuseInstance.flushAsync();
-})();