@empiricalrun/test-gen 0.71.2 → 0.73.0

package/dist/agent/chat/utils.js

@@ -11,8 +11,8 @@ const log = (...args) => {
     console.log((0, picocolors_1.gray)(args.join(" ")));
 };
 exports.log = log;
-function logError(context, error, trace) {
-    console.error((0, picocolors_1.gray)(`[Error in ${context}]:`), error instanceof Error ? error.stack || error.message : error);
+function logError(error, trace) {
+    console.error((0, picocolors_1.gray)(`[Error]:`), error instanceof Error ? error.stack || error.message : error);
     trace?.update({
         output: {
             error: error instanceof Error ? error.message : String(error),
@@ -29,7 +29,7 @@ function getModelName(model) {
         return "o4";
     return "AI";
 }
-async function handleAgentError({ context, error, chatModel, selectedModel, reporter, trace, }) {
+async function handleAgentError({ error, chatModel, selectedModel, reporter, trace, }) {
     const errorObject = {
         message: error.message,
         stack: error.stack || "Stack trace not available",
@@ -40,14 +40,8 @@ async function handleAgentError({ context, error, chatModel, selectedModel, repo
         selectedModel,
         error: errorObject,
     }), chatModel.getHumanReadableLatestMessage());
-    trace?.update({
-        output: {
-            error: errorObject,
-        },
-    });
-    if (context) {
-        logError(context, error, trace);
-    }
+    trace?.update({ output: { error: errorObject } });
+    logError(error, trace);
 }
 function extractAttachments(userPrompt) {
     const attachments = [];

package/dist/agent/code-review/prompt.d.ts

@@ -0,0 +1,2 @@
+export declare const PROMPT = "\nIdentify code smells in tests\n- try-catch\n- conditionals added without a comment\n\nPlaywright gotchas\n- isVisible(), count() do not auto-wait\n\nPlaywright code smells\n- waitForLoadState is not required\n    - Especially with \"networkidle\", because modern webapps keep doing network activity\n\nIdentify test assumptions\n- What data does the test require?\n- Clean up test entities\n";
+//# sourceMappingURL=prompt.d.ts.map

package/dist/agent/code-review/prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../src/agent/code-review/prompt.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,MAAM,8YAelB,CAAC"}

package/dist/agent/code-review/prompt.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PROMPT = void 0;
+exports.PROMPT = `
+Identify code smells in tests
+- try-catch
+- conditionals added without a comment
+
+Playwright gotchas
+- isVisible(), count() do not auto-wait
+
+Playwright code smells
+- waitForLoadState is not required
+    - Especially with "networkidle", because modern webapps keep doing network activity
+
+Identify test assumptions
+- What data does the test require?
+- Clean up test entities
+`;

package/dist/agent/codegen/create-test-block.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAc3D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BA+CA"}
+{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAa3D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAsCA"}

package/dist/agent/codegen/create-test-block.js

@@ -7,20 +7,10 @@ const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write an empty test block.\nBased on the inputs you need to create an empty playwright test block with correctly imported fixture.\n\nThe test will contain a test name which you will need to use to build the empty test case block.\n\nYou will be provided with current tests, fixtures and page object models for you to use and create test case block as\nper the task provided to you.\n\nBefore responding you need to ensure that the code change is minimal and the change is reusable across tests. You need\nto ensure the code follows DRY principle.\n\nHere is the list of current tests and fixtures:\n\n{{testFiles}}\n\nHere is the list of current page object models:\n\n{{pageFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nFollowing is the test scenario for which you need to write the empty test case block:\ntest name:\n{{scenarioName}}\n\ntask:\ncreate an empty test case block for the following test steps:\n{{scenario}}\n\ntest file path: {{scenarioFile}}\n\n------\n\nYou also need to ensure that the empty test case block has a starting page to begin test.\n\nIn order to identify the right page with which the test should start, follow the steps:\n- based on the similarities with other test cases mentioned in the file, identify the right page fixture to be imported\n- Read the page fixture methods step by step. Identify whether the fixture handles navigating to a page.\n- Identify whether other tests using the page fixture had to add separate steps for navigation or not\n- Based on the above analysis there will be following cases and choose either for the given test scenario:\n-- Case 1: if the test case scenario provided inside the task mentions about page navigation, then use that page\nnavigation. skip other cases if this case is satisfied.\n-- Case 2: refer other test cases which import similar fixtures and infer the first page navigation of this test case.\nYou should prefer tests which are in the same file. Tests within same file have higher overlaps in first page\nnavigation.\n- Once the page fixture is decided, look for userContext fixture in files. If its available then add \"userContext\" to\nthe test case block\n\n\n\nFollow these instructions before responding with output:\n- Read the code line by line and achieve the task provided to you\n- Read the dependencies of the code block by scanning through file paths and file provided to you. refer the same file\npath while responding with update\n- Focus only on the test case provided and associated JS methods called from the test case.\n- Respond only with the new empty test case block to be created and nothing else.\n- DO NOT respond with any backticks or markdown syntax\n- If \"userContext\" fixture is available in fixtures file, ensure importing that fixture in the test case block.\n- Provide a reason based on the test steps provided to you on why you chose the fixture or page.goto statement. The\nreason should be one of the list steps provided to you and mention why the case was chosen\n{{/section}}";
-const session_1 = require("../../session");
 async function createEmptyTestCaseBlock({ testCase, file, trace, }) {
     const logger = new logger_1.CustomLogger({ useReporter: false });
     logger.log("Creating new test block");
     const context = await (0, context_1.contextForGeneration)(file);
-    // TODO: move this to a common place
-    const session = (0, session_1.getSessionDetails)();
-    trace =
-        trace ||
-            llm_1.langfuseInstance?.trace({
-                name: "create-empty-test-block",
-                id: crypto.randomUUID(),
-                release: session.version,
-            });
     const promptSpan = trace?.span({
         name: "build-create-empty-test-case-prompt",
     });

package/dist/agent/diagnosis-agent/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/diagnosis-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAuChD,wBAAsB,+BAA+B,CAAC,EACpD,KAAK,EACL,SAAS,EACT,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,yBAAyB,CAAC;IACrC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CA6E5B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/diagnosis-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAoChD,wBAAsB,+BAA+B,CAAC,EACpD,KAAK,EACL,SAAS,EACT,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,yBAAyB,CAAC;IACrC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAqE5B"}

package/dist/agent/diagnosis-agent/index.js

@@ -2,9 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTaskUsingFailureDiagnosis = createTaskUsingFailureDiagnosis;
 const llm_1 = require("@empiricalrun/llm");
-const session_1 = require("../../session");
 const strict_mode_violation_1 = require("./strict-mode-violation");
-const session = (0, session_1.getSessionDetails)();
 const responseFormat = {
     type: "json_schema",
     json_schema: {
@@ -31,13 +29,6 @@ const responseFormat = {
     },
 };
 async function createTaskUsingFailureDiagnosis({ trace, diagnosis, logger, }) {
-    trace =
-        trace ||
-            llm_1.langfuseInstance?.trace({
-                name: "infer-agent-task",
-                id: crypto.randomUUID(),
-                release: session.version,
-            });
     const failureDiagnosisSpan = trace?.span({
         name: "auto-fix",
         input: {

package/dist/agent/master/browser-tests/index.spec.js

@@ -72,7 +72,7 @@ fixtures_1.test.skip("master agent can click icons accurately", async ({ page, s
     // expect(icons.length).toBe(4); // 1 for each unique icon
     fs_1.default.unlinkSync(iconsRegistryFile);
 });
-(0, fixtures_1.test)("cua agent can click icons accurately", async ({ page, server }) => {
+fixtures_1.test.skip("cua agent can click icons accurately", async ({ page, server }) => {
     await page.goto(`${server.baseURL}/icons-navbar.html`);
     await (0, fixtures_1.expect)(page.getByText("select an icon")).toBeVisible();
     const response = await (0, run_1.createTestUsingComputerUseAgent)({

package/dist/agent/master/execute-browser-action.d.ts

@@ -11,7 +11,7 @@ export declare function executeBrowserAction({ page, nextAction, flags, actions,
         useActionSpecificAnnotations: boolean;
     };
     actions: PlaywrightActions;
-    trace: TraceClient | undefined;
+    trace?: TraceClient | undefined;
     llm: LLM;
 }): Promise<{
     generatedCodeSteps: string[];

package/dist/agent/master/execute-browser-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAUlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,GAAG,EAAE,GAAG,CAAC;CACV;;;gBAIW,MAAM;gBACN,MAAM;4BACM,MAAM;;GA8H7B"}
+{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAUlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IAChC,GAAG,EAAE,GAAG,CAAC;CACV;;;gBAIW,MAAM;gBACN,MAAM;4BACM,MAAM;;GA8H7B"}

package/dist/agent/master/execute-skill-action.d.ts

@@ -6,6 +6,6 @@ export declare function executeSkillAction({ actions, nextAction, trace, }: {
         actionType: string;
         toolCallArgs: string;
     };
-    trace: TraceClient | undefined;
+    trace?: TraceClient | undefined;
 }): Promise<string | undefined>;
 //# sourceMappingURL=execute-skill-action.d.ts.map

package/dist/agent/master/execute-skill-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-skill-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-skill-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAMlD,wBAAsB,kBAAkB,CAAC,EACvC,OAAO,EACP,UAAU,EACV,KAAK,GACN,EAAE;IACD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;CAChC,+BAeA"}
+{"version":3,"file":"execute-skill-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-skill-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAMlD,wBAAsB,kBAAkB,CAAC,EACvC,OAAO,EACP,UAAU,EACV,KAAK,GACN,EAAE;IACD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACjC,+BAeA"}

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAAE,+BAA+B,EAAE,MAAM,QAAQ,CAAC;AAKzD,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAqB9C,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA4PA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAalC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAAE,+BAA+B,EAAE,MAAM,QAAQ,CAAC;AAKzD,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAqB9C,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAiLA"}

package/dist/agent/master/run.js

@@ -10,7 +10,6 @@ const logger_1 = require("../../bin/logger");
 const constants_1 = require("../../constants");
 const errors_1 = require("../../errors");
 const page_1 = require("../../page");
-const session_1 = require("../../session");
 const utils_2 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
 const run_1 = require("../planner/run");
@@ -40,24 +39,8 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
     const useActionSpecificAnnotations = options?.useActionSpecificAnnotations || false;
     const usePlannerInMaster = options?.usePlannerInMaster || false;
     const logger = new logger_1.CustomLogger({ useReporter: false });
-    const session = (0, session_1.getSessionDetails)();
     const testGenPage = new page_1.TestGenPage(page, (0, utils_1.getPageVarName)());
-    const trace = llm_1.langfuseInstance?.trace({
-        name: "test-generator",
-        id: crypto.randomUUID(),
-        version: session.version,
-        metadata: {
-            generationId: session.generationId,
-            sessionId: session.sessionId,
-            testUrl: session.testUrl,
-        },
-        tags: [
-            options.metadata?.projectName,
-            options.metadata?.environment,
-        ].filter((s) => !!s),
-    });
     const llm = new llm_1.LLM({
-        trace,
         provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
         defaultModel: options.model || constants_1.DEFAULT_MODEL,
         providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
@@ -68,7 +51,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
     if (testCase && exports.IS_ALLOWED_TO_USE_SKILLS) {
         skills = await (0, skills_retriever_1.getAppropriateSkills)({
             testCase,
-            trace,
             //@ts-ignore
             options,
         });
@@ -79,7 +61,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
         const plan = await (0, run_1.planTask)({
             task,
             specPath,
-            trace,
         });
         logger.log(`Here is the plan:\n${plan}`);
         // Will assume this is the task hereon
@@ -87,26 +68,13 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
     }
     const actions = new actions_1.PlaywrightActions(testGenPage, scopeVars);
     await (0, utils_2.injectPwLocatorGenerator)(page);
-    trace?.update({ input: { task } });
     let isGivenTaskDone = false;
     const masterAgentActions = [];
     let failedActions = [];
     let disableSkills = false;
     // Run the loop until task is done or we have reached max retry limit
     while (!isGivenTaskDone) {
-        if (await (0, session_1.shouldStopSession)()) {
-            break;
-        }
-        const masterAgentSpan = trace?.span({
-            name: "master-agent",
-            input: {
-                task,
-                executedActions: masterAgentActions,
-                failedActions,
-            },
-        });
         const plannerResp = await (0, run_time_planner_1.runtimePlanner)({
-            trace: masterAgentSpan,
             task,
             successfulActions: [...masterAgentActions],
             pages: getPageVariables(actions.getStateVariables()),
@@ -128,9 +96,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
         const pageScreenshot = buffer.toString("base64");
         let output;
         let generatedCodeSteps = [];
-        if (await (0, session_1.shouldStopSession)()) {
-            break;
-        }
         // Provides next action that needs to be taken
         const nextAction = await (0, next_action_1.getNextAction)({
             page,
@@ -138,25 +103,18 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
             task,
             executedActions: masterAgentActions,
             failedActions,
-            trace: masterAgentSpan,
             llm,
             options,
             actions,
             disableSkills,
             logger,
         });
-        if (await (0, session_1.shouldStopSession)()) {
-            break;
-        }
         if (nextAction) {
             if (nextAction.actionType === action_tool_calls_1.ActionType.UNKNOWN) {
                 logger.error("Agent is not able to figure out next action since element is not visible on screen.");
                 break;
             }
             const args = JSON.parse(nextAction.toolCallArgs);
-            const masterAgentActionSpan = masterAgentSpan?.span({
-                name: "master-agent-execute-action",
-            });
             output = {
                 action: args.action || args.skill,
                 reason: args.reason,
@@ -168,7 +126,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                         const code = await (0, execute_skill_action_1.executeSkillAction)({
                             actions,
                             nextAction,
-                            trace: masterAgentActionSpan,
                         });
                         if (code) {
                             generatedCodeSteps.push(code);
@@ -189,7 +146,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                             },
                             actions,
                             llm,
-                            trace: masterAgentActionSpan,
                         });
                         const { generatedCodeSteps: codeFromExecuteAction, output: outputFromExecuteAction, } = result;
                         generatedCodeSteps.push(...codeFromExecuteAction);
@@ -199,16 +155,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                 // resetting error count as there is a successful action
                 failedActions = [];
                 masterAgentActions.push(output.action);
-                masterAgentActionSpan?.end({
-                    input: {
-                        action: output.action,
-                        reason: output.reason,
-                        type: nextAction.actionType,
-                    },
-                    output: {
-                        success: true,
-                    },
-                });
                 // enable skills after success
                 disableSkills = false;
             }
@@ -217,17 +163,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                     logger.log("Disabling skill usage for next retry");
                     disableSkills = true;
                 }
-                masterAgentActionSpan?.end({
-                    input: {
-                        action: output.action,
-                        reason: output.reason,
-                        type: nextAction.actionType,
-                    },
-                    output: {
-                        error: true,
-                        errorSummary: e.message,
-                    },
-                });
                 if (!(e instanceof errors_1.HumanApprovalDenied)) {
                     logger.error("Failed to run master agent's next task", JSON.stringify(nextAction, null, 2), e);
                     failedActions.push(output.action);
@@ -239,17 +174,8 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                 }
             }
         }
-        trace?.update({ input: { task }, output: { output } });
-        masterAgentSpan?.end({
-            output: {
-                action: output?.action,
-                reason: output?.reason,
-                code: generatedCodeSteps,
-            },
-        });
     }
     const { code, importPaths } = actions.generateCode();
-    trace?.update({ input: { task }, output: { code } });
     logger.success("Successfully generated code for the given task");
     return {
         code,

package/dist/artifacts/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,QAAQ,EACR,aAAa,EAGd,MAAM,4BAA4B,CAAC;AAkBpC,wBAAgB,2BAA2B,uBAM1C;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,EAAE,EACvB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,QAAQ,EAAE,CAAC,CA8DrB;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,aAAa,CAA8B;gBAEvC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAKvC,OAAO,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAelD,iBAAiB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;CAMtD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,QAAQ,EACR,aAAa,EAId,MAAM,4BAA4B,CAAC;AAwBpC,wBAAgB,2BAA2B,uBAM1C;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,EAAE,EACvB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAsErB;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,aAAa,CAA8B;gBAEvC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAKvC,OAAO,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAelD,iBAAiB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;CAMtD"}

package/dist/artifacts/index.js

@@ -16,6 +16,9 @@ function isArtifactInputData(artifact) {
 function isArtifactInputPath(artifact) {
     return "path" in artifact;
 }
+function isArtifactInputUrl(artifact) {
+    return "url" in artifact;
+}
 function isArtifactCollectionEnabled() {
     return (process.env.R2_ACCOUNT_ID &&
         process.env.R2_ACCESS_KEY_ID &&
@@ -159,6 +162,7 @@ async function collectArtifacts(inputs, repoDir, toolCallId) {
         return [];
     const dataArtifacts = inputs.filter(isArtifactInputData);
     const pathArtifacts = inputs.filter(isArtifactInputPath);
+    const urlArtifacts = inputs.filter(isArtifactInputUrl);
     for (const artifact of pathArtifacts) {
         if (!isFullPath(artifact.path, repoDir)) {
             throw new Error(`Invalid path: "${artifact.path}". Path should be a full path relative to the repo directory: "${repoDir}"`);
@@ -179,14 +183,22 @@ async function collectArtifacts(inputs, repoDir, toolCallId) {
         const inputsWithFileNames = [
             ...pathArtifacts,
             ...dataArtifactsWithFileName,
+            ...urlArtifacts,
         ];
         const results = inputsWithFileNames.map((artifact) => {
-            const isPathArtifact = isArtifactInputPath(artifact);
-            const relativePath = isPathArtifact
-                ? path_1.default.relative(repoDir, artifact.path)
-                : artifact.fileName;
-            const pathInBucket = path_1.default.join(destinationDir, relativePath);
-            const url = `https://reports.empirical.run/${pathInBucket}`;
+            let url;
+            if (isArtifactInputUrl(artifact)) {
+                // url artifacts are passthrough
+                url = artifact.url;
+            }
+            else {
+                const isPathArtifact = isArtifactInputPath(artifact);
+                const relativePath = isPathArtifact
+                    ? path_1.default.relative(repoDir, artifact.path)
+                    : artifact.fileName;
+                const pathInBucket = path_1.default.join(destinationDir, relativePath);
+                url = `https://reports.empirical.run/${pathInBucket}`;
+            }
             return {
                 name: artifact.name,
                 contentType: artifact.contentType,

package/dist/artifacts/utils.d.ts

@@ -1,11 +1,11 @@
-import { ArtifactInput, ArtifactInputPath } from "@empiricalrun/shared-types";
+import { ArtifactInput, ArtifactInputPath, ArtifactInputUrl } from "@empiricalrun/shared-types";
 import { JSONReport as PlaywrightJSONReport } from "@playwright/test/reporter";
 /**
  * Extracts attachment information from a Playwright JSON report.
  * @param report The Playwright JSON report to extract attachments from
  * @returns An array of objects containing path and contentType for each attachment
  */
-export declare function extractAttachmentsFromPlaywrightJSONReport(report: PlaywrightJSONReport, testNameFilter?: string): ArtifactInputPath[];
+export declare function extractAttachmentsFromPlaywrightJSONReport(report: PlaywrightJSONReport, testNameFilter?: string): Array<ArtifactInputUrl | ArtifactInputPath>;
 /**
  * Scans the given repository directory for Playwright artifacts (attachments) by:
  * - Constructing the path to the Playwright report directory using the provided repoDir.

package/dist/artifacts/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/artifacts/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EACL,UAAU,IAAI,oBAAoB,EAGnC,MAAM,2BAA2B,CAAC;AAInC;;;;GAIG;AACH,wBAAgB,0CAA0C,CACxD,MAAM,EAAE,oBAAoB,EAC5B,cAAc,CAAC,EAAE,MAAM,GACtB,iBAAiB,EAAE,CA2DrB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,CAmCxE"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/artifacts/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,UAAU,IAAI,oBAAoB,EAGnC,MAAM,2BAA2B,CAAC;AAInC;;;;GAIG;AACH,wBAAgB,0CAA0C,CACxD,MAAM,EAAE,oBAAoB,EAC5B,cAAc,CAAC,EAAE,MAAM,GACtB,KAAK,CAAC,gBAAgB,GAAG,iBAAiB,CAAC,CAsE7C;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,CAmCxE"}

package/dist/artifacts/utils.js

@@ -40,11 +40,22 @@ function extractAttachmentsFromPlaywrightJSONReport(report, testNameFilter) {
                 for (const attachment of result.attachments) {
                     // Only collect attachments that have a 'path' (i.e., not embedded content)
                     if (attachment.path && attachment.contentType) {
-                        attachments.push({
-                            name: `${testTitle} - ${attachment.name}`,
-                            path: attachment.path,
-                            contentType: attachment.contentType,
-                        });
+                        const isUrl = attachment.path.startsWith("http://") ||
+                            attachment.path.startsWith("https://");
+                        if (isUrl) {
+                            attachments.push({
+                                name: `${testTitle} - ${attachment.name}`,
+                                url: attachment.path,
+                                contentType: attachment.contentType,
+                            });
+                        }
+                        else {
+                            attachments.push({
+                                name: `${testTitle} - ${attachment.name}`,
+                                path: attachment.path,
+                                contentType: attachment.contentType,
+                            });
+                        }
                     }
                 }
             }

package/dist/auth/cli-auth.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli-auth.d.ts","sourceRoot":"","sources":["../../src/auth/cli-auth.ts"],"names":[],"mappings":"AAmBA,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE;QACL,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,CAwKxD;AAED,wBAAsB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAE5C;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC;IAC7C,aAAa,EAAE,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CACtC,CAAC,CAeD"}
+{"version":3,"file":"cli-auth.d.ts","sourceRoot":"","sources":["../../src/auth/cli-auth.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE;QACL,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,CAwKxD;AAED,wBAAsB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAE5C;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC;IAC7C,aAAa,EAAE,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CACtC,CAAC,CAeD"}

package/dist/auth/cli-auth.js

@@ -11,14 +11,10 @@ const http_1 = require("http");
 const url_1 = require("url");
 const token_store_1 = require("./token-store");
 const CLIENT_PORT_DEFAULT = 8080;
-// Get dashboard URL
-const getDashboardUrl = () => {
-    return process.env.DASHBOARD_DOMAIN || "https://dash.empirical.run";
-};
 async function authenticate() {
     // Check if already authenticated
     if (await (0, token_store_1.isAuthenticated)()) {
-        const tokens = await (0, token_store_1.getStoredTokens)();
+        const tokens = await (0, token_store_1.getStoredUserTokens)();
         if (tokens) {
             return {
                 success: true,
@@ -29,7 +25,7 @@ async function authenticate() {
             };
         }
     }
-    const appUrl = getDashboardUrl();
+    const appUrl = process.env.DASHBOARD_DOMAIN || "https://dash.empirical.run";
     const clientPort = await (0, detect_port_1.default)(CLIENT_PORT_DEFAULT);
     const redirectUri = `http://localhost:${clientPort}/callback`;
     return new Promise((resolve) => {
@@ -175,7 +171,7 @@ async function logout() {
 }
 async function getAuthStatus() {
     if (await (0, token_store_1.isAuthenticated)()) {
-        const tokens = await (0, token_store_1.getStoredTokens)();
+        const tokens = await (0, token_store_1.getStoredUserTokens)();
         if (tokens) {
             return {
                 authenticated: true,

package/dist/auth/index.d.ts

@@ -1,4 +1,3 @@
-export { apiClient } from "./api-client";
 export { authenticate, getAuthStatus, logout } from "./cli-auth";
-export { clearTokens, getStoredTokens, isAuthenticated, type StoredTokens, storeTokens, } from "./token-store";
+export { clearTokens, getStoredUserTokens as getStoredTokens, isAuthenticated, type StoredTokens, storeTokens, } from "./token-store";
 //# sourceMappingURL=index.d.ts.map

package/dist/auth/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/auth/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EACL,WAAW,EACX,eAAe,EACf,eAAe,EACf,KAAK,YAAY,EACjB,WAAW,GACZ,MAAM,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/auth/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EACL,WAAW,EACX,mBAAmB,IAAI,eAAe,EACtC,eAAe,EACf,KAAK,YAAY,EACjB,WAAW,GACZ,MAAM,eAAe,CAAC"}

package/dist/auth/index.js

@@ -1,14 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.storeTokens = exports.isAuthenticated = exports.getStoredTokens = exports.clearTokens = exports.logout = exports.getAuthStatus = exports.authenticate = exports.apiClient = void 0;
-var api_client_1 = require("./api-client");
-Object.defineProperty(exports, "apiClient", { enumerable: true, get: function () { return api_client_1.apiClient; } });
+exports.storeTokens = exports.isAuthenticated = exports.getStoredTokens = exports.clearTokens = exports.logout = exports.getAuthStatus = exports.authenticate = void 0;
 var cli_auth_1 = require("./cli-auth");
 Object.defineProperty(exports, "authenticate", { enumerable: true, get: function () { return cli_auth_1.authenticate; } });
 Object.defineProperty(exports, "getAuthStatus", { enumerable: true, get: function () { return cli_auth_1.getAuthStatus; } });
 Object.defineProperty(exports, "logout", { enumerable: true, get: function () { return cli_auth_1.logout; } });
 var token_store_1 = require("./token-store");
 Object.defineProperty(exports, "clearTokens", { enumerable: true, get: function () { return token_store_1.clearTokens; } });
-Object.defineProperty(exports, "getStoredTokens", { enumerable: true, get: function () { return token_store_1.getStoredTokens; } });
+Object.defineProperty(exports, "getStoredTokens", { enumerable: true, get: function () { return token_store_1.getStoredUserTokens; } });
 Object.defineProperty(exports, "isAuthenticated", { enumerable: true, get: function () { return token_store_1.isAuthenticated; } });
 Object.defineProperty(exports, "storeTokens", { enumerable: true, get: function () { return token_store_1.storeTokens; } });

package/dist/auth/token-store.d.ts

@@ -9,7 +9,7 @@ export interface TokenStorage {
     [dashboardDomain: string]: StoredTokens;
 }
 export declare function storeTokens(tokens: StoredTokens): Promise<void>;
-export declare function getStoredTokens(): Promise<StoredTokens | null>;
+export declare function getStoredUserTokens(): Promise<StoredTokens | null>;
 export declare function clearTokens(): Promise<void>;
 export declare function isAuthenticated(): Promise<boolean>;
 //# sourceMappingURL=token-store.d.ts.map

package/dist/auth/token-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"token-store.d.ts","sourceRoot":"","sources":["../../src/auth/token-store.ts"],"names":[],"mappings":"AAqBA,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,CAAC,eAAe,EAAE,MAAM,GAAG,YAAY,CAAC;CACzC;AAgCD,wBAAsB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA2CrE;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAqBpE;AAED,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CA4CjD;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC,CAUxD"}
+{"version":3,"file":"token-store.d.ts","sourceRoot":"","sources":["../../src/auth/token-store.ts"],"names":[],"mappings":"AAqBA,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,CAAC,eAAe,EAAE,MAAM,GAAG,YAAY,CAAC;CACzC;AAgCD,wBAAsB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA2CrE;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAqBxE;AAED,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CA4CjD;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC,CAUxD"}

package/dist/auth/token-store.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.storeTokens = storeTokens;
-exports.getStoredTokens = getStoredTokens;
+exports.getStoredUserTokens = getStoredUserTokens;
 exports.clearTokens = clearTokens;
 exports.isAuthenticated = isAuthenticated;
 const crypto_1 = require("crypto");
@@ -81,7 +81,7 @@ async function storeTokens(tokens) {
         throw new Error(`Failed to store authentication tokens: ${error}`);
     }
 }
-async function getStoredTokens() {
+async function getStoredUserTokens() {
     try {
         const dashboardDomain = getDashboardDomain();
         const tokenStorage = await getTokenStorage();
@@ -138,7 +138,7 @@ async function clearTokens() {
     }
 }
 async function isAuthenticated() {
-    const tokens = await getStoredTokens();
+    const tokens = await getStoredUserTokens();
     if (!tokens)
         return false;
     // Check if token is not expired (with 5 minute buffer)

package/dist/bin/environments.d.ts

@@ -1,2 +1,6 @@
-export declare function listEnvironments(): Promise<void>;
+import type { IDashboardAPIClient } from "@empiricalrun/shared-types";
+export declare function listEnvironments({ apiClient, repoPath: repoDir, }: {
+    repoPath: string;
+    apiClient: IDashboardAPIClient;
+}): Promise<void>;
 //# sourceMappingURL=environments.d.ts.map

package/dist/bin/environments.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"environments.d.ts","sourceRoot":"","sources":["../../src/bin/environments.ts"],"names":[],"mappings":"AASA,wBAAsB,gBAAgB,kBAiErC"}
+{"version":3,"file":"environments.d.ts","sourceRoot":"","sources":["../../src/bin/environments.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EAEpB,MAAM,4BAA4B,CAAC;AAiEpC,wBAAsB,gBAAgB,CAAC,EACrC,SAAS,EACT,QAAQ,EAAE,OAAO,GAClB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,mBAAmB,CAAC;CAChC,iBAiDA"}