@empiricalrun/test-gen 0.46.1 → 0.46.3

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @empiricalrun/test-gen
 
+## 0.46.3
+
+### Patch Changes
+
+- f26142f: fix: browsing agent giving wrong executed action
+- 112b429: test: enable parallel test execution in playwright
+
+## 0.46.2
+
+### Patch Changes
+
+- 9770fd6: test: fill action with multiple pages
+- c723f61: fix: auto fix didnt work
+- e6d6174: chore: remove sentry from worker, service and test-gen
+
 ## 0.46.1
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts

@@ -7,6 +7,10 @@ export type BrowsingAgentOptions = Partial<TestGenConfigOptions> & {
         disallowedStrings?: string[];
     };
 };
+export type BrowserAgentAction = {
+    action: string;
+    code: string | undefined;
+};
 export declare function executeTaskUsingBrowsingAgent({ action, page, actions, llm, options, trace, }: {
     action: string;
     page: Page;
@@ -14,5 +18,5 @@ export declare function executeTaskUsingBrowsingAgent({ action, page, actions, l
     llm: LLM;
     trace?: TraceClient;
     options: BrowsingAgentOptions;
-}): Promise<string[] | undefined>;
+}): Promise<BrowserAgentAction | undefined>;
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAQlD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,EACP,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,CAiEhC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAQlD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,EACP,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAoE1C"}

package/dist/agent/browsing/index.js

@@ -9,7 +9,7 @@ const html_1 = require("../../utils/html");
 const utils_1 = require("../utils");
 const o1_completion_1 = require("./o1-completion");
 async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, options, trace, }) {
-    let generatedCodeSteps = [];
+    let generatedCodeSteps;
     const tools = actions.getBrowsingActionSchemas();
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     const browsingAgentSpan = trace?.span({
@@ -55,11 +55,14 @@ async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, optio
     const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
     for (const i in toolCalls) {
         const toolCall = toolCalls[i];
+        const args = (0, utils_1.parseJson)(toolCall.function.arguments);
         try {
             const code = await actions.executeAction(toolCall.function.name, (0, utils_1.parseJson)(toolCall.function.arguments), toolCallsSpan);
-            if (code) {
-                generatedCodeSteps.push(code);
-            }
+            generatedCodeSteps = {
+                // Passing reason as action, in order to pass the correct action that took place to runtime planner
+                action: args.reason,
+                code,
+            };
         }
         catch (e) {
             void testgenUpdatesReporter.sendMessage(e.message);

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAmBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB,iBA8EnB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAkBA,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,OAAO,GACR,EAAE,iBAAiB,iBA6EnB"}

package/dist/agent/browsing/run.js

@@ -1,33 +1,9 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateTestsUsingMasterAgent = void 0;
-const Sentry = __importStar(require("@sentry/node"));
 const detect_port_1 = __importDefault(require("detect-port"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const web_1 = require("../../bin/utils/platform/web");
@@ -80,7 +56,6 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
                 PAGE_VAR_NAME: pageVar || "page",
                 DISPLAY: ":99",
             },
-            span: Sentry.getActiveSpan(),
         });
     }
     catch (e) {

package/dist/agent/master/browser-tests/index.spec.js

@@ -10,27 +10,31 @@ const path_1 = __importDefault(require("path"));
 const utils_1 = require("../../browsing/utils");
 const element_annotation_1 = require("../element-annotation");
 const run_1 = require("../run");
-let server;
-let PORT = 2345;
-test_1.test.beforeAll(async () => {
-    server = http_server_1.default.createServer({
-        root: path_1.default.join(process.cwd(), "test-data"),
-    });
-    // Start the server
-    await new Promise((resolve) => {
-        server.listen(PORT, () => {
-            console.log(`Server running at http://localhost:${PORT}`);
-            resolve(true);
-        });
-    });
-});
-test_1.test.afterAll(() => {
-    if (server) {
-        server.close();
-    }
+const test = test_1.test.extend({
+    server: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use, workerInfo) => {
+            const port = 2345 + workerInfo.workerIndex;
+            const server = http_server_1.default.createServer({
+                root: path_1.default.join(process.cwd(), "test-data"),
+            });
+            await new Promise((resolve) => {
+                server.listen(port, () => {
+                    console.log(`Server running at http://localhost:${port}`);
+                    resolve();
+                });
+            });
+            await use({
+                port,
+                baseURL: `http://localhost:${port}`,
+            });
+            server.close();
+        },
+        { scope: "worker" },
+    ],
 });
-(0, test_1.test)("able to scroll and interact with elements", async ({ page }) => {
-    await page.goto(`http://localhost:${PORT}/blog-page.html`);
+test("able to scroll and interact with elements", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
     const response = await (0, run_1.createTestUsingMasterAgent)({
         task: `fill test@test.com into the email field and click the submit`,
         page,
@@ -41,8 +45,8 @@ test_1.test.afterAll(() => {
     (0, test_1.expect)(response.code).toContain("await page.getByPlaceholder('Enter your email').fill(\"test@test.com\")");
     (0, test_1.expect)(response.code).toContain("await page.getByRole('button', { name: 'Subscribe' }).click()");
 });
-(0, test_1.test)("scroll when element does not exist", async ({ page }) => {
-    await page.goto(`http://localhost:${PORT}/blog-page.html`);
+test("scroll when element does not exist", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/blog-page.html`);
     const response = await (0, run_1.createTestUsingMasterAgent)({
         task: `click search button`,
         page,
@@ -52,8 +56,8 @@ test_1.test.afterAll(() => {
     (0, test_1.expect)(response.importPaths.length).toBe(0);
     (0, test_1.expect)(response.code.length).toBe(0);
 });
-(0, test_1.test)("agent can click icons accurately", async ({ page }) => {
-    await page.goto(`http://localhost:${PORT}/icons-navbar.html`);
+test("agent can click icons accurately", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/icons-navbar.html`);
     await (0, test_1.expect)(page.getByText("select an icon")).toBeVisible();
     const response = await (0, run_1.createTestUsingMasterAgent)({
         task: `click on the gear icon`,
@@ -72,9 +76,9 @@ test_1.test.afterAll(() => {
     (0, test_1.expect)(icons.length).toBe(4); // 1 for each unique icon
     fs_1.default.unlinkSync(iconsRegistryFile);
 });
-(0, test_1.test)("annotate and enrich annotations correctly", async ({ page }) => {
+test("annotate and enrich annotations correctly", async ({ page, server }) => {
     await (0, utils_1.injectPwLocatorGenerator)(page);
-    await page.goto(`http://localhost:${PORT}/iframe-elements.html`);
+    await page.goto(`${server.baseURL}/iframe-elements.html`);
     const { annotationKeys: keys } = await (0, element_annotation_1.getAnnotationKeys)({
         page,
         preference: {
@@ -92,3 +96,24 @@ test_1.test.afterAll(() => {
     // 2 clickable divs: 1 in main frame, 1 in iframe
     (0, test_1.expect)(keys.filter((k) => k.text.includes("Lorem Ipsum")).length).toBe(2);
 });
+test("fill action with multiple pages", async ({ context }) => {
+    const page1 = await context.newPage();
+    const page2 = await context.newPage();
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `goto empirical.run on page1 and goto google.com in page2. Enter text empirical on page2 and click on search.`,
+        page: page2,
+        options: {},
+        scopeVars: {
+            context,
+            page1,
+            page2,
+        },
+    });
+    (0, test_1.expect)(response.code).toBeTruthy();
+    console.log(response.code);
+    const lines = response.code.split("\n");
+    (0, test_1.expect)(lines.find((l) => l.match(/^await page1\.goto(.+)empirical\.run/))).toBeTruthy();
+    (0, test_1.expect)(lines.find((l) => l.match(/^await page2\.goto(.+)google\.com/))).toBeTruthy();
+    (0, test_1.expect)(lines.find((l) => l.match(/^await page2(.+)fill(.+)empirical/))).toBeTruthy();
+    (0, test_1.expect)(lines.find((l) => l.match(/^await page2(.+)click/))).toBeTruthy();
+});

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAqBlC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA8XA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAqBlC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAmYA"}

package/dist/agent/master/run.js

@@ -282,7 +282,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                         }
                     }
                     else {
-                        generatedCodeSteps = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
+                        const browserAction = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
                             trace: masterAgentActionSpan,
                             action: output.action,
                             page,
@@ -290,6 +290,12 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                             llm,
                             actions,
                         });
+                        if (browserAction) {
+                            output.action = browserAction.action;
+                            if (browserAction.code) {
+                                generatedCodeSteps.push(browserAction.code);
+                            }
+                        }
                     }
                 }
                 // resetting error count as there is a successful action

package/dist/bin/index.d.ts

@@ -1,3 +1,3 @@
 #!/usr/bin/env node
-import "../initSentry";
+export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/bin/index.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/bin/index.ts"],"names":[],"mappings":""}

package/dist/bin/index.js

@@ -1,35 +1,10 @@
 #!/usr/bin/env node
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-require("../initSentry");
 const llm_1 = require("@empiricalrun/llm");
-const Sentry = __importStar(require("@sentry/node"));
 const commander_1 = require("commander");
 const dotenv_1 = __importDefault(require("dotenv"));
 const run_1 = require("../agent/browsing/run");
@@ -51,7 +26,6 @@ dotenv_1.default.config({
 });
 const flushEvents = async () => {
     await (0, llm_1.flushAllTraces)();
-    await Sentry.flush();
 };
 process.on("beforeExit", async () => await flushEvents());
 process.on("exit", async () => await flushEvents());
@@ -64,7 +38,7 @@ async function resolveAgentUsingTask({ testCase, trace, }) {
     });
     return response;
 }
-async function runAgent(testGenConfig, testGenToken, span) {
+async function runAgent(testGenConfig, testGenToken) {
     const logger = new logger_1.CustomLogger();
     const { specPath, testCase } = testGenConfig;
     if (process.env.LOG_URL) {
@@ -72,7 +46,6 @@ async function runAgent(testGenConfig, testGenToken, span) {
             void new reporter_1.TestGenUpdatesReporter().sendLogUrl(process.env.LOG_URL);
         }
         catch (e) {
-            span?.recordException(e);
             console.warn("Failed to send log url to test gen update", e);
         }
     }
@@ -120,7 +93,7 @@ async function runAgent(testGenConfig, testGenToken, span) {
     if (testGenConfig.testErrorDiagnosis &&
         testGenConfig.testErrorDiagnosis.failingLine &&
         // TODO: fix this hardcoding of user prompt - ideally its an auto fix intent
-        testCase.steps[0] == "Can you please fix the test") {
+        testCase.steps[0]?.toLowerCase().trim() == "can you please fix the test") {
         const { task: updatedTask } = await (0, diagnosis_agent_1.createTaskUsingFailureDiagnosis)({
             options: testGenConfig.options,
             trace,
@@ -178,69 +151,62 @@ async function runAgent(testGenConfig, testGenToken, span) {
     return agent;
 }
 (async function main() {
-    await Sentry.continueTrace({ sentryTrace: utils_2.sentryTrace, baggage: utils_2.baggage }, async () => {
-        await Sentry.startSpan({ name: "test-gen" }, async (span) => {
-            const program = new commander_1.Command();
-            program
-                .option("--token <token>", "Test generation token")
-                .option("--name <test-name>", "Name of the test case")
-                .option("--prompt <prompt>", "Prompt for the test case")
-                .option("--file <test-file>", "File path of the test case (inside tests dir)")
-                .option("--suites <suites>", "Comma separated list of describe blocks")
-                .parse(process.argv);
-            const options = program.opts();
-            (0, utils_2.validateCliOptions)(options);
-            const testGenConfig = options.token
-                ? (0, scenarios_1.loadTestConfigs)(options.token)
-                : (0, scenarios_1.buildTestConfigFromOptions)(options);
-            const testGenToken = options.token
-                ? options.token
-                : (0, scenarios_1.buildTokenFromOptions)(options);
-            (0, reporter_1.setReporterConfig)({
-                projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-                testSessionId: testGenConfig.options?.metadata.testSessionId,
-                generationId: testGenConfig.options?.metadata.generationId,
-            });
-            (0, session_1.setSessionDetails)({
-                testCaseId: testGenConfig.testCase.id,
-                sessionId: testGenConfig.options?.metadata.testSessionId,
-                generationId: testGenConfig.options?.metadata.generationId,
-                projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-            });
-            let testGenFailed = false;
-            let agentUsed;
-            try {
-                // download the build if it exists
-                await (0, test_build_1.downloadBuild)(testGenConfig.build || {});
-                agentUsed = await runAgent(testGenConfig, testGenToken);
-            }
-            catch (e) {
-                span.recordException(e);
-                Sentry.captureException(e);
-                testGenFailed = true;
-                new logger_1.CustomLogger().error(`Failed to generate test for the scenario. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`, e?.message, e?.stack);
-            }
-            if (agentUsed &&
-                agentUsed !== "code" &&
-                agentUsed !== "plan" &&
-                testGenConfig.testCase.name &&
-                testGenConfig.options) {
-                await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
-                    projectRepoName: testGenConfig.options.metadata.projectRepoName,
-                    testName: testGenConfig.testCase.name,
-                });
-            }
-            // TODO: move these reporters to a better lifecycle
-            await (0, llm_1.flushAllTraces)();
-            await (0, logger_1.waitForLogsToFlush)();
-            await (0, session_1.endSession)();
-            span.end();
-            if (testGenFailed) {
-                process.exit(1);
-            }
-            else {
-                process.exit(0);
-            }
-        });
+    const program = new commander_1.Command();
+    program
+        .option("--token <token>", "Test generation token")
+        .option("--name <test-name>", "Name of the test case")
+        .option("--prompt <prompt>", "Prompt for the test case")
+        .option("--file <test-file>", "File path of the test case (inside tests dir)")
+        .option("--suites <suites>", "Comma separated list of describe blocks")
+        .parse(process.argv);
+    const options = program.opts();
+    (0, utils_2.validateCliOptions)(options);
+    const testGenConfig = options.token
+        ? (0, scenarios_1.loadTestConfigs)(options.token)
+        : (0, scenarios_1.buildTestConfigFromOptions)(options);
+    const testGenToken = options.token
+        ? options.token
+        : (0, scenarios_1.buildTokenFromOptions)(options);
+    (0, reporter_1.setReporterConfig)({
+        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+        testSessionId: testGenConfig.options?.metadata.testSessionId,
+        generationId: testGenConfig.options?.metadata.generationId,
     });
+    (0, session_1.setSessionDetails)({
+        testCaseId: testGenConfig.testCase.id,
+        sessionId: testGenConfig.options?.metadata.testSessionId,
+        generationId: testGenConfig.options?.metadata.generationId,
+        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+    });
+    let testGenFailed = false;
+    let agentUsed;
+    try {
+        // download the build if it exists
+        await (0, test_build_1.downloadBuild)(testGenConfig.build || {});
+        agentUsed = await runAgent(testGenConfig, testGenToken);
+    }
+    catch (e) {
+        testGenFailed = true;
+        new logger_1.CustomLogger().error(`Failed to generate test for the scenario. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`, e?.message, e?.stack);
+    }
+    if (agentUsed &&
+        agentUsed !== "code" &&
+        agentUsed !== "plan" &&
+        testGenConfig.testCase.name &&
+        testGenConfig.options) {
+        await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
+            projectRepoName: testGenConfig.options.metadata.projectRepoName,
+            testName: testGenConfig.testCase.name,
+        });
+    }
+    // TODO: move these reporters to a better lifecycle
+    await (0, llm_1.flushAllTraces)();
+    await (0, logger_1.waitForLogsToFlush)();
+    await (0, session_1.endSession)();
+    if (testGenFailed) {
+        process.exit(1);
+    }
+    else {
+        process.exit(0);
+    }
 })();

package/dist/bin/utils/index.d.ts

@@ -1,5 +1,3 @@
-export declare const sentryTrace: string | undefined;
-export declare const baggage: string | undefined;
 export interface CliOptions {
     token?: string;
     name?: string;

package/dist/bin/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW,oBAA2B,CAAC;AACpD,eAAO,MAAM,OAAO,oBAA6B,CAAC;AAElD,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAS5D"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAS5D"}

package/dist/bin/utils/index.js

@@ -1,8 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateCliOptions = exports.baggage = exports.sentryTrace = void 0;
-exports.sentryTrace = process.env.SENTRY_TRACE;
-exports.baggage = process.env.SENTRY_BAGGAGE;
+exports.validateCliOptions = void 0;
 function validateCliOptions(options) {
     const hasToken = !!options.token;
     const hasNameAndFile = !!options.name && !!options.file && !!options.prompt;

package/dist/test-build/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test-build/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAWnD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAa/D"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test-build/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAUnD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAY/D"}

package/dist/test-build/index.js

@@ -1,33 +1,9 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.downloadBuild = void 0;
-const Sentry = __importStar(require("@sentry/node"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../bin/logger");
 const exec_1 = require("../utils/exec");
@@ -48,7 +24,6 @@ async function downloadBuild(build) {
         logger.log(`Downloading build from ${build.url}`);
         await (0, exec_1.cmd)(`npm run download ${build.url}`.split(" "), {
             env: { ...Object(process.env) },
-            span: Sentry.getActiveSpan(),
         });
     }
 }

package/dist/utils/exec.d.ts

@@ -1,6 +1,4 @@
-import * as Sentry from "@sentry/node";
 export declare function cmd(command: string[], options: {
     env?: Record<string, string>;
-    span?: Sentry.Span;
 }): Promise<number>;
 //# sourceMappingURL=exec.d.ts.map

package/dist/utils/exec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,cAAc,CAAC;AAMvC,wBAAsB,GAAG,CACvB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAA;CAAE,GAC5D,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
+{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAGA,wBAAsB,GAAG,CACvB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GACxC,OAAO,CAAC,MAAM,CAAC,CAuCjB"}

package/dist/utils/exec.js

@@ -1,77 +1,44 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.cmd = void 0;
-const Sentry = __importStar(require("@sentry/node"));
 const child_process_1 = require("child_process");
 const process_1 = __importDefault(require("process"));
-const utils_1 = require("../bin/utils");
 async function cmd(command, options) {
-    return Sentry.continueTrace({ sentryTrace: utils_1.sentryTrace, baggage: utils_1.baggage }, async () => {
-        return Sentry.startSpan({ name: "test-gen" }, async (span) => {
-            let errorLogs = [];
-            return new Promise((resolveFunc, rejectFunc) => {
-                if (!command[0]) {
-                    rejectFunc(new Error("Command cannot be empty"));
-                    return;
-                }
-                let p = (0, child_process_1.spawn)(command[0], command.slice(1), {
-                    env: { ...process_1.default.env, ...options.env },
-                });
-                p.stdout.on("data", (x) => {
-                    const log = x.toString();
-                    if (log.includes("Error")) {
-                        errorLogs.push(log);
-                    }
-                    process_1.default.stdout.write(log);
-                });
-                p.stderr.on("data", (x) => {
-                    const log = x.toString();
-                    process_1.default.stderr.write(log);
-                    errorLogs.push(log);
-                });
-                p.on("error", async (err) => {
-                    span?.recordException(err);
-                    Sentry.captureException(err);
-                    rejectFunc(err);
-                });
-                p.on("exit", async (code) => {
-                    if (code !== 0) {
-                        const errorMessage = errorLogs.slice(-3).join("\n");
-                        span?.recordException(errorMessage);
-                        Sentry.captureException(new Error(errorMessage));
-                        rejectFunc(new Error(errorMessage));
-                    }
-                    else {
-                        resolveFunc(code ?? 1);
-                    }
-                });
-            });
+    let errorLogs = [];
+    return new Promise((resolveFunc, rejectFunc) => {
+        if (!command[0]) {
+            rejectFunc(new Error("Command cannot be empty"));
+            return;
+        }
+        let p = (0, child_process_1.spawn)(command[0], command.slice(1), {
+            env: { ...process_1.default.env, ...options.env },
+        });
+        p.stdout.on("data", (x) => {
+            const log = x.toString();
+            if (log.includes("Error")) {
+                errorLogs.push(log);
+            }
+            process_1.default.stdout.write(log);
+        });
+        p.stderr.on("data", (x) => {
+            const log = x.toString();
+            process_1.default.stderr.write(log);
+            errorLogs.push(log);
+        });
+        p.on("error", async (err) => {
+            rejectFunc(err);
+        });
+        p.on("exit", async (code) => {
+            if (code !== 0) {
+                const errorMessage = errorLogs.slice(-3).join("\n");
+                rejectFunc(new Error(errorMessage));
+            }
+            else {
+                resolveFunc(code ?? 1);
+            }
         });
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.46.1",
+  "version": "0.46.3",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -47,7 +47,6 @@
   "dependencies": {
     "@actions/core": "^1.10.1",
     "@babel/parser": "^7.26.3",
-    "@sentry/node": "^8.54.0",
     "@types/sanitize-html": "^2.11.0",
     "commander": "^12.1.0",
     "detect-port": "^1.6.1",
@@ -74,9 +73,9 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
+    "@empiricalrun/llm": "^0.9.35",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/reporter": "^0.23.1",
-    "@empiricalrun/llm": "^0.9.35"
+    "@empiricalrun/reporter": "^0.23.1"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",
@@ -92,7 +91,7 @@
     "js-levenshtein": "^1.1.6",
     "playwright": "1.47.1",
     "ts-patch": "^3.3.0",
-    "@empiricalrun/shared-types": "0.0.4"
+    "@empiricalrun/shared-types": "0.0.5"
   },
   "scripts": {
     "dev": "tspc --build --watch",

package/playwright.config.ts

@@ -8,5 +8,7 @@ export default defineConfig({
   testDir: "./dist/.",
   testMatch: "**/*.spec.js",
   // Setting large timeout as master agent takes time to run.
-  timeout: 120_000,
+  timeout: 240_000,
+  fullyParallel: true,
+  workers: "50%",
 });

package/dist/initSentry.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=initSentry.d.ts.map

package/dist/initSentry.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"initSentry.d.ts","sourceRoot":"","sources":["../src/initSentry.ts"],"names":[],"mappings":""}

package/dist/initSentry.js

@@ -1,37 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Sentry = __importStar(require("@sentry/node"));
-Sentry.init({
-    enabled: process.env.NODE_ENV === "production",
-    dsn: "https://87e61b11ede1431d7156bcd26da997cc@o4506822020235264.ingest.us.sentry.io/4508806031015936",
-    tracesSampleRate: 1.0,
-    serverName: "test-gen",
-    debug: process.env.NODE_ENV === "development",
-    integrations: [Sentry.consoleIntegration(), Sentry.httpIntegration()],
-    clientReportFlushInterval: 2000,
-    shutdownTimeout: 5000,
-    enableTracing: true,
-});