@empiricalrun/test-gen 0.42.27 → 0.42.29

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @empiricalrun/test-gen
 
+## 0.42.29
+
+### Patch Changes
+
+- f39500b: fix: removed explicit delays from master agent
+
+## 0.42.28
+
+### Patch Changes
+
+- df9f428: feat: add distributed tracing for ci-service, ci-worker, test-gen and test-run
+
 ## 0.42.27
 
 ### Patch Changes

package/dist/actions/click.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAuDlC,CAAC"}
+{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAsDlC,CAAC"}

package/dist/actions/click.js

@@ -11,7 +11,6 @@ const clickActionGenerator = (page) => {
             const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(selector, args.xpath, page.pwPageInstance, args?.elementAnnotation);
             const exec = new Function("page", `return page.${locator}.click({ timeout: 3000 })`);
             await exec(page.pwPageInstance);
-            await page.pwPageInstance.waitForTimeout(3000);
             return {
                 locator,
             };

package/dist/agent/browsing/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAmBA,KAAK,iBAAiB,GAAG;IACvB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,EAAE,iBAAiB,iBA6EnB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/run.ts"],"names":[],"mappings":"AAoBA,KAAK,iBAAiB,GAAG;IACvB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,EAClD,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,EAAE,iBAAiB,iBA8EnB"}

package/dist/agent/browsing/run.js

@@ -1,9 +1,33 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateTestsUsingMasterAgent = void 0;
+const Sentry = __importStar(require("@sentry/node"));
 const detect_port_1 = __importDefault(require("detect-port"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const utils_1 = require("../../bin/utils");
@@ -57,6 +81,7 @@ async function generateTestsUsingMasterAgent({ testFilePath, filePathToUpdate, p
                 PAGE_VAR_NAME: pageVar || "page",
                 DISPLAY: ":99",
             },
+            span: Sentry.getActiveSpan(),
         });
     }
     catch (e) {

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAoDD"}

package/dist/agent/master/element-annotation.js

@@ -120,7 +120,6 @@ async function getAnnotationKeys({ page, preference, options, }) {
         }));
         return annotations;
     }, { preference, options });
-    await page.waitForTimeout(2000);
     const annotationBuffer = await page.screenshot({
     // path: `screenshots/screenshot-${screenshotIndex++}.png`,
     });

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA0WA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAsWA"}

package/dist/agent/master/run.js

@@ -37,8 +37,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     const session = (0, session_1.getSessionDetails)();
     const testGenPage = new page_1.TestGenPage(page, (0, utils_1.getPageVarName)());
-    // add timeout for the page to settle in
-    await page.waitForTimeout(3000);
     const trace = llm_1.langfuseInstance?.trace({
         name: "test-generator",
         id: crypto.randomUUID(),
@@ -182,7 +180,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                     let shouldTriggerHintsFlow;
                     let hintsExecutionCompletion;
                     let elementAnnotation;
-                    await page.waitForTimeout(2000);
                     const actionType = toolCall.actionType;
                     let preference = {
                         actionType: "all",

package/dist/bin/index.d.ts

@@ -1,3 +1,3 @@
 #!/usr/bin/env node
-export {};
+import "../initSentry";
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/bin/index.ts"],"names":[],"mappings":""}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/bin/index.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAC"}

package/dist/bin/index.js

@@ -1,10 +1,35 @@
 #!/usr/bin/env node
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+require("../initSentry");
 const llm_1 = require("@empiricalrun/llm");
+const Sentry = __importStar(require("@sentry/node"));
 const dotenv_1 = __importDefault(require("dotenv"));
 const run_1 = require("../agent/browsing/run");
 const utils_1 = require("../agent/browsing/utils");
@@ -21,10 +46,14 @@ const utils_2 = require("./utils");
 dotenv_1.default.config({
     path: [".env.local", ".env"],
 });
-process.on("beforeExit", async () => await (0, llm_1.flushAllTraces)());
-process.on("exit", async () => await (0, llm_1.flushAllTraces)());
-process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
-process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());
+const flushEvents = async () => {
+    await (0, llm_1.flushAllTraces)();
+    await Sentry.flush();
+};
+process.on("beforeExit", async () => await flushEvents());
+process.on("exit", async () => await flushEvents());
+process.on("SIGINT", async () => await flushEvents());
+process.on("SIGTERM", async () => await flushEvents());
 async function resolveAgentUsingTask({ testCase, trace, }) {
     const { response } = await (0, infer_agent_1.inferAgentBasedTask)({
         task: testCase.steps.join("\n"),
@@ -32,7 +61,7 @@ async function resolveAgentUsingTask({ testCase, trace, }) {
     });
     return response;
 }
-async function runAgent(testGenConfig) {
+async function runAgent(testGenConfig, span) {
     const logger = new logger_1.CustomLogger();
     const { specPath, testCase } = testGenConfig;
     if (process.env.LOG_URL) {
@@ -40,6 +69,7 @@ async function runAgent(testGenConfig) {
             void new reporter_1.TestGenUpdatesReporter().sendLogUrl(process.env.LOG_URL);
         }
         catch (e) {
+            span?.recordException(e);
             console.warn("Failed to send log url to test gen update", e);
         }
     }
@@ -127,50 +157,57 @@ async function runAgent(testGenConfig) {
     }
 }
 (async function main() {
-    // this is where test gen starts executing on giving the command from ci
-    const logger = new logger_1.CustomLogger({ useReporter: false });
-    if (process.argv.length < 3) {
-        logger.error("Please provide path to scenarios using command:", "npx @empiricalrun/test-gen <TEST_GEN_TOKEN>");
-        process.exit(1);
-    }
-    const { testGenConfig } = await (0, utils_2.parseCliArgs)();
-    (0, reporter_1.setReporterConfig)({
-        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-        testSessionId: testGenConfig.options?.metadata.testSessionId,
-        generationId: testGenConfig.options?.metadata.generationId,
-    });
-    (0, session_1.setSessionDetails)({
-        sessionId: testGenConfig.options?.metadata.testSessionId,
-        generationId: testGenConfig.options?.metadata.generationId,
-        testCaseId: testGenConfig.testCase.id,
-        projectRepoName: testGenConfig.options?.metadata.projectRepoName,
-    });
-    let testGenFailed = false;
-    try {
-        // download the build if it exists
-        await (0, test_build_1.downloadBuild)(testGenConfig.build || {});
-        await runAgent(testGenConfig);
-    }
-    catch (e) {
-        testGenFailed = true;
-        new logger_1.CustomLogger().error(`Failed to generate test for the scenario. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`, e?.message, e?.stack);
-    }
-    if (testGenConfig.options?.agent !== "code" &&
-        testGenConfig.options?.agent !== "plan" &&
-        testGenConfig.testCase.name) {
-        await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
-            projectRepoName: testGenConfig.options.metadata.projectRepoName,
-            testName: testGenConfig.testCase.name,
+    await Sentry.continueTrace({ sentryTrace: utils_2.sentryTrace, baggage: utils_2.baggage }, async () => {
+        await Sentry.startSpan({ name: "test-gen" }, async (span) => {
+            // this is where test gen starts executing on giving the command from ci
+            const logger = new logger_1.CustomLogger({ useReporter: false });
+            if (process.argv.length < 3) {
+                logger.error("Please provide path to scenarios using command:", "npx @empiricalrun/test-gen <TEST_GEN_TOKEN>");
+                process.exit(1);
+            }
+            const { testGenConfig } = await (0, utils_2.parseCliArgs)();
+            (0, reporter_1.setReporterConfig)({
+                projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+                testSessionId: testGenConfig.options?.metadata.testSessionId,
+                generationId: testGenConfig.options?.metadata.generationId,
+            });
+            (0, session_1.setSessionDetails)({
+                sessionId: testGenConfig.options?.metadata.testSessionId,
+                generationId: testGenConfig.options?.metadata.generationId,
+                testCaseId: testGenConfig.testCase.id,
+                projectRepoName: testGenConfig.options?.metadata.projectRepoName,
+            });
+            let testGenFailed = false;
+            try {
+                // download the build if it exists
+                await (0, test_build_1.downloadBuild)(testGenConfig.build || {});
+                await runAgent(testGenConfig);
+            }
+            catch (e) {
+                span.recordException(e);
+                Sentry.captureException(e);
+                testGenFailed = true;
+                new logger_1.CustomLogger().error(`Failed to generate test for the scenario. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`, e?.message, e?.stack);
+            }
+            if (testGenConfig.options?.agent !== "code" &&
+                testGenConfig.options?.agent !== "plan" &&
+                testGenConfig.testCase.name) {
+                await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
+                    projectRepoName: testGenConfig.options.metadata.projectRepoName,
+                    testName: testGenConfig.testCase.name,
+                });
+            }
+            // TODO: move these reporters to a better lifecycle
+            await (0, llm_1.flushAllTraces)();
+            await (0, logger_1.waitForLogsToFlush)();
+            await (0, session_1.endSession)();
+            span.end();
+            if (testGenFailed) {
+                process.exit(1);
+            }
+            else {
+                process.exit(0);
+            }
         });
-    }
-    // TODO: move these reporters to a better lifecycle
-    await (0, llm_1.flushAllTraces)();
-    await (0, logger_1.waitForLogsToFlush)();
-    await (0, session_1.endSession)();
-    if (testGenFailed) {
-        process.exit(1);
-    }
-    else {
-        process.exit(0);
-    }
+    });
 })();

package/dist/bin/utils/index.d.ts

@@ -3,4 +3,6 @@ export declare function parseCliArgs(testGenToken?: string): Promise<{
     testGenConfig: TestGenConfig;
 }>;
 export declare function getTestConfigCliArg(): string;
+export declare const sentryTrace: string | undefined;
+export declare const baggage: string | undefined;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAIhE,wBAAsB,YAAY,CAChC,YAAY,GAAE,MAA8B;;GAM7C;AAED,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAIhE,wBAAsB,YAAY,CAChC,YAAY,GAAE,MAA8B;;GAM7C;AAED,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C;AAED,eAAO,MAAM,WAAW,oBAA2B,CAAC;AACpD,eAAO,MAAM,OAAO,oBAA6B,CAAC"}

package/dist/bin/utils/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTestConfigCliArg = exports.parseCliArgs = void 0;
+exports.baggage = exports.sentryTrace = exports.getTestConfigCliArg = exports.parseCliArgs = void 0;
 const scenarios_1 = require("./scenarios");
 async function parseCliArgs(testGenToken = getTestConfigCliArg()) {
     const testGenConfig = await (0, scenarios_1.loadTestConfigs)(testGenToken);
@@ -13,3 +13,5 @@ function getTestConfigCliArg() {
     return process.argv[2];
 }
 exports.getTestConfigCliArg = getTestConfigCliArg;
+exports.sentryTrace = process.env.SENTRY_TRACE;
+exports.baggage = process.env.SENTRY_BAGGAGE;

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+import "./initSentry";
 import { Page } from "playwright";
 import { ScopeVars } from "./types";
 export declare function createTest(task: string, page: Page, scope?: ScopeVars): Promise<void>;

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAQpC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAuC3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,cAAc,CAAC;AAItB,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAapC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAuC3E"}

package/dist/index.js

@@ -1,20 +1,49 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createTest = void 0;
+require("./initSentry");
 const llm_1 = require("@empiricalrun/llm");
+const Sentry = __importStar(require("@sentry/node"));
 const run_1 = require("./agent/master/run");
 const utils_1 = require("./bin/utils");
 const client_1 = __importDefault(require("./file/client"));
 const reporter_1 = require("./reporter");
 const session_1 = require("./session");
 const pw_test_1 = require("./utils/pw-test");
-process.on("beforeExit", async () => await (0, llm_1.flushAllTraces)());
-process.on("exit", async () => await (0, llm_1.flushAllTraces)());
-process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
-process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());
+const flushEvents = async () => {
+    await (0, llm_1.flushAllTraces)();
+    await Sentry.flush();
+};
+process.on("beforeExit", async () => await flushEvents());
+process.on("exit", async () => await flushEvents());
+process.on("SIGINT", async () => await flushEvents());
+process.on("SIGTERM", async () => await flushEvents());
 async function createTest(task, page, scope) {
     const port = process.env.APP_PORT || 3030;
     const testConfigArg = process.env.TEST_GEN_TOKEN;

package/dist/initSentry.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=initSentry.d.ts.map

package/dist/initSentry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"initSentry.d.ts","sourceRoot":"","sources":["../src/initSentry.ts"],"names":[],"mappings":""}

package/dist/initSentry.js

@@ -0,0 +1,37 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Sentry = __importStar(require("@sentry/node"));
+Sentry.init({
+    enabled: process.env.NODE_ENV !== "development",
+    dsn: "https://87e61b11ede1431d7156bcd26da997cc@o4506822020235264.ingest.us.sentry.io/4508806031015936",
+    tracesSampleRate: 1.0,
+    serverName: "test-gen",
+    debug: process.env.NODE_ENV === "development",
+    integrations: [Sentry.consoleIntegration(), Sentry.httpIntegration()],
+    clientReportFlushInterval: 2000,
+    shutdownTimeout: 5000,
+    enableTracing: true,
+});

package/dist/test-build/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test-build/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAUnD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAY/D"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test-build/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAWnD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAa/D"}

package/dist/test-build/index.js

@@ -1,9 +1,33 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.downloadBuild = void 0;
+const Sentry = __importStar(require("@sentry/node"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../bin/logger");
 const exec_1 = require("../utils/exec");
@@ -24,6 +48,7 @@ async function downloadBuild(build) {
         logger.log(`Downloading build from ${build.url}`);
         await (0, exec_1.cmd)(`npm run download ${build.url}`.split(" "), {
             env: { ...Object(process.env) },
+            span: Sentry.getActiveSpan(),
         });
     }
 }

package/dist/utils/exec.d.ts

@@ -1,4 +1,6 @@
+import * as Sentry from "@sentry/node";
 export declare function cmd(command: string[], options: {
     env?: Record<string, string>;
+    span?: Sentry.Span;
 }): Promise<number>;
 //# sourceMappingURL=exec.d.ts.map

package/dist/utils/exec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAGA,wBAAgB,GAAG,CACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GACxC,OAAO,CAAC,MAAM,CAAC,CA2BjB"}
+{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../../src/utils/exec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,cAAc,CAAC;AAMvC,wBAAsB,GAAG,CACvB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAA;CAAE,GAC5D,OAAO,CAAC,MAAM,CAAC,CA+CjB"}

package/dist/utils/exec.js

@@ -1,37 +1,77 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.cmd = void 0;
+const Sentry = __importStar(require("@sentry/node"));
 const child_process_1 = require("child_process");
 const process_1 = __importDefault(require("process"));
-function cmd(command, options) {
-    let errorLogs = [];
-    return new Promise((resolveFunc, rejectFunc) => {
-        let p = (0, child_process_1.spawn)(command[0], command.slice(1), {
-            env: { ...process_1.default.env, ...options.env },
-        });
-        p.stdout.on("data", (x) => {
-            const log = x.toString();
-            if (log.includes("Error")) {
-                errorLogs.push(log);
-            }
-            process_1.default.stdout.write(log);
-        });
-        p.stderr.on("data", (x) => {
-            const log = x.toString();
-            process_1.default.stderr.write(x.toString());
-            errorLogs.push(log);
-        });
-        p.on("exit", (code) => {
-            if (code != 0) {
-                // assuming last log is the error message before exiting
-                rejectFunc(errorLogs.slice(-3).join("\n"));
-            }
-            else {
-                resolveFunc(code);
-            }
+const utils_1 = require("../bin/utils");
+async function cmd(command, options) {
+    return Sentry.continueTrace({ sentryTrace: utils_1.sentryTrace, baggage: utils_1.baggage }, async () => {
+        return Sentry.startSpan({ name: "test-gen" }, async (span) => {
+            let errorLogs = [];
+            return new Promise((resolveFunc, rejectFunc) => {
+                if (!command[0]) {
+                    rejectFunc(new Error("Command cannot be empty"));
+                    return;
+                }
+                let p = (0, child_process_1.spawn)(command[0], command.slice(1), {
+                    env: { ...process_1.default.env, ...options.env },
+                });
+                p.stdout.on("data", (x) => {
+                    const log = x.toString();
+                    if (log.includes("Error")) {
+                        errorLogs.push(log);
+                    }
+                    process_1.default.stdout.write(log);
+                });
+                p.stderr.on("data", (x) => {
+                    const log = x.toString();
+                    process_1.default.stderr.write(log);
+                    errorLogs.push(log);
+                });
+                p.on("error", async (err) => {
+                    span?.recordException(err);
+                    Sentry.captureException(err);
+                    rejectFunc(err);
+                });
+                p.on("exit", async (code) => {
+                    if (code !== 0) {
+                        const errorMessage = errorLogs.slice(-3).join("\n");
+                        span?.recordException(errorMessage);
+                        Sentry.captureException(new Error(errorMessage));
+                        rejectFunc(new Error(errorMessage));
+                    }
+                    else {
+                        resolveFunc(code ?? 1);
+                    }
+                });
+            });
         });
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.42.27",
+  "version": "0.42.29",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -47,6 +47,7 @@
   "dependencies": {
     "@actions/core": "^1.10.1",
     "@babel/parser": "^7.26.3",
+    "@sentry/node": "^8.54.0",
     "@types/sanitize-html": "^2.11.0",
     "commander": "^12.1.0",
     "detect-port": "^1.6.1",
@@ -72,8 +73,8 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "@empiricalrun/reporter": "^0.23.1",
-    "@empiricalrun/llm": "^0.9.34",
-    "@empiricalrun/r2-uploader": "^0.3.8"
+    "@empiricalrun/r2-uploader": "^0.3.8",
+    "@empiricalrun/llm": "^0.9.34"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",