@empiricalrun/test-gen 0.23.1 → 0.23.3

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.23.3
+
+### Patch Changes
+
+- Updated dependencies [f02f7a7]
+  - @empiricalrun/llm@0.7.3
+
+## 0.23.2
+
+### Patch Changes
+
+- f6f488d: feat: add support for cancelling generation
+
 ## 0.23.1
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBA2K9B;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAqG9B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBA0L9B;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAqG9B"}

package/dist/agent/browsing/index.js

@@ -50,6 +50,11 @@ async function browsingAgentUsingMasterAgent(task, page, options) {
                 break;
             }
         }
+        const sessionState = await (0, session_1.getSessionState)();
+        if (sessionState === "request_complete") {
+            await testgenUpdatesReporter.sendMessage("Aborting task, marking the task as done.");
+            break;
+        }
         const { action, reason } = await (0, run_1.masterAgent)(task, page, masterAgentActions, masterAgentSpan, llm, options);
         logger.log(`Next action: ${action} \n reason: ${reason}`);
         if (!action) {
@@ -64,6 +69,10 @@ async function browsingAgentUsingMasterAgent(task, page, options) {
             const browsingAgentSpan = masterAgentSpan.span({
                 name: `browsing-agent`,
             });
+            const sessionState = await (0, session_1.getSessionState)();
+            if (sessionState === "request_complete") {
+                break;
+            }
             const pageContentSpan = browsingAgentSpan.span({ name: "page-content" });
             const pageContent = await page.content();
             pageContentSpan.end({ output: { pageContent } });
@@ -118,6 +127,10 @@ async function browsingAgentUsingMasterAgent(task, page, options) {
             const toolCallsSpan = browsingAgentSpan.span({ name: "tool-calls" });
             for (const i in toolCalls) {
                 const toolCall = toolCalls[i];
+                const sessionState = await (0, session_1.getSessionState)();
+                if (sessionState === "request_complete") {
+                    break;
+                }
                 try {
                     await actions.executeAction(toolCall.function.name, JSON.parse(toolCall.function.arguments));
                     executedActions.push({

package/dist/bin/index.js

@@ -11,6 +11,7 @@ const utils_1 = require("../agent/browsing/utils");
 const run_2 = require("../agent/codegen/run");
 const reporter_1 = require("../reporter");
 const ci_1 = require("../reporter/ci");
+const session_1 = require("../session");
 const logger_1 = require("./logger");
 const utils_2 = require("./utils");
 dotenv_1.default.config({
@@ -51,11 +52,17 @@ async function runAgent(testGenConfig) {
         testGroup: testGenConfig.testCase.group,
         projectRepoName: testGenConfig.options?.metadata.projectRepoName,
         testSessionId: testGenConfig.options?.metadata.testSessionId,
+        generationId: testGenConfig.options?.metadata.generationId,
+    });
+    (0, session_1.setSessionDetails)({
+        sessionId: testGenConfig.options?.metadata.testSessionId,
+        generationId: testGenConfig.options?.metadata.generationId,
     });
     await runAgent(testGenConfig);
     // TODO: move these reporters to a better lifecycle
     await (0, ci_1.reportOnCI)(testGenConfig.testCase);
     await (0, llm_1.flushAllTraces)();
     await (0, logger_1.waitForLogsToFlush)();
+    await (0, session_1.endSession)();
     process.exit(0);
 })();

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBA0BnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAalC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBA+BnE"}

package/dist/index.js

@@ -9,6 +9,7 @@ const browsing_1 = require("./agent/browsing");
 const utils_1 = require("./bin/utils");
 const client_1 = __importDefault(require("./file/client"));
 const reporter_1 = require("./reporter");
+const session_1 = require("./session");
 process.on("beforeExit", async () => await (0, llm_1.flushAllTraces)());
 process.on("exit", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
@@ -22,6 +23,11 @@ async function createTest(task, page, test) {
         testGroup: testGenConfig.testCase.group,
         projectRepoName: testGenConfig.options?.metadata.projectRepoName,
         testSessionId: testGenConfig.options?.metadata.testSessionId,
+        generationId: testGenConfig.options?.metadata.generationId,
+    });
+    (0, session_1.setSessionDetails)({
+        sessionId: testGenConfig.options?.metadata.testSessionId,
+        generationId: testGenConfig.options?.metadata.generationId,
     });
     const fileService = new client_1.default(Number(port));
     test.setTimeout(900000);

package/dist/reporter/index.d.ts

@@ -1,6 +1,7 @@
 import { Reporter } from "@empiricalrun/reporter";
 type ReporterConfigType = {
     testSessionId: number;
+    generationId: number;
     projectRepoName: string;
     testCaseName: string;
     testGroup: string;

package/dist/reporter/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAc5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAkCK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2B9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAUlD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAc5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAkCK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2B9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAUlD"}

package/dist/session/index.d.ts

@@ -1,6 +1,12 @@
 declare function getSessionDetails(): {
-    id: `${string}-${string}-${string}-${string}-${string}`;
+    id: string;
     version: string;
 };
+export declare function setSessionDetails({ sessionId, generationId, }: {
+    sessionId: number;
+    generationId: number;
+}): void;
+export declare function getSessionState(): Promise<"started" | "completed" | "request_complete">;
+export declare function endSession(): Promise<void>;
 export { getSessionDetails };
 //# sourceMappingURL=index.d.ts.map

package/dist/session/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAKA,iBAAS,iBAAiB;;;EAKzB;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAeA,iBAAS,iBAAiB;;;EAKzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB,QAGA;AAED,wBAAsB,eAAe,0DAkBpC;AAED,wBAAsB,UAAU,kBAmB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/session/index.js

@@ -3,14 +3,59 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getSessionDetails = void 0;
-const crypto_1 = __importDefault(require("crypto"));
+exports.getSessionDetails = exports.endSession = exports.getSessionState = exports.setSessionDetails = void 0;
 const package_json_1 = __importDefault(require("./../../package.json"));
-const sessionId = crypto_1.default.randomUUID();
+const sessionDetails = {
+    sessionId: undefined,
+    version: package_json_1.default.version,
+    generationId: undefined,
+};
+const DASHBOARD_DOMAIN = process.env.DASHBOARD_DOMAIN || "https://dash.empirical.run";
 function getSessionDetails() {
     return {
-        id: sessionId,
+        id: `${sessionDetails.generationId}`,
         version: package_json_1.default.version,
     };
 }
 exports.getSessionDetails = getSessionDetails;
+function setSessionDetails({ sessionId, generationId, }) {
+    sessionDetails.sessionId = sessionId;
+    sessionDetails.generationId = generationId;
+}
+exports.setSessionDetails = setSessionDetails;
+async function getSessionState() {
+    const apiPath = `${DASHBOARD_DOMAIN}/api/sessions/${sessionDetails.sessionId}/generation?generation_id=${sessionDetails.generationId}`;
+    const response = await fetch(apiPath, {
+        method: "GET",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: "weQPMWKT",
+        },
+    });
+    const generationStateData = (await response.json());
+    // TODO: fix this heirarchy
+    return generationStateData.data.state.state;
+}
+exports.getSessionState = getSessionState;
+async function endSession() {
+    const apiPath = `${DASHBOARD_DOMAIN}/api/sessions/${sessionDetails.sessionId}/generation?generation_id=${sessionDetails.generationId}`;
+    try {
+        await fetch(apiPath, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: "weQPMWKT",
+            },
+            body: JSON.stringify({
+                generationId: sessionDetails.generationId,
+                state: {
+                    state: "completed",
+                },
+            }),
+        });
+    }
+    catch (error) {
+        console.error("Failed to end session:", error);
+    }
+}
+exports.endSession = endSession;

package/dist/types/index.d.ts

@@ -12,6 +12,7 @@ export type TestGenConfigOptions = {
     modelParameters?: ModelParameters;
     metadata: {
         testSessionId: number;
+        generationId: number;
         testCaseName: string;
         projectRepoName: string;
         projectName: string;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;IACrC,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;IACrC,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.23.1",
+  "version": "0.23.3",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.7.2",
+    "@empiricalrun/llm": "^0.7.3",
     "@empiricalrun/reporter": "^0.17.5"
   },
   "devDependencies": {