@mastra/evals 0.1.0-alpha.7 → 0.1.0-alpha.9

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @mastra/evals
 
+## 0.1.0-alpha.9
+
+### Patch Changes
+
+- 1944807: Unified logger and major step in better logs
+- 9ade36e: Changed measure for evals, added endpoints, attached metrics to agent, added ui for evals in playground, and updated docs
+- Updated dependencies [1944807]
+- Updated dependencies [9ade36e]
+  - @mastra/core@0.1.27-alpha.69
+
+## 0.1.0-alpha.8
+
+### Patch Changes
+
+- Updated dependencies [0be7181]
+- Updated dependencies [0be7181]
+  - @mastra/core@0.1.27-alpha.68
+
 ## 0.1.0-alpha.7
 
 ### Patch Changes

package/dist/attachListeners.d.ts

@@ -0,0 +1,3 @@
+export declare function attachListeners(): Promise<void>;
+export declare function globalSetup(): Promise<void>;
+//# sourceMappingURL=attachListeners.d.ts.map

package/dist/attachListeners.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attachListeners.d.ts","sourceRoot":"","sources":["../src/attachListeners.ts"],"names":[],"mappings":"AAMA,wBAAsB,eAAe,kBAUpC;AAED,wBAAsB,WAAW,kBAOhC"}

package/dist/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const GLOBAL_RUN_ID_ENV_KEY = "_MASTRA_GLOBAL_RUN_ID_";
+//# sourceMappingURL=constants.d.ts.map

package/dist/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,qBAAqB,2BAA2B,CAAC"}

package/dist/evals.cjs.development.js

@@ -3,6 +3,8 @@
 Object.defineProperty(exports, '__esModule', { value: true });
 
 var core = require('@mastra/core');
+var fs = require('fs');
+var path = require('path');
 
 function asyncGeneratorStep(n, t, e, r, o, a, c) {
   try {
@@ -331,41 +333,154 @@ function _regeneratorRuntime() {
   }, e;
 }
 
+var GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';
+
 function evaluate(_x, _x2, _x3) {
   return _evaluate.apply(this, arguments);
 }
 function _evaluate() {
-  _evaluate = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(agent, input, metric) {
-    var agentOutput, metricResult;
-    return _regeneratorRuntime().wrap(function _callee$(_context) {
-      while (1) switch (_context.prev = _context.next) {
+  _evaluate = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(agent, input, metric) {
+    var testInfo, globalRunId, runId, agentOutput, metricResult;
+    return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+      while (1) switch (_context2.prev = _context2.next) {
         case 0:
-          _context.next = 2;
-          return agent.generate(input);
+          _context2.next = 2;
+          return getCurrentTestInfo();
         case 2:
-          agentOutput = _context.sent;
-          _context.next = 5;
-          return metric.measure({
-            input: input.toString(),
-            output: agentOutput.text
+          testInfo = _context2.sent;
+          globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];
+          runId = crypto.randomUUID();
+          _context2.next = 7;
+          return agent.generate(input, {
+            runId: runId
           });
-        case 5:
-          metricResult = _context.sent;
-          // capture infomration about the evaluation
-          core.executeHook(core.AvailableHooks.ON_EVALUATION, {
-            input: input.toString(),
+        case 7:
+          agentOutput = _context2.sent;
+          if (!globalRunId) {
+            globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();
+            console.warn('Global run id not set, you should run "globalSetup" from "@mastra/evals" before evaluating.');
+          }
+          _context2.next = 11;
+          return core.evaluate({
+            agentName: agent.name,
+            input: input,
+            metric: metric,
             output: agentOutput.text,
-            result: metricResult
+            globalRunId: globalRunId,
+            runId: runId,
+            testInfo: testInfo
           });
-          return _context.abrupt("return", metricResult);
-        case 8:
+        case 11:
+          metricResult = _context2.sent;
+          return _context2.abrupt("return", metricResult);
+        case 13:
+        case "end":
+          return _context2.stop();
+      }
+    }, _callee2);
+  }));
+  return _evaluate.apply(this, arguments);
+}
+var getCurrentTestInfo = /*#__PURE__*/function () {
+  var _ref = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee() {
+    var state, _vitest$expect, vitest, _state;
+    return _regeneratorRuntime().wrap(function _callee$(_context) {
+      while (1) switch (_context.prev = _context.next) {
+        case 0:
+          if (!(typeof expect !== 'undefined' && expect.getState)) {
+            _context.next = 3;
+            break;
+          }
+          state = expect.getState();
+          return _context.abrupt("return", {
+            testName: state.currentTestName,
+            testPath: state.testPath
+          });
+        case 3:
+          _context.prev = 3;
+          _context.next = 6;
+          return import('vitest');
+        case 6:
+          vitest = _context.sent;
+          if (!(typeof vitest !== 'undefined' && (_vitest$expect = vitest.expect) != null && _vitest$expect.getState)) {
+            _context.next = 10;
+            break;
+          }
+          _state = vitest.expect.getState();
+          return _context.abrupt("return", {
+            testName: _state.currentTestName,
+            testPath: _state.testPath
+          });
+        case 10:
+          _context.next = 14;
+          break;
+        case 12:
+          _context.prev = 12;
+          _context.t0 = _context["catch"](3);
+        case 14:
+          return _context.abrupt("return", null);
+        case 15:
+        case "end":
+          return _context.stop();
+      }
+    }, _callee, null, [[3, 12]]);
+  }));
+  return function getCurrentTestInfo() {
+    return _ref.apply(this, arguments);
+  };
+}();
+
+function attachListeners() {
+  return _attachListeners.apply(this, arguments);
+}
+function _attachListeners() {
+  _attachListeners = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee() {
+    var dotMastraPath;
+    return _regeneratorRuntime().wrap(function _callee$(_context) {
+      while (1) switch (_context.prev = _context.next) {
+        case 0:
+          dotMastraPath = path.join(process.cwd(), '.mastra');
+          try {
+            fs.mkdirSync(dotMastraPath);
+          } catch (error) {}
+          core.registerHook(core.AvailableHooks.ON_EVALUATION, function (traceObject) {
+            fs.appendFile(path.join(dotMastraPath, 'evals.json'), JSON.stringify(traceObject) + '\n', function () {});
+          });
+        case 3:
         case "end":
           return _context.stop();
       }
     }, _callee);
   }));
-  return _evaluate.apply(this, arguments);
+  return _attachListeners.apply(this, arguments);
+}
+function globalSetup() {
+  return _globalSetup.apply(this, arguments);
+}
+function _globalSetup() {
+  _globalSetup = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2() {
+    var globalRunId;
+    return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+      while (1) switch (_context2.prev = _context2.next) {
+        case 0:
+          if (!process.env[GLOBAL_RUN_ID_ENV_KEY]) {
+            _context2.next = 2;
+            break;
+          }
+          throw new Error('Global run id already set, you should only run "GlobalSetup" once');
+        case 2:
+          globalRunId = crypto.randomUUID();
+          process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;
+        case 4:
+        case "end":
+          return _context2.stop();
+      }
+    }, _callee2);
+  }));
+  return _globalSetup.apply(this, arguments);
 }
 
+exports.attachListeners = attachListeners;
 exports.evaluate = evaluate;
+exports.globalSetup = globalSetup;
 //# sourceMappingURL=evals.cjs.development.js.map

package/dist/evals.cjs.development.js.map

@@ -1 +1 @@
-{"version":3,"file":"evals.cjs.development.js","sources":["../src/evaluation.ts"],"sourcesContent":["import { AvailableHooks, executeHook } from '@mastra/core';\nimport { type Agent, type Metric } from '@mastra/core';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const agentOutput = await agent.generate(input);\n\n  const metricResult = await metric.measure({\n    input: input.toString(),\n    output: agentOutput.text,\n  });\n\n  // capture infomration about the evaluation\n  executeHook(AvailableHooks.ON_EVALUATION, {\n    input: input.toString(),\n    output: agentOutput.text,\n    result: metricResult,\n  });\n\n  return metricResult;\n}\n"],"names":["evaluate","_x","_x2","_x3","_evaluate","apply","arguments","_asyncToGenerator","_regeneratorRuntime","mark","_callee","agent","input","metric","agentOutput","metricResult","wrap","_callee$","_context","prev","next","generate","sent","measure","toString","output","text","executeHook","AvailableHooks","ON_EVALUATION","result","abrupt","stop"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,SAAsBA,QAAQA,CAAAC,EAAA,EAAAC,GAAA,EAAAC,GAAA,EAAA;AAAA,EAAA,OAAAC,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAgB7B,SAAAF,SAAA,GAAA;AAAAA,EAAAA,SAAA,GAAAG,iBAAA,cAAAC,mBAAA,EAAAC,CAAAA,IAAA,CAhBM,SAAAC,OAAyCC,CAAAA,KAAQ,EAAEC,KAAmC,EAAEC,MAAc,EAAA;IAAA,IAAAC,WAAA,EAAAC,YAAA,CAAA;AAAA,IAAA,OAAAP,mBAAA,EAAA,CAAAQ,IAAA,CAAA,SAAAC,SAAAC,QAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;AAAA,QAAA,KAAA,CAAA;AAAAF,UAAAA,QAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;AAAA,UAAA,OACjFT,KAAK,CAACU,QAAQ,CAACT,KAAK,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAAzCE,WAAW,GAAAI,QAAA,CAAAI,IAAA,CAAA;AAAAJ,UAAAA,QAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;UAAA,OAEUP,MAAM,CAACU,OAAO,CAAC;AACxCX,YAAAA,KAAK,EAAEA,KAAK,CAACY,QAAQ,EAAE;YACvBC,MAAM,EAAEX,WAAW,CAACY,IAAAA;AACrB,WAAA,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAHIX,YAAY,GAAAG,QAAA,CAAAI,IAAA,CAAA;AAKlB;AACAK,UAAAA,gBAAW,CAACC,mBAAc,CAACC,aAAa,EAAE;AACxCjB,YAAAA,KAAK,EAAEA,KAAK,CAACY,QAAQ,EAAE;YACvBC,MAAM,EAAEX,WAAW,CAACY,IAAI;AACxBI,YAAAA,MAAM,EAAEf,YAAAA;AACT,WAAA,CAAC,CAAA;AAAC,UAAA,OAAAG,QAAA,CAAAa,MAAA,CAAA,QAAA,EAEIhB,YAAY,CAAA,CAAA;AAAA,QAAA,KAAA,CAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAG,QAAA,CAAAc,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAAtB,OAAA,CAAA,CAAA;GACpB,CAAA,CAAA,CAAA;AAAA,EAAA,OAAAN,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA;;;;"}
+{"version":3,"file":"evals.cjs.development.js","sources":["../src/constants.ts","../src/evaluation.ts","../src/attachListeners.ts"],"sourcesContent":["export const GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';\n","import { type Agent, type Metric, evaluate as coreEvaluate } from '@mastra/core';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const testInfo = await getCurrentTestInfo();\n  let globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];\n  const runId = crypto.randomUUID();\n  const agentOutput = await agent.generate(input, {\n    runId,\n  });\n\n  if (!globalRunId) {\n    globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();\n    console.warn('Global run id not set, you should run \"globalSetup\" from \"@mastra/evals\" before evaluating.');\n  }\n\n  const metricResult = await coreEvaluate({\n    agentName: agent.name,\n    input,\n    metric,\n    output: agentOutput.text,\n    globalRunId,\n    runId,\n    testInfo,\n  });\n\n  return metricResult;\n}\n\nexport const getCurrentTestInfo = async () => {\n  // Jest\n  if (typeof expect !== 'undefined' && expect.getState) {\n    const state = expect.getState();\n    return {\n      testName: state.currentTestName,\n      testPath: state.testPath,\n    };\n  }\n\n  try {\n    const vitest = await import('vitest');\n    if (typeof vitest !== 'undefined' && vitest.expect?.getState) {\n      const state = vitest.expect.getState();\n      return {\n        testName: state.currentTestName,\n        testPath: state.testPath,\n      };\n    }\n  } catch {}\n\n  return null;\n};\n","import { AvailableHooks, registerHook } from '@mastra/core';\nimport { mkdirSync, appendFile } from 'fs';\nimport { join } from 'path';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function attachListeners() {\n  const dotMastraPath = join(process.cwd(), '.mastra');\n\n  try {\n    mkdirSync(dotMastraPath);\n  } catch (error) {}\n\n  registerHook(AvailableHooks.ON_EVALUATION, traceObject => {\n    appendFile(join(dotMastraPath, 'evals.json'), JSON.stringify(traceObject) + '\\n', () => {});\n  });\n}\n\nexport async function globalSetup() {\n  if (process.env[GLOBAL_RUN_ID_ENV_KEY]) {\n    throw new Error('Global run id already set, you should only run \"GlobalSetup\" once');\n  }\n\n  const globalRunId = crypto.randomUUID();\n  process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;\n}\n"],"names":["GLOBAL_RUN_ID_ENV_KEY","evaluate","_x","_x2","_x3","_evaluate","apply","arguments","_asyncToGenerator","_regeneratorRuntime","mark","_callee2","agent","input","metric","testInfo","globalRunId","runId","agentOutput","metricResult","wrap","_callee2$","_context2","prev","next","getCurrentTestInfo","sent","process","env","crypto","randomUUID","generate","console","warn","coreEvaluate","agentName","name","output","text","abrupt","stop","_ref","_callee","state","_vitest$expect","vitest","_state","_callee$","_context","expect","getState","testName","currentTestName","testPath","t0","attachListeners","_attachListeners","dotMastraPath","join","cwd","mkdirSync","error","registerHook","AvailableHooks","ON_EVALUATION","traceObject","appendFile","JSON","stringify","globalSetup","_globalSetup","Error"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAO,IAAMA,qBAAqB,GAAG,wBAAwB;;ACI7D,SAAsBC,QAAQA,CAAAC,EAAA,EAAAC,GAAA,EAAAC,GAAA,EAAA;AAAA,EAAA,OAAAC,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAwB7B,SAAAF,SAAA,GAAA;AAAAA,EAAAA,SAAA,GAAAG,iBAAA,cAAAC,mBAAA,EAAAC,CAAAA,IAAA,CAxBM,SAAAC,QAAyCC,CAAAA,KAAQ,EAAEC,KAAmC,EAAEC,MAAc,EAAA;IAAA,IAAAC,QAAA,EAAAC,WAAA,EAAAC,KAAA,EAAAC,WAAA,EAAAC,YAAA,CAAA;AAAA,IAAA,OAAAV,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAAC,UAAAC,SAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,SAAA,CAAAC,IAAA,GAAAD,SAAA,CAAAE,IAAA;AAAA,QAAA,KAAA,CAAA;AAAAF,UAAAA,SAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;UAAA,OACpFC,kBAAkB,EAAE,CAAA;AAAA,QAAA,KAAA,CAAA;UAArCV,QAAQ,GAAAO,SAAA,CAAAI,IAAA,CAAA;AACVV,UAAAA,WAAW,GAAGW,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,CAAA;AAC9CiB,UAAAA,KAAK,GAAGY,MAAM,CAACC,UAAU,EAAE,CAAA;AAAAR,UAAAA,SAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;AAAA,UAAA,OACPZ,KAAK,CAACmB,QAAQ,CAAClB,KAAK,EAAE;AAC9CI,YAAAA,KAAK,EAALA,KAAAA;AACD,WAAA,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAFIC,WAAW,GAAAI,SAAA,CAAAI,IAAA,CAAA;UAIjB,IAAI,CAACV,WAAW,EAAE;AAChBA,YAAAA,WAAW,GAAGW,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,GAAG6B,MAAM,CAACC,UAAU,EAAE,CAAA;AACtEE,YAAAA,OAAO,CAACC,IAAI,CAAC,6FAA6F,CAAC,CAAA;AAC7G,WAAA;AAACX,UAAAA,SAAA,CAAAE,IAAA,GAAA,EAAA,CAAA;AAAA,UAAA,OAE0BU,aAAY,CAAC;YACtCC,SAAS,EAAEvB,KAAK,CAACwB,IAAI;AACrBvB,YAAAA,KAAK,EAALA,KAAK;AACLC,YAAAA,MAAM,EAANA,MAAM;YACNuB,MAAM,EAAEnB,WAAW,CAACoB,IAAI;AACxBtB,YAAAA,WAAW,EAAXA,WAAW;AACXC,YAAAA,KAAK,EAALA,KAAK;AACLF,YAAAA,QAAQ,EAARA,QAAAA;AACD,WAAA,CAAC,CAAA;AAAA,QAAA,KAAA,EAAA;UARII,YAAY,GAAAG,SAAA,CAAAI,IAAA,CAAA;AAAA,UAAA,OAAAJ,SAAA,CAAAiB,MAAA,CAAA,QAAA,EAUXpB,YAAY,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAG,SAAA,CAAAkB,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAA7B,QAAA,CAAA,CAAA;GACpB,CAAA,CAAA,CAAA;AAAA,EAAA,OAAAN,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAEM,IAAMkB,kBAAkB,gBAAA,YAAA;EAAA,IAAAgB,IAAA,gBAAAjC,iBAAA,cAAAC,mBAAA,EAAAC,CAAAA,IAAA,CAAG,SAAAgC,OAAA,GAAA;AAAA,IAAA,IAAAC,KAAA,EAAAC,cAAA,EAAAC,MAAA,EAAAC,MAAA,CAAA;AAAA,IAAA,OAAArC,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAA2B,SAAAC,QAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,QAAA,CAAAzB,IAAA,GAAAyB,QAAA,CAAAxB,IAAA;AAAA,QAAA,KAAA,CAAA;AAAA,UAAA,IAAA,EAE5B,OAAOyB,MAAM,KAAK,WAAW,IAAIA,MAAM,CAACC,QAAQ,CAAA,EAAA;AAAAF,YAAAA,QAAA,CAAAxB,IAAA,GAAA,CAAA,CAAA;AAAA,YAAA,MAAA;AAAA,WAAA;AAC5CmB,UAAAA,KAAK,GAAGM,MAAM,CAACC,QAAQ,EAAE,CAAA;UAAA,OAAAF,QAAA,CAAAT,MAAA,CACxB,QAAA,EAAA;YACLY,QAAQ,EAAER,KAAK,CAACS,eAAe;YAC/BC,QAAQ,EAAEV,KAAK,CAACU,QAAAA;WACjB,CAAA,CAAA;AAAA,QAAA,KAAA,CAAA;AAAAL,UAAAA,QAAA,CAAAzB,IAAA,GAAA,CAAA,CAAA;AAAAyB,UAAAA,QAAA,CAAAxB,IAAA,GAAA,CAAA,CAAA;UAAA,OAIoB,OAAO,QAAQ,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAA/BqB,MAAM,GAAAG,QAAA,CAAAtB,IAAA,CAAA;AAAA,UAAA,IAAA,EACR,OAAOmB,MAAM,KAAK,WAAW,IAAAD,CAAAA,cAAA,GAAIC,MAAM,CAACI,MAAM,KAAbL,IAAAA,IAAAA,cAAA,CAAeM,QAAQ,CAAA,EAAA;AAAAF,YAAAA,QAAA,CAAAxB,IAAA,GAAA,EAAA,CAAA;AAAA,YAAA,MAAA;AAAA,WAAA;AACpDmB,UAAAA,MAAK,GAAGE,MAAM,CAACI,MAAM,CAACC,QAAQ,EAAE,CAAA;UAAA,OAAAF,QAAA,CAAAT,MAAA,CAC/B,QAAA,EAAA;YACLY,QAAQ,EAAER,MAAK,CAACS,eAAe;YAC/BC,QAAQ,EAAEV,MAAK,CAACU,QAAAA;WACjB,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA;AAAAL,UAAAA,QAAA,CAAAxB,IAAA,GAAA,EAAA,CAAA;AAAA,UAAA,MAAA;AAAA,QAAA,KAAA,EAAA;AAAAwB,UAAAA,QAAA,CAAAzB,IAAA,GAAA,EAAA,CAAA;UAAAyB,QAAA,CAAAM,EAAA,GAAAN,QAAA,CAAA,OAAA,CAAA,CAAA,CAAA,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA;AAAA,UAAA,OAAAA,QAAA,CAAAT,MAAA,CAAA,QAAA,EAIE,IAAI,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAS,QAAA,CAAAR,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAAE,OAAA,EAAA,IAAA,EAAA,CAAA,CAAA,CAAA,EAAA,EAAA,CAAA,CAAA,CAAA,CAAA;GACZ,CAAA,CAAA,CAAA;AAAA,EAAA,OAAA,SAtBYjB,kBAAkBA,GAAA;AAAA,IAAA,OAAAgB,IAAA,CAAAnC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,GAAA,CAAA;AAAA,CAsB9B,EAAA;;AC9CD,SAAsBgD,eAAeA,GAAA;AAAA,EAAA,OAAAC,gBAAA,CAAAlD,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAUpC,SAAAiD,gBAAA,GAAA;EAAAA,gBAAA,GAAAhD,iBAAA,cAAAC,mBAAA,GAAAC,IAAA,CAVM,SAAAgC,OAAA,GAAA;AAAA,IAAA,IAAAe,aAAA,CAAA;AAAA,IAAA,OAAAhD,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAA2B,SAAAC,QAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,QAAA,CAAAzB,IAAA,GAAAyB,QAAA,CAAAxB,IAAA;AAAA,QAAA,KAAA,CAAA;UACCiC,aAAa,GAAGC,SAAI,CAAC/B,OAAO,CAACgC,GAAG,EAAE,EAAE,SAAS,CAAC,CAAA;UAEpD,IAAI;YACFC,YAAS,CAACH,aAAa,CAAC,CAAA;AAC1B,WAAC,CAAC,OAAOI,KAAK,EAAE,EAAC;AAEjBC,UAAAA,iBAAY,CAACC,mBAAc,CAACC,aAAa,EAAE,UAAAC,WAAW,EAAG;YACvDC,aAAU,CAACR,SAAI,CAACD,aAAa,EAAE,YAAY,CAAC,EAAEU,IAAI,CAACC,SAAS,CAACH,WAAW,CAAC,GAAG,IAAI,EAAE,YAAK,EAAG,CAAC,CAAA;AAC7F,WAAC,CAAC,CAAA;AAAC,QAAA,KAAA,CAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAjB,QAAA,CAAAR,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAAE,OAAA,CAAA,CAAA;GACJ,CAAA,CAAA,CAAA;AAAA,EAAA,OAAAc,gBAAA,CAAAlD,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAED,SAAsB8D,WAAWA,GAAA;AAAA,EAAA,OAAAC,YAAA,CAAAhE,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAOhC,SAAA+D,YAAA,GAAA;EAAAA,YAAA,GAAA9D,iBAAA,cAAAC,mBAAA,GAAAC,IAAA,CAPM,SAAAC,QAAA,GAAA;AAAA,IAAA,IAAAK,WAAA,CAAA;AAAA,IAAA,OAAAP,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAAC,UAAAC,SAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,SAAA,CAAAC,IAAA,GAAAD,SAAA,CAAAE,IAAA;AAAA,QAAA,KAAA,CAAA;AAAA,UAAA,IAAA,CACDG,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,EAAA;AAAAsB,YAAAA,SAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;AAAA,YAAA,MAAA;AAAA,WAAA;AAAA,UAAA,MAC9B,IAAI+C,KAAK,CAAC,mEAAmE,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;AAGhFvD,UAAAA,WAAW,GAAGa,MAAM,CAACC,UAAU,EAAE,CAAA;AACvCH,UAAAA,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,GAAGgB,WAAW,CAAA;AAAC,QAAA,KAAA,CAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAM,SAAA,CAAAkB,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAA7B,QAAA,CAAA,CAAA;GAClD,CAAA,CAAA,CAAA;AAAA,EAAA,OAAA2D,YAAA,CAAAhE,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA;;;;;;"}

package/dist/evals.cjs.production.min.js

@@ -1,2 +1,2 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var t=require("@mastra/core");function r(t,r,e,n,o,i,a){try{var u=t[i](a),c=u.value}catch(t){return void e(t)}u.done?r(c):Promise.resolve(c).then(n,o)}function e(){e=function(){return r};var t,r={},n=Object.prototype,o=n.hasOwnProperty,i=Object.defineProperty||function(t,r,e){t[r]=e.value},a="function"==typeof Symbol?Symbol:{},u=a.iterator||"@@iterator",c=a.asyncIterator||"@@asyncIterator",s=a.toStringTag||"@@toStringTag";function f(t,r,e){return Object.defineProperty(t,r,{value:e,enumerable:!0,configurable:!0,writable:!0}),t[r]}try{f({},"")}catch(t){f=function(t,r,e){return t[r]=e}}function h(t,r,e,n){var o=Object.create((r&&r.prototype instanceof m?r:m).prototype),a=new S(n||[]);return i(o,"_invoke",{value:k(t,e,a)}),o}function l(t,r,e){try{return{type:"normal",arg:t.call(r,e)}}catch(t){return{type:"throw",arg:t}}}r.wrap=h;var p="suspendedStart",v="suspendedYield",y="executing",d="completed",g={};function m(){}function w(){}function x(){}var b={};f(b,u,(function(){return this}));var L=Object.getPrototypeOf,E=L&&L(L(T([])));E&&E!==n&&o.call(E,u)&&(b=E);var _=x.prototype=m.prototype=Object.create(b);function O(t){["next","throw","return"].forEach((function(r){f(t,r,(function(t){return this._invoke(r,t)}))}))}function j(t,r){function e(n,i,a,u){var c=l(t[n],t,i);if("throw"!==c.type){var s=c.arg,f=s.value;return f&&"object"==typeof f&&o.call(f,"__await")?r.resolve(f.__await).then((function(t){e("next",t,a,u)}),(function(t){e("throw",t,a,u)})):r.resolve(f).then((function(t){s.value=t,a(s)}),(function(t){return e("throw",t,a,u)}))}u(c.arg)}var n;i(this,"_invoke",{value:function(t,o){function i(){return new r((function(r,n){e(t,o,r,n)}))}return n=n?n.then(i,i):i()}})}function k(r,e,n){var o=p;return function(i,a){if(o===y)throw Error("Generator is already running");if(o===d){if("throw"===i)throw a;return{value:t,done:!0}}for(n.method=i,n.arg=a;;){var u=n.delegate;if(u){var c=N(u,n);if(c){if(c===g)continue;return c}}if("next"===n.method)n.sent=n._sent=n.arg;else if("throw"===n.method){if(o===p)throw o=d,n.arg;n.dispatchException(n.arg)}else"return"===n.method&&n.abrupt("return",n.arg);o=y;var s=l(r,e,n);if("normal"===s.type){if(o=n.done?d:v,s.arg===g)continue;return{value:s.arg,done:n.done}}"throw"===s.type&&(o=d,n.method="throw",n.arg=s.arg)}}}function N(r,e){var n=e.method,o=r.iterator[n];if(o===t)return e.delegate=null,"throw"===n&&r.iterator.return&&(e.method="return",e.arg=t,N(r,e),"throw"===e.method)||"return"!==n&&(e.method="throw",e.arg=new TypeError("The iterator does not provide a '"+n+"' method")),g;var i=l(o,r.iterator,e.arg);if("throw"===i.type)return e.method="throw",e.arg=i.arg,e.delegate=null,g;var a=i.arg;return a?a.done?(e[r.resultName]=a.value,e.next=r.nextLoc,"return"!==e.method&&(e.method="next",e.arg=t),e.delegate=null,g):a:(e.method="throw",e.arg=new TypeError("iterator result is not an object"),e.delegate=null,g)}function P(t){var r={tryLoc:t[0]};1 in t&&(r.catchLoc=t[1]),2 in t&&(r.finallyLoc=t[2],r.afterLoc=t[3]),this.tryEntries.push(r)}function G(t){var r=t.completion||{};r.type="normal",delete r.arg,t.completion=r}function S(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(P,this),this.reset(!0)}function T(r){if(r||""===r){var e=r[u];if(e)return e.call(r);if("function"==typeof r.next)return r;if(!isNaN(r.length)){var n=-1,i=function e(){for(;++n<r.length;)if(o.call(r,n))return e.value=r[n],e.done=!1,e;return e.value=t,e.done=!0,e};return i.next=i}}throw new TypeError(typeof r+" is not iterable")}return w.prototype=x,i(_,"constructor",{value:x,configurable:!0}),i(x,"constructor",{value:w,configurable:!0}),w.displayName=f(x,s,"GeneratorFunction"),r.isGeneratorFunction=function(t){var r="function"==typeof t&&t.constructor;return!!r&&(r===w||"GeneratorFunction"===(r.displayName||r.name))},r.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,x):(t.__proto__=x,f(t,s,"GeneratorFunction")),t.prototype=Object.create(_),t},r.awrap=function(t){return{__await:t}},O(j.prototype),f(j.prototype,c,(function(){return this})),r.AsyncIterator=j,r.async=function(t,e,n,o,i){void 0===i&&(i=Promise);var a=new j(h(t,e,n,o),i);return r.isGeneratorFunction(e)?a:a.next().then((function(t){return t.done?t.value:a.next()}))},O(_),f(_,s,"Generator"),f(_,u,(function(){return this})),f(_,"toString",(function(){return"[object Generator]"})),r.keys=function(t){var r=Object(t),e=[];for(var n in r)e.push(n);return e.reverse(),function t(){for(;e.length;){var n=e.pop();if(n in r)return t.value=n,t.done=!1,t}return t.done=!0,t}},r.values=T,S.prototype={constructor:S,reset:function(r){if(this.prev=0,this.next=0,this.sent=this._sent=t,this.done=!1,this.delegate=null,this.method="next",this.arg=t,this.tryEntries.forEach(G),!r)for(var e in this)"t"===e.charAt(0)&&o.call(this,e)&&!isNaN(+e.slice(1))&&(this[e]=t)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if("throw"===t.type)throw t.arg;return this.rval},dispatchException:function(r){if(this.done)throw r;var e=this;function n(n,o){return u.type="throw",u.arg=r,e.next=n,o&&(e.method="next",e.arg=t),!!o}for(var i=this.tryEntries.length-1;i>=0;--i){var a=this.tryEntries[i],u=a.completion;if("root"===a.tryLoc)return n("end");if(a.tryLoc<=this.prev){var c=o.call(a,"catchLoc"),s=o.call(a,"finallyLoc");if(c&&s){if(this.prev<a.catchLoc)return n(a.catchLoc,!0);if(this.prev<a.finallyLoc)return n(a.finallyLoc)}else if(c){if(this.prev<a.catchLoc)return n(a.catchLoc,!0)}else{if(!s)throw Error("try statement without catch or finally");if(this.prev<a.finallyLoc)return n(a.finallyLoc)}}}},abrupt:function(t,r){for(var e=this.tryEntries.length-1;e>=0;--e){var n=this.tryEntries[e];if(n.tryLoc<=this.prev&&o.call(n,"finallyLoc")&&this.prev<n.finallyLoc){var i=n;break}}i&&("break"===t||"continue"===t)&&i.tryLoc<=r&&r<=i.finallyLoc&&(i=null);var a=i?i.completion:{};return a.type=t,a.arg=r,i?(this.method="next",this.next=i.finallyLoc,g):this.complete(a)},complete:function(t,r){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&r&&(this.next=r),g},finish:function(t){for(var r=this.tryEntries.length-1;r>=0;--r){var e=this.tryEntries[r];if(e.finallyLoc===t)return this.complete(e.completion,e.afterLoc),G(e),g}},catch:function(t){for(var r=this.tryEntries.length-1;r>=0;--r){var e=this.tryEntries[r];if(e.tryLoc===t){var n=e.completion;if("throw"===n.type){var o=n.arg;G(e)}return o}}throw Error("illegal catch attempt")},delegateYield:function(r,e,n){return this.delegate={iterator:T(r),resultName:e,nextLoc:n},"next"===this.method&&(this.arg=t),g}},r}function n(){var o;return o=e().mark((function r(n,o,i){var a,u;return e().wrap((function(r){for(;;)switch(r.prev=r.next){case 0:return r.next=2,n.generate(o);case 2:return a=r.sent,r.next=5,i.measure({input:o.toString(),output:a.text});case 5:return u=r.sent,t.executeHook(t.AvailableHooks.ON_EVALUATION,{input:o.toString(),output:a.text,result:u}),r.abrupt("return",u);case 8:case"end":return r.stop()}}),r)})),n=function(){var t=this,e=arguments;return new Promise((function(n,i){var a=o.apply(t,e);function u(t){r(a,n,i,u,c,"next",t)}function c(t){r(a,n,i,u,c,"throw",t)}u(void 0)}))},n.apply(this,arguments)}exports.evaluate=function(t,r,e){return n.apply(this,arguments)};
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var t=require("@mastra/core"),e=require("fs"),r=require("path");function n(t,e,r,n,o,a,i){try{var c=t[a](i),u=c.value}catch(t){return void r(t)}c.done?e(u):Promise.resolve(u).then(n,o)}function o(t){return function(){var e=this,r=arguments;return new Promise((function(o,a){var i=t.apply(e,r);function c(t){n(i,o,a,c,u,"next",t)}function u(t){n(i,o,a,c,u,"throw",t)}c(void 0)}))}}function a(){a=function(){return e};var t,e={},r=Object.prototype,n=r.hasOwnProperty,o=Object.defineProperty||function(t,e,r){t[e]=r.value},i="function"==typeof Symbol?Symbol:{},c=i.iterator||"@@iterator",u=i.asyncIterator||"@@asyncIterator",s=i.toStringTag||"@@toStringTag";function l(t,e,r){return Object.defineProperty(t,e,{value:r,enumerable:!0,configurable:!0,writable:!0}),t[e]}try{l({},"")}catch(t){l=function(t,e,r){return t[e]=r}}function f(t,e,r,n){var a=Object.create((e&&e.prototype instanceof m?e:m).prototype),i=new G(n||[]);return o(a,"_invoke",{value:j(t,r,i)}),a}function h(t,e,r){try{return{type:"normal",arg:t.call(e,r)}}catch(t){return{type:"throw",arg:t}}}e.wrap=f;var p="suspendedStart",v="suspendedYield",y="executing",d="completed",g={};function m(){}function w(){}function x(){}var b={};l(b,c,(function(){return this}));var L=Object.getPrototypeOf,E=L&&L(L(I([])));E&&E!==r&&n.call(E,c)&&(b=E);var _=x.prototype=m.prototype=Object.create(b);function k(t){["next","throw","return"].forEach((function(e){l(t,e,(function(t){return this._invoke(e,t)}))}))}function O(t,e){function r(o,a,i,c){var u=h(t[o],t,a);if("throw"!==u.type){var s=u.arg,l=s.value;return l&&"object"==typeof l&&n.call(l,"__await")?e.resolve(l.__await).then((function(t){r("next",t,i,c)}),(function(t){r("throw",t,i,c)})):e.resolve(l).then((function(t){s.value=t,i(s)}),(function(t){return r("throw",t,i,c)}))}c(u.arg)}var a;o(this,"_invoke",{value:function(t,n){function o(){return new e((function(e,o){r(t,n,e,o)}))}return a=a?a.then(o,o):o()}})}function j(e,r,n){var o=p;return function(a,i){if(o===y)throw Error("Generator is already running");if(o===d){if("throw"===a)throw i;return{value:t,done:!0}}for(n.method=a,n.arg=i;;){var c=n.delegate;if(c){var u=N(c,n);if(u){if(u===g)continue;return u}}if("next"===n.method)n.sent=n._sent=n.arg;else if("throw"===n.method){if(o===p)throw o=d,n.arg;n.dispatchException(n.arg)}else"return"===n.method&&n.abrupt("return",n.arg);o=y;var s=h(e,r,n);if("normal"===s.type){if(o=n.done?d:v,s.arg===g)continue;return{value:s.arg,done:n.done}}"throw"===s.type&&(o=d,n.method="throw",n.arg=s.arg)}}}function N(e,r){var n=r.method,o=e.iterator[n];if(o===t)return r.delegate=null,"throw"===n&&e.iterator.return&&(r.method="return",r.arg=t,N(e,r),"throw"===r.method)||"return"!==n&&(r.method="throw",r.arg=new TypeError("The iterator does not provide a '"+n+"' method")),g;var a=h(o,e.iterator,r.arg);if("throw"===a.type)return r.method="throw",r.arg=a.arg,r.delegate=null,g;var i=a.arg;return i?i.done?(r[e.resultName]=i.value,r.next=e.nextLoc,"return"!==r.method&&(r.method="next",r.arg=t),r.delegate=null,g):i:(r.method="throw",r.arg=new TypeError("iterator result is not an object"),r.delegate=null,g)}function S(t){var e={tryLoc:t[0]};1 in t&&(e.catchLoc=t[1]),2 in t&&(e.finallyLoc=t[2],e.afterLoc=t[3]),this.tryEntries.push(e)}function P(t){var e=t.completion||{};e.type="normal",delete e.arg,t.completion=e}function G(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(S,this),this.reset(!0)}function I(e){if(e||""===e){var r=e[c];if(r)return r.call(e);if("function"==typeof e.next)return e;if(!isNaN(e.length)){var o=-1,a=function r(){for(;++o<e.length;)if(n.call(e,o))return r.value=e[o],r.done=!1,r;return r.value=t,r.done=!0,r};return a.next=a}}throw new TypeError(typeof e+" is not iterable")}return w.prototype=x,o(_,"constructor",{value:x,configurable:!0}),o(x,"constructor",{value:w,configurable:!0}),w.displayName=l(x,s,"GeneratorFunction"),e.isGeneratorFunction=function(t){var e="function"==typeof t&&t.constructor;return!!e&&(e===w||"GeneratorFunction"===(e.displayName||e.name))},e.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,x):(t.__proto__=x,l(t,s,"GeneratorFunction")),t.prototype=Object.create(_),t},e.awrap=function(t){return{__await:t}},k(O.prototype),l(O.prototype,u,(function(){return this})),e.AsyncIterator=O,e.async=function(t,r,n,o,a){void 0===a&&(a=Promise);var i=new O(f(t,r,n,o),a);return e.isGeneratorFunction(r)?i:i.next().then((function(t){return t.done?t.value:i.next()}))},k(_),l(_,s,"Generator"),l(_,c,(function(){return this})),l(_,"toString",(function(){return"[object Generator]"})),e.keys=function(t){var e=Object(t),r=[];for(var n in e)r.push(n);return r.reverse(),function t(){for(;r.length;){var n=r.pop();if(n in e)return t.value=n,t.done=!1,t}return t.done=!0,t}},e.values=I,G.prototype={constructor:G,reset:function(e){if(this.prev=0,this.next=0,this.sent=this._sent=t,this.done=!1,this.delegate=null,this.method="next",this.arg=t,this.tryEntries.forEach(P),!e)for(var r in this)"t"===r.charAt(0)&&n.call(this,r)&&!isNaN(+r.slice(1))&&(this[r]=t)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if("throw"===t.type)throw t.arg;return this.rval},dispatchException:function(e){if(this.done)throw e;var r=this;function o(n,o){return c.type="throw",c.arg=e,r.next=n,o&&(r.method="next",r.arg=t),!!o}for(var a=this.tryEntries.length-1;a>=0;--a){var i=this.tryEntries[a],c=i.completion;if("root"===i.tryLoc)return o("end");if(i.tryLoc<=this.prev){var u=n.call(i,"catchLoc"),s=n.call(i,"finallyLoc");if(u&&s){if(this.prev<i.catchLoc)return o(i.catchLoc,!0);if(this.prev<i.finallyLoc)return o(i.finallyLoc)}else if(u){if(this.prev<i.catchLoc)return o(i.catchLoc,!0)}else{if(!s)throw Error("try statement without catch or finally");if(this.prev<i.finallyLoc)return o(i.finallyLoc)}}}},abrupt:function(t,e){for(var r=this.tryEntries.length-1;r>=0;--r){var o=this.tryEntries[r];if(o.tryLoc<=this.prev&&n.call(o,"finallyLoc")&&this.prev<o.finallyLoc){var a=o;break}}a&&("break"===t||"continue"===t)&&a.tryLoc<=e&&e<=a.finallyLoc&&(a=null);var i=a?a.completion:{};return i.type=t,i.arg=e,a?(this.method="next",this.next=a.finallyLoc,g):this.complete(i)},complete:function(t,e){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&e&&(this.next=e),g},finish:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var r=this.tryEntries[e];if(r.finallyLoc===t)return this.complete(r.completion,r.afterLoc),P(r),g}},catch:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var r=this.tryEntries[e];if(r.tryLoc===t){var n=r.completion;if("throw"===n.type){var o=n.arg;P(r)}return o}}throw Error("illegal catch attempt")},delegateYield:function(e,r,n){return this.delegate={iterator:I(e),resultName:r,nextLoc:n},"next"===this.method&&(this.arg=t),g}},e}var i="_MASTRA_GLOBAL_RUN_ID_";function c(){return(c=o(a().mark((function e(r,n,o){var c,s,l,f;return a().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=2,u();case 2:return c=e.sent,s=process.env[i],l=crypto.randomUUID(),e.next=7,r.generate(n,{runId:l});case 7:return f=e.sent,s||(s=process.env[i]=crypto.randomUUID(),console.warn('Global run id not set, you should run "globalSetup" from "@mastra/evals" before evaluating.')),e.next=11,t.evaluate({agentName:r.name,input:n,metric:o,output:f.text,globalRunId:s,runId:l,testInfo:c});case 11:return e.abrupt("return",e.sent);case 13:case"end":return e.stop()}}),e)})))).apply(this,arguments)}var u=function(){var t=o(a().mark((function t(){var e,r,n,o;return a().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if("undefined"==typeof expect||!expect.getState){t.next=3;break}return e=expect.getState(),t.abrupt("return",{testName:e.currentTestName,testPath:e.testPath});case 3:return t.prev=3,t.next=6,import("vitest");case 6:if(void 0===(n=t.sent)||null==(r=n.expect)||!r.getState){t.next=10;break}return o=n.expect.getState(),t.abrupt("return",{testName:o.currentTestName,testPath:o.testPath});case 10:t.next=14;break;case 12:t.prev=12,t.t0=t.catch(3);case 14:return t.abrupt("return",null);case 15:case"end":return t.stop()}}),t,null,[[3,12]])})));return function(){return t.apply(this,arguments)}}();function s(){return(s=o(a().mark((function n(){var o;return a().wrap((function(n){for(;;)switch(n.prev=n.next){case 0:o=r.join(process.cwd(),".mastra");try{e.mkdirSync(o)}catch(t){}t.registerHook(t.AvailableHooks.ON_EVALUATION,(function(t){e.appendFile(r.join(o,"evals.json"),JSON.stringify(t)+"\n",(function(){}))}));case 3:case"end":return n.stop()}}),n)})))).apply(this,arguments)}function l(){return(l=o(a().mark((function t(){var e;return a().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(!process.env[i]){t.next=2;break}throw new Error('Global run id already set, you should only run "GlobalSetup" once');case 2:e=crypto.randomUUID(),process.env[i]=e;case 4:case"end":return t.stop()}}),t)})))).apply(this,arguments)}exports.attachListeners=function(){return s.apply(this,arguments)},exports.evaluate=function(t,e,r){return c.apply(this,arguments)},exports.globalSetup=function(){return l.apply(this,arguments)};
 //# sourceMappingURL=evals.cjs.production.min.js.map

package/dist/evals.cjs.production.min.js.map

@@ -1 +1 @@
-{"version":3,"file":"evals.cjs.production.min.js","sources":["../src/evaluation.ts"],"sourcesContent":["import { AvailableHooks, executeHook } from '@mastra/core';\nimport { type Agent, type Metric } from '@mastra/core';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const agentOutput = await agent.generate(input);\n\n  const metricResult = await metric.measure({\n    input: input.toString(),\n    output: agentOutput.text,\n  });\n\n  // capture infomration about the evaluation\n  executeHook(AvailableHooks.ON_EVALUATION, {\n    input: input.toString(),\n    output: agentOutput.text,\n    result: metricResult,\n  });\n\n  return metricResult;\n}\n"],"names":["_evaluate","_regeneratorRuntime","mark","_callee","agent","input","metric","agentOutput","metricResult","wrap","_context","prev","next","generate","sent","measure","toString","output","text","executeHook","AvailableHooks","ON_EVALUATION","result","abrupt","stop","apply","this","arguments","_x","_x2","_x3"],"mappings":"+jNAmBC,SAAAA,UAAA,SAAAC,IAAAC,MAhBM,SAAAC,EAAyCC,EAAUC,EAAqCC,GAAc,IAAAC,EAAAC,EAAA,OAAAP,IAAAQ,MAAA,SAAAC,GAAA,cAAAA,EAAAC,KAAAD,EAAAE,MAAA,KAAA,EAAA,OAAAF,EAAAE,KAAA,EACjFR,EAAMS,SAASR,GAAM,KAAA,EAA9B,OAAXE,EAAWG,EAAAI,KAAAJ,EAAAE,KAAA,EAEUN,EAAOS,QAAQ,CACxCV,MAAOA,EAAMW,WACbC,OAAQV,EAAYW,OACpB,KAAA,EAOC,OAVGV,EAAYE,EAAAI,KAMlBK,EAAWA,YAACC,EAAcA,eAACC,cAAe,CACxChB,MAAOA,EAAMW,WACbC,OAAQV,EAAYW,KACpBI,OAAQd,IACPE,EAAAa,OAAA,SAEIf,GAAY,KAAA,EAAA,IAAA,MAAA,OAAAE,EAAAc,OAAA,GAAArB,EACpB,IAAAH,gLAAAA,EAAAyB,MAAAC,KAAAC,UAAA,kBAhBD,SAA8BC,EAAAC,EAAAC,GAAA,OAAA9B,EAAAyB,MAAAC,KAAAC,UAAA"}
+{"version":3,"file":"evals.cjs.production.min.js","sources":["../src/constants.ts","../src/evaluation.ts","../src/attachListeners.ts"],"sourcesContent":["export const GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';\n","import { type Agent, type Metric, evaluate as coreEvaluate } from '@mastra/core';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const testInfo = await getCurrentTestInfo();\n  let globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];\n  const runId = crypto.randomUUID();\n  const agentOutput = await agent.generate(input, {\n    runId,\n  });\n\n  if (!globalRunId) {\n    globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();\n    console.warn('Global run id not set, you should run \"globalSetup\" from \"@mastra/evals\" before evaluating.');\n  }\n\n  const metricResult = await coreEvaluate({\n    agentName: agent.name,\n    input,\n    metric,\n    output: agentOutput.text,\n    globalRunId,\n    runId,\n    testInfo,\n  });\n\n  return metricResult;\n}\n\nexport const getCurrentTestInfo = async () => {\n  // Jest\n  if (typeof expect !== 'undefined' && expect.getState) {\n    const state = expect.getState();\n    return {\n      testName: state.currentTestName,\n      testPath: state.testPath,\n    };\n  }\n\n  try {\n    const vitest = await import('vitest');\n    if (typeof vitest !== 'undefined' && vitest.expect?.getState) {\n      const state = vitest.expect.getState();\n      return {\n        testName: state.currentTestName,\n        testPath: state.testPath,\n      };\n    }\n  } catch {}\n\n  return null;\n};\n","import { AvailableHooks, registerHook } from '@mastra/core';\nimport { mkdirSync, appendFile } from 'fs';\nimport { join } from 'path';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function attachListeners() {\n  const dotMastraPath = join(process.cwd(), '.mastra');\n\n  try {\n    mkdirSync(dotMastraPath);\n  } catch (error) {}\n\n  registerHook(AvailableHooks.ON_EVALUATION, traceObject => {\n    appendFile(join(dotMastraPath, 'evals.json'), JSON.stringify(traceObject) + '\\n', () => {});\n  });\n}\n\nexport async function globalSetup() {\n  if (process.env[GLOBAL_RUN_ID_ENV_KEY]) {\n    throw new Error('Global run id already set, you should only run \"GlobalSetup\" once');\n  }\n\n  const globalRunId = crypto.randomUUID();\n  process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;\n}\n"],"names":["GLOBAL_RUN_ID_ENV_KEY","_evaluate","_asyncToGenerator","_regeneratorRuntime","mark","_callee2","agent","input","metric","testInfo","globalRunId","runId","agentOutput","wrap","_context2","prev","next","getCurrentTestInfo","sent","process","env","crypto","randomUUID","generate","console","warn","coreEvaluate","agentName","name","output","text","abrupt","stop","apply","this","arguments","_ref","_callee","state","_vitest$expect","vitest","_state","_context","expect","getState","testName","currentTestName","testPath","import","t0","_attachListeners","dotMastraPath","join","cwd","mkdirSync","error","registerHook","AvailableHooks","ON_EVALUATION","traceObject","appendFile","JSON","stringify","_globalSetup","Error","_x","_x2","_x3"],"mappings":"oyNAAO,IAAMA,EAAwB,yBC4BpC,SAAAC,IAAA,OAAAA,EAAAC,EAAAC,IAAAC,MAxBM,SAAAC,EAAyCC,EAAUC,EAAqCC,GAAc,IAAAC,EAAAC,EAAAC,EAAAC,EAAA,OAAAT,IAAAU,MAAA,SAAAC,GAAA,cAAAA,EAAAC,KAAAD,EAAAE,MAAA,KAAA,EAAA,OAAAF,EAAAE,KAAA,EACpFC,IAAoB,KAAA,EAEV,OAF3BR,EAAQK,EAAAI,KACVR,EAAcS,QAAQC,IAAIpB,GACxBW,EAAQU,OAAOC,aAAYR,EAAAE,KAAA,EACPV,EAAMiB,SAAShB,EAAO,CAC9CI,MAAAA,IACA,KAAA,EAKD,OAPKC,EAAWE,EAAAI,KAIZR,IACHA,EAAcS,QAAQC,IAAIpB,GAAyBqB,OAAOC,aAC1DE,QAAQC,KAAK,gGACdX,EAAAE,KAAA,GAE0BU,WAAa,CACtCC,UAAWrB,EAAMsB,KACjBrB,MAAAA,EACAC,OAAAA,EACAqB,OAAQjB,EAAYkB,KACpBpB,YAAAA,EACAC,MAAAA,EACAF,SAAAA,IACA,KAAA,GARgB,OAAAK,EAAAiB,OAAA,SAAAjB,EAAAI,MAUC,KAAA,GAAA,IAAA,MAAA,OAAAJ,EAAAkB,OAAA,GAAA3B,EACpB,MAAA4B,MAAAC,KAAAC,UAAA,CAEM,IAAMlB,EAAkB,WAAA,IAAAmB,EAAAlC,EAAAC,IAAAC,MAAG,SAAAiC,IAAA,IAAAC,EAAAC,EAAAC,EAAAC,EAAA,OAAAtC,IAAAU,MAAA,SAAA6B,GAAA,cAAAA,EAAA3B,KAAA2B,EAAA1B,MAAA,KAAA,EAAA,GAEV,oBAAX2B,SAA0BA,OAAOC,SAAQ,CAAAF,EAAA1B,KAAA,EAAA,KAAA,CACnB,OAAzBsB,EAAQK,OAAOC,WAAUF,EAAAX,OACxB,SAAA,CACLc,SAAUP,EAAMQ,gBAChBC,SAAUT,EAAMS,WACjB,KAAA,EAAA,OAAAL,EAAA3B,KAAA,EAAA2B,EAAA1B,KAAA,EAIoBgC,OAAO,UAAS,KAAA,EAAzB,QACU,KADhBR,EAAME,EAAAxB,OACyBqB,OAAJA,EAAIC,EAAOG,UAAPJ,EAAeK,SAAQ,CAAAF,EAAA1B,KAAA,GAAA,KAAA,CACpB,OAAhCsB,EAAQE,EAAOG,OAAOC,WAAUF,EAAAX,OAC/B,SAAA,CACLc,SAAUP,EAAMQ,gBAChBC,SAAUT,EAAMS,WACjB,KAAA,GAAAL,EAAA1B,KAAA,GAAA,MAAA,KAAA,GAAA0B,EAAA3B,KAAA,GAAA2B,EAAAO,GAAAP,EAAA,MAAA,GAAA,KAAA,GAAA,OAAAA,EAAAX,OAAA,SAIE,MAAI,KAAA,GAAA,IAAA,MAAA,OAAAW,EAAAV,OAAA,GAAAK,EAAA,KAAA,CAAA,CAAA,EAAA,KACZ,KAAA,OAAA,WAtB8B,OAAAD,EAAAH,MAAAC,KAAAC,UAAA,CAAA,CAAA,GCd9B,SAAAe,IAAA,OAAAA,EAAAhD,EAAAC,IAAAC,MAVM,SAAAiC,IAAA,IAAAc,EAAA,OAAAhD,IAAAU,MAAA,SAAA6B,GAAA,cAAAA,EAAA3B,KAAA2B,EAAA1B,MAAA,KAAA,EACCmC,EAAgBC,EAAAA,KAAKjC,QAAQkC,MAAO,WAE1C,IACEC,EAASA,UAACH,EACZ,CAAE,MAAOI,GAAQ,CAEjBC,EAAAA,aAAaC,EAAcA,eAACC,eAAe,SAAAC,GACzCC,EAAAA,WAAWR,EAAIA,KAACD,EAAe,cAAeU,KAAKC,UAAUH,GAAe,MAAM,WAAQ,GAC5F,IAAG,KAAA,EAAA,IAAA,MAAA,OAAAjB,EAAAV,OAAA,GAAAK,EACJ,MAAAJ,MAAAC,KAAAC,UAAA,CASA,SAAA4B,IAAA,OAAAA,EAAA7D,EAAAC,IAAAC,MAPM,SAAAC,IAAA,IAAAK,EAAA,OAAAP,IAAAU,MAAA,SAAAC,GAAA,cAAAA,EAAAC,KAAAD,EAAAE,MAAA,KAAA,EAAA,IACDG,QAAQC,IAAIpB,GAAsB,CAAAc,EAAAE,KAAA,EAAA,KAAA,CAAA,MAC9B,IAAIgD,MAAM,qEAAoE,KAAA,EAGhFtD,EAAcW,OAAOC,aAC3BH,QAAQC,IAAIpB,GAAyBU,EAAY,KAAA,EAAA,IAAA,MAAA,OAAAI,EAAAkB,OAAA,GAAA3B,EAClD,MAAA4B,MAAAC,KAAAC,UAAA,yBAnBD,WAAqC,OAAAe,EAAAjB,MAAAC,KAAAC,UAAA,mBDFrC,SAA8B8B,EAAAC,EAAAC,GAAA,OAAAlE,EAAAgC,MAAAC,KAAAC,UAAA,sBCc9B,WAAiC,OAAA4B,EAAA9B,MAAAC,KAAAC,UAAA"}

package/dist/evals.esm.js

@@ -1,4 +1,6 @@
-import { executeHook, AvailableHooks } from '@mastra/core';
+import { evaluate as evaluate$1, registerHook, AvailableHooks } from '@mastra/core';
+import { mkdirSync, appendFile } from 'fs';
+import { join } from 'path';
 
 function asyncGeneratorStep(n, t, e, r, o, a, c) {
   try {
@@ -327,41 +329,152 @@ function _regeneratorRuntime() {
   }, e;
 }
 
+var GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';
+
 function evaluate(_x, _x2, _x3) {
   return _evaluate.apply(this, arguments);
 }
 function _evaluate() {
-  _evaluate = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(agent, input, metric) {
-    var agentOutput, metricResult;
-    return _regeneratorRuntime().wrap(function _callee$(_context) {
-      while (1) switch (_context.prev = _context.next) {
+  _evaluate = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(agent, input, metric) {
+    var testInfo, globalRunId, runId, agentOutput, metricResult;
+    return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+      while (1) switch (_context2.prev = _context2.next) {
         case 0:
-          _context.next = 2;
-          return agent.generate(input);
+          _context2.next = 2;
+          return getCurrentTestInfo();
         case 2:
-          agentOutput = _context.sent;
-          _context.next = 5;
-          return metric.measure({
-            input: input.toString(),
-            output: agentOutput.text
+          testInfo = _context2.sent;
+          globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];
+          runId = crypto.randomUUID();
+          _context2.next = 7;
+          return agent.generate(input, {
+            runId: runId
           });
-        case 5:
-          metricResult = _context.sent;
-          // capture infomration about the evaluation
-          executeHook(AvailableHooks.ON_EVALUATION, {
-            input: input.toString(),
+        case 7:
+          agentOutput = _context2.sent;
+          if (!globalRunId) {
+            globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();
+            console.warn('Global run id not set, you should run "globalSetup" from "@mastra/evals" before evaluating.');
+          }
+          _context2.next = 11;
+          return evaluate$1({
+            agentName: agent.name,
+            input: input,
+            metric: metric,
             output: agentOutput.text,
-            result: metricResult
+            globalRunId: globalRunId,
+            runId: runId,
+            testInfo: testInfo
           });
-          return _context.abrupt("return", metricResult);
-        case 8:
+        case 11:
+          metricResult = _context2.sent;
+          return _context2.abrupt("return", metricResult);
+        case 13:
+        case "end":
+          return _context2.stop();
+      }
+    }, _callee2);
+  }));
+  return _evaluate.apply(this, arguments);
+}
+var getCurrentTestInfo = /*#__PURE__*/function () {
+  var _ref = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee() {
+    var state, _vitest$expect, vitest, _state;
+    return _regeneratorRuntime().wrap(function _callee$(_context) {
+      while (1) switch (_context.prev = _context.next) {
+        case 0:
+          if (!(typeof expect !== 'undefined' && expect.getState)) {
+            _context.next = 3;
+            break;
+          }
+          state = expect.getState();
+          return _context.abrupt("return", {
+            testName: state.currentTestName,
+            testPath: state.testPath
+          });
+        case 3:
+          _context.prev = 3;
+          _context.next = 6;
+          return import('vitest');
+        case 6:
+          vitest = _context.sent;
+          if (!(typeof vitest !== 'undefined' && (_vitest$expect = vitest.expect) != null && _vitest$expect.getState)) {
+            _context.next = 10;
+            break;
+          }
+          _state = vitest.expect.getState();
+          return _context.abrupt("return", {
+            testName: _state.currentTestName,
+            testPath: _state.testPath
+          });
+        case 10:
+          _context.next = 14;
+          break;
+        case 12:
+          _context.prev = 12;
+          _context.t0 = _context["catch"](3);
+        case 14:
+          return _context.abrupt("return", null);
+        case 15:
+        case "end":
+          return _context.stop();
+      }
+    }, _callee, null, [[3, 12]]);
+  }));
+  return function getCurrentTestInfo() {
+    return _ref.apply(this, arguments);
+  };
+}();
+
+function attachListeners() {
+  return _attachListeners.apply(this, arguments);
+}
+function _attachListeners() {
+  _attachListeners = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee() {
+    var dotMastraPath;
+    return _regeneratorRuntime().wrap(function _callee$(_context) {
+      while (1) switch (_context.prev = _context.next) {
+        case 0:
+          dotMastraPath = join(process.cwd(), '.mastra');
+          try {
+            mkdirSync(dotMastraPath);
+          } catch (error) {}
+          registerHook(AvailableHooks.ON_EVALUATION, function (traceObject) {
+            appendFile(join(dotMastraPath, 'evals.json'), JSON.stringify(traceObject) + '\n', function () {});
+          });
+        case 3:
         case "end":
           return _context.stop();
       }
     }, _callee);
   }));
-  return _evaluate.apply(this, arguments);
+  return _attachListeners.apply(this, arguments);
+}
+function globalSetup() {
+  return _globalSetup.apply(this, arguments);
+}
+function _globalSetup() {
+  _globalSetup = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2() {
+    var globalRunId;
+    return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+      while (1) switch (_context2.prev = _context2.next) {
+        case 0:
+          if (!process.env[GLOBAL_RUN_ID_ENV_KEY]) {
+            _context2.next = 2;
+            break;
+          }
+          throw new Error('Global run id already set, you should only run "GlobalSetup" once');
+        case 2:
+          globalRunId = crypto.randomUUID();
+          process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;
+        case 4:
+        case "end":
+          return _context2.stop();
+      }
+    }, _callee2);
+  }));
+  return _globalSetup.apply(this, arguments);
 }
 
-export { evaluate };
+export { attachListeners, evaluate, globalSetup };
 //# sourceMappingURL=evals.esm.js.map

package/dist/evals.esm.js.map

@@ -1 +1 @@
-{"version":3,"file":"evals.esm.js","sources":["../src/evaluation.ts"],"sourcesContent":["import { AvailableHooks, executeHook } from '@mastra/core';\nimport { type Agent, type Metric } from '@mastra/core';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const agentOutput = await agent.generate(input);\n\n  const metricResult = await metric.measure({\n    input: input.toString(),\n    output: agentOutput.text,\n  });\n\n  // capture infomration about the evaluation\n  executeHook(AvailableHooks.ON_EVALUATION, {\n    input: input.toString(),\n    output: agentOutput.text,\n    result: metricResult,\n  });\n\n  return metricResult;\n}\n"],"names":["evaluate","_x","_x2","_x3","_evaluate","apply","arguments","_asyncToGenerator","_regeneratorRuntime","mark","_callee","agent","input","metric","agentOutput","metricResult","wrap","_callee$","_context","prev","next","generate","sent","measure","toString","output","text","executeHook","AvailableHooks","ON_EVALUATION","result","abrupt","stop"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,SAAsBA,QAAQA,CAAAC,EAAA,EAAAC,GAAA,EAAAC,GAAA,EAAA;AAAA,EAAA,OAAAC,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAgB7B,SAAAF,SAAA,GAAA;AAAAA,EAAAA,SAAA,GAAAG,iBAAA,cAAAC,mBAAA,EAAAC,CAAAA,IAAA,CAhBM,SAAAC,OAAyCC,CAAAA,KAAQ,EAAEC,KAAmC,EAAEC,MAAc,EAAA;IAAA,IAAAC,WAAA,EAAAC,YAAA,CAAA;AAAA,IAAA,OAAAP,mBAAA,EAAA,CAAAQ,IAAA,CAAA,SAAAC,SAAAC,QAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;AAAA,QAAA,KAAA,CAAA;AAAAF,UAAAA,QAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;AAAA,UAAA,OACjFT,KAAK,CAACU,QAAQ,CAACT,KAAK,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAAzCE,WAAW,GAAAI,QAAA,CAAAI,IAAA,CAAA;AAAAJ,UAAAA,QAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;UAAA,OAEUP,MAAM,CAACU,OAAO,CAAC;AACxCX,YAAAA,KAAK,EAAEA,KAAK,CAACY,QAAQ,EAAE;YACvBC,MAAM,EAAEX,WAAW,CAACY,IAAAA;AACrB,WAAA,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAHIX,YAAY,GAAAG,QAAA,CAAAI,IAAA,CAAA;AAKlB;AACAK,UAAAA,WAAW,CAACC,cAAc,CAACC,aAAa,EAAE;AACxCjB,YAAAA,KAAK,EAAEA,KAAK,CAACY,QAAQ,EAAE;YACvBC,MAAM,EAAEX,WAAW,CAACY,IAAI;AACxBI,YAAAA,MAAM,EAAEf,YAAAA;AACT,WAAA,CAAC,CAAA;AAAC,UAAA,OAAAG,QAAA,CAAAa,MAAA,CAAA,QAAA,EAEIhB,YAAY,CAAA,CAAA;AAAA,QAAA,KAAA,CAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAG,QAAA,CAAAc,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAAtB,OAAA,CAAA,CAAA;GACpB,CAAA,CAAA,CAAA;AAAA,EAAA,OAAAN,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA;;;;"}
+{"version":3,"file":"evals.esm.js","sources":["../src/constants.ts","../src/evaluation.ts","../src/attachListeners.ts"],"sourcesContent":["export const GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';\n","import { type Agent, type Metric, evaluate as coreEvaluate } from '@mastra/core';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const testInfo = await getCurrentTestInfo();\n  let globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];\n  const runId = crypto.randomUUID();\n  const agentOutput = await agent.generate(input, {\n    runId,\n  });\n\n  if (!globalRunId) {\n    globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();\n    console.warn('Global run id not set, you should run \"globalSetup\" from \"@mastra/evals\" before evaluating.');\n  }\n\n  const metricResult = await coreEvaluate({\n    agentName: agent.name,\n    input,\n    metric,\n    output: agentOutput.text,\n    globalRunId,\n    runId,\n    testInfo,\n  });\n\n  return metricResult;\n}\n\nexport const getCurrentTestInfo = async () => {\n  // Jest\n  if (typeof expect !== 'undefined' && expect.getState) {\n    const state = expect.getState();\n    return {\n      testName: state.currentTestName,\n      testPath: state.testPath,\n    };\n  }\n\n  try {\n    const vitest = await import('vitest');\n    if (typeof vitest !== 'undefined' && vitest.expect?.getState) {\n      const state = vitest.expect.getState();\n      return {\n        testName: state.currentTestName,\n        testPath: state.testPath,\n      };\n    }\n  } catch {}\n\n  return null;\n};\n","import { AvailableHooks, registerHook } from '@mastra/core';\nimport { mkdirSync, appendFile } from 'fs';\nimport { join } from 'path';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function attachListeners() {\n  const dotMastraPath = join(process.cwd(), '.mastra');\n\n  try {\n    mkdirSync(dotMastraPath);\n  } catch (error) {}\n\n  registerHook(AvailableHooks.ON_EVALUATION, traceObject => {\n    appendFile(join(dotMastraPath, 'evals.json'), JSON.stringify(traceObject) + '\\n', () => {});\n  });\n}\n\nexport async function globalSetup() {\n  if (process.env[GLOBAL_RUN_ID_ENV_KEY]) {\n    throw new Error('Global run id already set, you should only run \"GlobalSetup\" once');\n  }\n\n  const globalRunId = crypto.randomUUID();\n  process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;\n}\n"],"names":["GLOBAL_RUN_ID_ENV_KEY","evaluate","_x","_x2","_x3","_evaluate","apply","arguments","_asyncToGenerator","_regeneratorRuntime","mark","_callee2","agent","input","metric","testInfo","globalRunId","runId","agentOutput","metricResult","wrap","_callee2$","_context2","prev","next","getCurrentTestInfo","sent","process","env","crypto","randomUUID","generate","console","warn","coreEvaluate","agentName","name","output","text","abrupt","stop","_ref","_callee","state","_vitest$expect","vitest","_state","_callee$","_context","expect","getState","testName","currentTestName","testPath","t0","attachListeners","_attachListeners","dotMastraPath","join","cwd","mkdirSync","error","registerHook","AvailableHooks","ON_EVALUATION","traceObject","appendFile","JSON","stringify","globalSetup","_globalSetup","Error"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAO,IAAMA,qBAAqB,GAAG,wBAAwB;;ACI7D,SAAsBC,QAAQA,CAAAC,EAAA,EAAAC,GAAA,EAAAC,GAAA,EAAA;AAAA,EAAA,OAAAC,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAwB7B,SAAAF,SAAA,GAAA;AAAAA,EAAAA,SAAA,GAAAG,iBAAA,cAAAC,mBAAA,EAAAC,CAAAA,IAAA,CAxBM,SAAAC,QAAyCC,CAAAA,KAAQ,EAAEC,KAAmC,EAAEC,MAAc,EAAA;IAAA,IAAAC,QAAA,EAAAC,WAAA,EAAAC,KAAA,EAAAC,WAAA,EAAAC,YAAA,CAAA;AAAA,IAAA,OAAAV,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAAC,UAAAC,SAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,SAAA,CAAAC,IAAA,GAAAD,SAAA,CAAAE,IAAA;AAAA,QAAA,KAAA,CAAA;AAAAF,UAAAA,SAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;UAAA,OACpFC,kBAAkB,EAAE,CAAA;AAAA,QAAA,KAAA,CAAA;UAArCV,QAAQ,GAAAO,SAAA,CAAAI,IAAA,CAAA;AACVV,UAAAA,WAAW,GAAGW,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,CAAA;AAC9CiB,UAAAA,KAAK,GAAGY,MAAM,CAACC,UAAU,EAAE,CAAA;AAAAR,UAAAA,SAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;AAAA,UAAA,OACPZ,KAAK,CAACmB,QAAQ,CAAClB,KAAK,EAAE;AAC9CI,YAAAA,KAAK,EAALA,KAAAA;AACD,WAAA,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAFIC,WAAW,GAAAI,SAAA,CAAAI,IAAA,CAAA;UAIjB,IAAI,CAACV,WAAW,EAAE;AAChBA,YAAAA,WAAW,GAAGW,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,GAAG6B,MAAM,CAACC,UAAU,EAAE,CAAA;AACtEE,YAAAA,OAAO,CAACC,IAAI,CAAC,6FAA6F,CAAC,CAAA;AAC7G,WAAA;AAACX,UAAAA,SAAA,CAAAE,IAAA,GAAA,EAAA,CAAA;AAAA,UAAA,OAE0BU,UAAY,CAAC;YACtCC,SAAS,EAAEvB,KAAK,CAACwB,IAAI;AACrBvB,YAAAA,KAAK,EAALA,KAAK;AACLC,YAAAA,MAAM,EAANA,MAAM;YACNuB,MAAM,EAAEnB,WAAW,CAACoB,IAAI;AACxBtB,YAAAA,WAAW,EAAXA,WAAW;AACXC,YAAAA,KAAK,EAALA,KAAK;AACLF,YAAAA,QAAQ,EAARA,QAAAA;AACD,WAAA,CAAC,CAAA;AAAA,QAAA,KAAA,EAAA;UARII,YAAY,GAAAG,SAAA,CAAAI,IAAA,CAAA;AAAA,UAAA,OAAAJ,SAAA,CAAAiB,MAAA,CAAA,QAAA,EAUXpB,YAAY,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAG,SAAA,CAAAkB,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAA7B,QAAA,CAAA,CAAA;GACpB,CAAA,CAAA,CAAA;AAAA,EAAA,OAAAN,SAAA,CAAAC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAEM,IAAMkB,kBAAkB,gBAAA,YAAA;EAAA,IAAAgB,IAAA,gBAAAjC,iBAAA,cAAAC,mBAAA,EAAAC,CAAAA,IAAA,CAAG,SAAAgC,OAAA,GAAA;AAAA,IAAA,IAAAC,KAAA,EAAAC,cAAA,EAAAC,MAAA,EAAAC,MAAA,CAAA;AAAA,IAAA,OAAArC,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAA2B,SAAAC,QAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,QAAA,CAAAzB,IAAA,GAAAyB,QAAA,CAAAxB,IAAA;AAAA,QAAA,KAAA,CAAA;AAAA,UAAA,IAAA,EAE5B,OAAOyB,MAAM,KAAK,WAAW,IAAIA,MAAM,CAACC,QAAQ,CAAA,EAAA;AAAAF,YAAAA,QAAA,CAAAxB,IAAA,GAAA,CAAA,CAAA;AAAA,YAAA,MAAA;AAAA,WAAA;AAC5CmB,UAAAA,KAAK,GAAGM,MAAM,CAACC,QAAQ,EAAE,CAAA;UAAA,OAAAF,QAAA,CAAAT,MAAA,CACxB,QAAA,EAAA;YACLY,QAAQ,EAAER,KAAK,CAACS,eAAe;YAC/BC,QAAQ,EAAEV,KAAK,CAACU,QAAAA;WACjB,CAAA,CAAA;AAAA,QAAA,KAAA,CAAA;AAAAL,UAAAA,QAAA,CAAAzB,IAAA,GAAA,CAAA,CAAA;AAAAyB,UAAAA,QAAA,CAAAxB,IAAA,GAAA,CAAA,CAAA;UAAA,OAIoB,OAAO,QAAQ,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;UAA/BqB,MAAM,GAAAG,QAAA,CAAAtB,IAAA,CAAA;AAAA,UAAA,IAAA,EACR,OAAOmB,MAAM,KAAK,WAAW,IAAAD,CAAAA,cAAA,GAAIC,MAAM,CAACI,MAAM,KAAbL,IAAAA,IAAAA,cAAA,CAAeM,QAAQ,CAAA,EAAA;AAAAF,YAAAA,QAAA,CAAAxB,IAAA,GAAA,EAAA,CAAA;AAAA,YAAA,MAAA;AAAA,WAAA;AACpDmB,UAAAA,MAAK,GAAGE,MAAM,CAACI,MAAM,CAACC,QAAQ,EAAE,CAAA;UAAA,OAAAF,QAAA,CAAAT,MAAA,CAC/B,QAAA,EAAA;YACLY,QAAQ,EAAER,MAAK,CAACS,eAAe;YAC/BC,QAAQ,EAAEV,MAAK,CAACU,QAAAA;WACjB,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA;AAAAL,UAAAA,QAAA,CAAAxB,IAAA,GAAA,EAAA,CAAA;AAAA,UAAA,MAAA;AAAA,QAAA,KAAA,EAAA;AAAAwB,UAAAA,QAAA,CAAAzB,IAAA,GAAA,EAAA,CAAA;UAAAyB,QAAA,CAAAM,EAAA,GAAAN,QAAA,CAAA,OAAA,CAAA,CAAA,CAAA,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA;AAAA,UAAA,OAAAA,QAAA,CAAAT,MAAA,CAAA,QAAA,EAIE,IAAI,CAAA,CAAA;AAAA,QAAA,KAAA,EAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAS,QAAA,CAAAR,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAAE,OAAA,EAAA,IAAA,EAAA,CAAA,CAAA,CAAA,EAAA,EAAA,CAAA,CAAA,CAAA,CAAA;GACZ,CAAA,CAAA,CAAA;AAAA,EAAA,OAAA,SAtBYjB,kBAAkBA,GAAA;AAAA,IAAA,OAAAgB,IAAA,CAAAnC,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,GAAA,CAAA;AAAA,CAsB9B,EAAA;;AC9CD,SAAsBgD,eAAeA,GAAA;AAAA,EAAA,OAAAC,gBAAA,CAAAlD,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAUpC,SAAAiD,gBAAA,GAAA;EAAAA,gBAAA,GAAAhD,iBAAA,cAAAC,mBAAA,GAAAC,IAAA,CAVM,SAAAgC,OAAA,GAAA;AAAA,IAAA,IAAAe,aAAA,CAAA;AAAA,IAAA,OAAAhD,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAA2B,SAAAC,QAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,QAAA,CAAAzB,IAAA,GAAAyB,QAAA,CAAAxB,IAAA;AAAA,QAAA,KAAA,CAAA;UACCiC,aAAa,GAAGC,IAAI,CAAC/B,OAAO,CAACgC,GAAG,EAAE,EAAE,SAAS,CAAC,CAAA;UAEpD,IAAI;YACFC,SAAS,CAACH,aAAa,CAAC,CAAA;AAC1B,WAAC,CAAC,OAAOI,KAAK,EAAE,EAAC;AAEjBC,UAAAA,YAAY,CAACC,cAAc,CAACC,aAAa,EAAE,UAAAC,WAAW,EAAG;YACvDC,UAAU,CAACR,IAAI,CAACD,aAAa,EAAE,YAAY,CAAC,EAAEU,IAAI,CAACC,SAAS,CAACH,WAAW,CAAC,GAAG,IAAI,EAAE,YAAK,EAAG,CAAC,CAAA;AAC7F,WAAC,CAAC,CAAA;AAAC,QAAA,KAAA,CAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAjB,QAAA,CAAAR,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAAE,OAAA,CAAA,CAAA;GACJ,CAAA,CAAA,CAAA;AAAA,EAAA,OAAAc,gBAAA,CAAAlD,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAED,SAAsB8D,WAAWA,GAAA;AAAA,EAAA,OAAAC,YAAA,CAAAhE,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA,CAAA;AAOhC,SAAA+D,YAAA,GAAA;EAAAA,YAAA,GAAA9D,iBAAA,cAAAC,mBAAA,GAAAC,IAAA,CAPM,SAAAC,QAAA,GAAA;AAAA,IAAA,IAAAK,WAAA,CAAA;AAAA,IAAA,OAAAP,mBAAA,EAAA,CAAAW,IAAA,CAAA,SAAAC,UAAAC,SAAA,EAAA;AAAA,MAAA,OAAA,CAAA,EAAA,QAAAA,SAAA,CAAAC,IAAA,GAAAD,SAAA,CAAAE,IAAA;AAAA,QAAA,KAAA,CAAA;AAAA,UAAA,IAAA,CACDG,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,EAAA;AAAAsB,YAAAA,SAAA,CAAAE,IAAA,GAAA,CAAA,CAAA;AAAA,YAAA,MAAA;AAAA,WAAA;AAAA,UAAA,MAC9B,IAAI+C,KAAK,CAAC,mEAAmE,CAAC,CAAA;AAAA,QAAA,KAAA,CAAA;AAGhFvD,UAAAA,WAAW,GAAGa,MAAM,CAACC,UAAU,EAAE,CAAA;AACvCH,UAAAA,OAAO,CAACC,GAAG,CAAC5B,qBAAqB,CAAC,GAAGgB,WAAW,CAAA;AAAC,QAAA,KAAA,CAAA,CAAA;AAAA,QAAA,KAAA,KAAA;UAAA,OAAAM,SAAA,CAAAkB,IAAA,EAAA,CAAA;AAAA,OAAA;AAAA,KAAA,EAAA7B,QAAA,CAAA,CAAA;GAClD,CAAA,CAAA,CAAA;AAAA,EAAA,OAAA2D,YAAA,CAAAhE,KAAA,CAAA,IAAA,EAAAC,SAAA,CAAA,CAAA;AAAA;;;;"}

package/dist/evaluation.d.ts

@@ -1,3 +1,7 @@
 import { type Agent, type Metric } from '@mastra/core';
 export declare function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric): Promise<import("@mastra/core").MetricResult>;
+export declare const getCurrentTestInfo: () => Promise<{
+    testName: string | undefined;
+    testPath: string | undefined;
+} | null>;
 //# sourceMappingURL=evaluation.d.ts.map

package/dist/evaluation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../src/evaluation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAEvD,wBAAsB,QAAQ,CAAC,CAAC,SAAS,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,gDAgB5G"}
+{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../src/evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,MAAM,EAA4B,MAAM,cAAc,CAAC;AAIjF,wBAAsB,QAAQ,CAAC,CAAC,SAAS,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,gDAwB5G;AAED,eAAO,MAAM,kBAAkB;;;SAsB9B,CAAC"}

package/dist/index.d.ts

@@ -1,2 +1,3 @@
 export { evaluate } from './evaluation';
+export { attachListeners, globalSetup } from './attachListeners';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC"}