@mastra/evals 0.1.0-alpha.44 → 0.1.0-alpha.46

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @mastra/evals
 
+## 0.1.0-alpha.46
+
+### Patch Changes
+
+- 5fdc87c: Update evals storage in attachListeners
+- b97ca96: Tracing into default storage
+- 72d1990: Updated evals table schema
+- Updated dependencies [5fdc87c]
+- Updated dependencies [b97ca96]
+- Updated dependencies [72d1990]
+- Updated dependencies [cf6d825]
+- Updated dependencies [10870bc]
+  - @mastra/core@0.2.0-alpha.104
+
+## 0.1.0-alpha.45
+
+### Patch Changes
+
+- Updated dependencies [4534e77]
+  - @mastra/core@0.2.0-alpha.103
+
 ## 0.1.0-alpha.44
 
 ### Patch Changes

package/README.md

@@ -158,6 +158,8 @@ await attachListeners();
 
 // Store evals in Mastra Storage (if storage is enabled)
 await attachListeners(mastra);
+// Note: When using in-memory storage, evaluations are isolated to the test process.
+// When using file storage, evaluations are persisted and can be queried later.
 ```
 
 ## Environment Variables

package/dist/_tsup-dts-rollup.d.ts

@@ -470,7 +470,7 @@ export declare function generateReasonPrompt_alias_9({ originalText, summary, al
 export declare const getCurrentTestInfo: () => Promise<{
     testName: string | undefined;
     testPath: string | undefined;
-} | null>;
+} | undefined>;
 
 export declare function getReasonPrompt({ score, toxics }: {
     score: number;

package/dist/index.js

@@ -50,19 +50,27 @@ var getCurrentTestInfo = async () => {
     }
   } catch {
   }
-  return null;
+  return undefined;
 };
 async function attachListeners(mastra) {
+  if (mastra?.storage) {
+    await mastra.storage.init();
+  }
   registerHook(AvailableHooks.ON_EVALUATION, async (traceObject) => {
-    if (mastra?.memory?.storage) {
-      await mastra.memory.storage.insert({
+    if (mastra?.storage) {
+      await mastra.storage.insert({
         tableName: MastraStorage.TABLE_EVALS,
         record: {
-          result: JSON.stringify(traceObject.result),
-          meta: JSON.stringify(traceObject.meta),
           input: traceObject.input,
           output: traceObject.output,
-          createdAt: (/* @__PURE__ */ new Date()).toISOString()
+          result: JSON.stringify(traceObject.result),
+          agent_name: traceObject.agentName,
+          metric_name: traceObject.metricName,
+          instructions: traceObject.instructions,
+          test_info: traceObject.testInfo ? JSON.stringify(traceObject.testInfo) : null,
+          global_run_id: traceObject.globalRunId,
+          run_id: traceObject.runId,
+          created_at: (/* @__PURE__ */ new Date()).toISOString()
         }
       });
     }

package/dist/metrics/llm/index.js

@@ -1237,7 +1237,7 @@ var ToxicityJudge = class extends MastraAgentJudge {
         verdicts: z.array(
           z.object({
             verdict: z.string(),
-            reason: z.string()
+            reason: z.string().optional()
           })
         )
       })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/evals",
-  "version": "0.1.0-alpha.44",
+  "version": "0.1.0-alpha.46",
   "description": "",
   "type": "module",
   "main": "dist/index.js",
@@ -37,7 +37,7 @@
     "sentiment": "^5.0.2",
     "string-similarity": "^4.0.4",
     "zod": "^3.24.1",
-    "@mastra/core": "^0.2.0-alpha.102"
+    "@mastra/core": "^0.2.0-alpha.104"
   },
   "peerDependencies": {
     "ai": "^4.0.0"

package/src/attachListeners.ts

@@ -5,16 +5,25 @@ import { MastraStorage } from '@mastra/core/storage';
 import { GLOBAL_RUN_ID_ENV_KEY } from './constants';
 
 export async function attachListeners(mastra?: Mastra) {
+  if (mastra?.storage) {
+    await mastra.storage.init();
+  }
+
   registerHook(AvailableHooks.ON_EVALUATION, async traceObject => {
-    if (mastra?.memory?.storage) {
-      await mastra.memory.storage.insert({
+    if (mastra?.storage) {
+      await mastra.storage.insert({
         tableName: MastraStorage.TABLE_EVALS,
         record: {
-          result: JSON.stringify(traceObject.result),
-          meta: JSON.stringify(traceObject.meta),
           input: traceObject.input,
           output: traceObject.output,
-          createdAt: new Date().toISOString(),
+          result: JSON.stringify(traceObject.result),
+          agent_name: traceObject.agentName,
+          metric_name: traceObject.metricName,
+          instructions: traceObject.instructions,
+          test_info: traceObject.testInfo ? JSON.stringify(traceObject.testInfo) : null,
+          global_run_id: traceObject.globalRunId,
+          run_id: traceObject.runId,
+          created_at: new Date().toISOString(),
         },
       });
     }

package/src/evaluation.ts

@@ -53,5 +53,5 @@ export const getCurrentTestInfo = async () => {
     }
   } catch {}
 
-  return null;
+  return undefined;
 };

package/src/metrics/llm/toxicity/metricJudge.ts

@@ -17,7 +17,7 @@ export class ToxicityJudge extends MastraAgentJudge {
         verdicts: z.array(
           z.object({
             verdict: z.string(),
-            reason: z.string(),
+            reason: z.string().optional(),
           }),
         ),
       }),