@mastra/evals 1.2.2 → 1.2.3-alpha.0

package/dist/scorers/prebuilt/index.cjs

@@ -2,7 +2,6 @@
 
 var chunk33T2SZZ2_cjs = require('../../chunk-33T2SZZ2.cjs');
 var evals = require('@mastra/core/evals');
-var zod = require('zod');
 var nlp = require('compromise');
 var keyword_extractor = require('keyword-extractor');
 var stringSimilarity = require('string-similarity');
@@ -219,9 +218,21 @@ var ANSWER_RELEVANCY_AGENT_INSTRUCTIONS = `
     5. Empty inputs or error messages should always be marked as "no"
     6. Responses that discuss the type of information being asked show partial relevance
 `;
-var extractOutputSchema = zod.z.object({
-  statements: zod.z.array(zod.z.string())
-});
+var extractOutputSchema = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "statements": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": [
+    "statements"
+  ]
+};
 function createAnswerRelevancyScorer({
   model,
   options = DEFAULT_OPTIONS
@@ -244,7 +255,33 @@ function createAnswerRelevancyScorer({
     }
   }).analyze({
     description: "Score the relevance of the statements to the input",
-    outputSchema: zod.z.object({ results: zod.z.array(zod.z.object({ result: zod.z.string(), reason: zod.z.string() })) }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "results": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "result": {
+                "type": "string"
+              },
+              "reason": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "result",
+              "reason"
+            ]
+          }
+        }
+      },
+      "required": [
+        "results"
+      ]
+    },
     createPrompt: ({ run, results }) => {
       const input = chunk33T2SZZ2_cjs.getUserMessageFromRunInput(run.input) ?? "";
       return createScorePrompt(JSON.stringify(input), results.preprocessStepResult?.statements || []);
@@ -417,28 +454,106 @@ Key Principles:
 5. Provide actionable feedback for improving answer accuracy
 6. Be strict but fair - partial credit for partial matches
 `;
-var extractOutputSchema2 = zod.z.object({
-  outputUnits: zod.z.array(zod.z.string()),
-  groundTruthUnits: zod.z.array(zod.z.string())
-});
-var analyzeOutputSchema = zod.z.object({
-  matches: zod.z.array(
-    zod.z.object({
-      groundTruthUnit: zod.z.string(),
-      outputUnit: zod.z.string().nullable(),
-      matchType: zod.z.enum(["exact", "semantic", "partial", "missing"]),
-      explanation: zod.z.string()
-    })
-  ),
-  extraInOutput: zod.z.array(zod.z.string()),
-  contradictions: zod.z.array(
-    zod.z.object({
-      outputUnit: zod.z.string(),
-      groundTruthUnit: zod.z.string(),
-      explanation: zod.z.string()
-    })
-  )
-});
+var extractOutputSchema2 = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "outputUnits": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "groundTruthUnits": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": [
+    "outputUnits",
+    "groundTruthUnits"
+  ]
+};
+var analyzeOutputSchema = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "matches": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "groundTruthUnit": {
+            "type": "string"
+          },
+          "outputUnit": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "matchType": {
+            "type": "string",
+            "enum": [
+              "exact",
+              "semantic",
+              "partial",
+              "missing"
+            ]
+          },
+          "explanation": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "groundTruthUnit",
+          "outputUnit",
+          "matchType",
+          "explanation"
+        ]
+      }
+    },
+    "extraInOutput": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "contradictions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "outputUnit": {
+            "type": "string"
+          },
+          "groundTruthUnit": {
+            "type": "string"
+          },
+          "explanation": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "outputUnit",
+          "groundTruthUnit",
+          "explanation"
+        ]
+      }
+    }
+  },
+  "required": [
+    "matches",
+    "extraInOutput",
+    "contradictions"
+  ]
+};
 function createAnswerSimilarityScorer({
   model,
   options = ANSWER_SIMILARITY_DEFAULT_OPTIONS
@@ -717,16 +832,54 @@ function createFaithfulnessScorer({
     type: "agent"
   }).preprocess({
     description: "Extract relevant statements from the LLM output",
-    outputSchema: zod.z.object({
-      claims: zod.z.array(zod.z.string())
-    }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "claims": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": [
+        "claims"
+      ]
+    },
     createPrompt: ({ run }) => {
       const prompt = createFaithfulnessExtractPrompt({ output: chunk33T2SZZ2_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" });
       return prompt;
     }
   }).analyze({
     description: "Score the relevance of the statements to the input",
-    outputSchema: zod.z.object({ verdicts: zod.z.array(zod.z.object({ verdict: zod.z.string(), reason: zod.z.string() })) }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "verdicts": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "verdict": {
+                "type": "string"
+              },
+              "reason": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "verdict",
+              "reason"
+            ]
+          }
+        }
+      },
+      "required": [
+        "verdicts"
+      ]
+    },
     createPrompt: ({ results, run }) => {
       const context = options?.context ?? getToolInvocationContext(run.output);
       const prompt = createFaithfulnessAnalyzePrompt({
@@ -878,13 +1031,51 @@ function createBiasScorer({ model, options }) {
     type: "agent"
   }).preprocess({
     description: "Extract relevant statements from the LLM output",
-    outputSchema: zod.z.object({
-      opinions: zod.z.array(zod.z.string())
-    }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "opinions": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": [
+        "opinions"
+      ]
+    },
     createPrompt: ({ run }) => createBiasExtractPrompt({ output: chunk33T2SZZ2_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" })
   }).analyze({
     description: "Score the relevance of the statements to the input",
-    outputSchema: zod.z.object({ results: zod.z.array(zod.z.object({ result: zod.z.string(), reason: zod.z.string() })) }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "results": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "result": {
+                "type": "string"
+              },
+              "reason": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "result",
+              "reason"
+            ]
+          }
+        }
+      },
+      "required": [
+        "results"
+      ]
+    },
     createPrompt: ({ run, results }) => {
       const prompt = createBiasAnalyzePrompt({
         output: chunk33T2SZZ2_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
@@ -1113,18 +1304,58 @@ function createHallucinationScorer({
     type: "agent"
   }).preprocess({
     description: "Extract all claims from the given output",
-    outputSchema: zod.z.object({
-      claims: zod.z.array(zod.z.string())
-    }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "claims": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": [
+        "claims"
+      ]
+    },
     createPrompt: ({ run }) => {
       const prompt = createHallucinationExtractPrompt({ output: chunk33T2SZZ2_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" });
       return prompt;
     }
   }).analyze({
     description: "Score the relevance of the statements to the input",
-    outputSchema: zod.z.object({
-      verdicts: zod.z.array(zod.z.object({ statement: zod.z.string(), verdict: zod.z.string(), reason: zod.z.string() }))
-    }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "verdicts": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "statement": {
+                "type": "string"
+              },
+              "verdict": {
+                "type": "string"
+              },
+              "reason": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "statement",
+              "verdict",
+              "reason"
+            ]
+          }
+        }
+      },
+      "required": [
+        "verdicts"
+      ]
+    },
     createPrompt: async ({ run, results }) => {
       let context;
       if (options?.getContext) {
@@ -1268,7 +1499,33 @@ function createToxicityScorer({
     type: "agent"
   }).analyze({
     description: "Score the relevance of the statements to the input",
-    outputSchema: zod.z.object({ verdicts: zod.z.array(zod.z.object({ verdict: zod.z.string(), reason: zod.z.string() })) }),
+    outputSchema: {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "verdicts": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "verdict": {
+                "type": "string"
+              },
+              "reason": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "verdict",
+              "reason"
+            ]
+          }
+        }
+      },
+      "required": [
+        "verdicts"
+      ]
+    },
     createPrompt: ({ run }) => {
       const prompt = createToxicityAnalyzePrompt({
         input: chunk33T2SZZ2_cjs.getUserMessageFromRunInput(run.input) ?? "",
@@ -1395,16 +1652,43 @@ Provide a single, concise sentence explaining why this score was given.
 };
 
 // src/scorers/llm/tool-call-accuracy/index.ts
-var analyzeOutputSchema2 = zod.z.object({
-  evaluations: zod.z.array(
-    zod.z.object({
-      toolCalled: zod.z.string(),
-      wasAppropriate: zod.z.boolean(),
-      reasoning: zod.z.string()
-    })
-  ),
-  missingTools: zod.z.array(zod.z.string()).optional()
-});
+var analyzeOutputSchema2 = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "evaluations": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "toolCalled": {
+            "type": "string"
+          },
+          "wasAppropriate": {
+            "type": "boolean"
+          },
+          "reasoning": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "toolCalled",
+          "wasAppropriate",
+          "reasoning"
+        ]
+      }
+    },
+    "missingTools": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": [
+    "evaluations"
+  ]
+};
 function createToolCallAccuracyScorerLLM({ model, availableTools }) {
   const toolDefinitions = availableTools.map((tool) => `${tool.id}: ${tool.description}`).join("\n");
   return evals.createScorer({
@@ -1615,19 +1899,62 @@ Example responses:
 }
 
 // src/scorers/llm/context-relevance/index.ts
-var analyzeOutputSchema3 = zod.z.object({
-  evaluations: zod.z.array(
-    zod.z.object({
-      context_index: zod.z.number(),
-      contextPiece: zod.z.string(),
-      relevanceLevel: zod.z.enum(["high", "medium", "low", "none"]),
-      wasUsed: zod.z.boolean(),
-      reasoning: zod.z.string()
-    })
-  ),
-  missingContext: zod.z.array(zod.z.string()).optional().default([]),
-  overallAssessment: zod.z.string()
-});
+var analyzeOutputSchema3 = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "evaluations": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "context_index": {
+            "type": "number"
+          },
+          "contextPiece": {
+            "type": "string"
+          },
+          "relevanceLevel": {
+            "type": "string",
+            "enum": [
+              "high",
+              "medium",
+              "low",
+              "none"
+            ]
+          },
+          "wasUsed": {
+            "type": "boolean"
+          },
+          "reasoning": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "context_index",
+          "contextPiece",
+          "relevanceLevel",
+          "wasUsed",
+          "reasoning"
+        ]
+      }
+    },
+    "missingContext": {
+      "default": [],
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "overallAssessment": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "evaluations",
+    "overallAssessment"
+  ]
+};
 var DEFAULT_PENALTIES = {
   UNUSED_HIGH_RELEVANCE_CONTEXT: 0.1,
   // 10% penalty per unused high-relevance context
@@ -1861,15 +2188,37 @@ Example responses:
 }
 
 // src/scorers/llm/context-precision/index.ts
-var contextRelevanceOutputSchema = zod.z.object({
-  verdicts: zod.z.array(
-    zod.z.object({
-      context_index: zod.z.number(),
-      verdict: zod.z.string(),
-      reason: zod.z.string()
-    })
-  )
-});
+var contextRelevanceOutputSchema = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "verdicts": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "context_index": {
+            "type": "number"
+          },
+          "verdict": {
+            "type": "string"
+          },
+          "reason": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "context_index",
+          "verdict",
+          "reason"
+        ]
+      }
+    }
+  },
+  "required": [
+    "verdicts"
+  ]
+};
 var getContext2 = ({
   input,
   output,
@@ -2148,20 +2497,63 @@ Example responses:
 }
 
 // src/scorers/llm/noise-sensitivity/index.ts
-var scoreSchema = zod.z.number().refine((n) => n >= 0 && n <= 1, { message: "Score must be between 0 and 1" });
-var analyzeOutputSchema4 = zod.z.object({
-  dimensions: zod.z.array(
-    zod.z.object({
-      dimension: zod.z.string(),
-      impactLevel: zod.z.enum(["none", "minimal", "moderate", "significant", "severe"]),
-      specificChanges: zod.z.string(),
-      noiseInfluence: zod.z.string()
-    })
-  ),
-  overallAssessment: zod.z.string(),
-  majorIssues: zod.z.array(zod.z.string()).optional().default([]),
-  robustnessScore: scoreSchema
-});
+var analyzeOutputSchema4 = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "dimensions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "dimension": {
+            "type": "string"
+          },
+          "impactLevel": {
+            "type": "string",
+            "enum": [
+              "none",
+              "minimal",
+              "moderate",
+              "significant",
+              "severe"
+            ]
+          },
+          "specificChanges": {
+            "type": "string"
+          },
+          "noiseInfluence": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "dimension",
+          "impactLevel",
+          "specificChanges",
+          "noiseInfluence"
+        ]
+      }
+    },
+    "overallAssessment": {
+      "type": "string"
+    },
+    "majorIssues": {
+      "default": [],
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "robustnessScore": {
+      "type": "number"
+    }
+  },
+  "required": [
+    "dimensions",
+    "overallAssessment",
+    "robustnessScore"
+  ]
+};
 var DEFAULT_IMPACT_WEIGHTS = {
   none: 1,
   minimal: 0.85,
@@ -2494,37 +2886,124 @@ Example responses:
 }
 
 // src/scorers/llm/prompt-alignment/index.ts
-var scoreSchema2 = zod.z.number().refine((n) => n >= 0 && n <= 1, { message: "Score must be between 0 and 1" });
-var analyzeOutputSchema5 = zod.z.object({
-  intentAlignment: zod.z.object({
-    score: scoreSchema2,
-    primaryIntent: zod.z.string(),
-    isAddressed: zod.z.boolean(),
-    reasoning: zod.z.string()
-  }),
-  requirementsFulfillment: zod.z.object({
-    requirements: zod.z.array(
-      zod.z.object({
-        requirement: zod.z.string(),
-        isFulfilled: zod.z.boolean(),
-        reasoning: zod.z.string()
-      })
-    ),
-    overallScore: scoreSchema2
-  }),
-  completeness: zod.z.object({
-    score: scoreSchema2,
-    missingElements: zod.z.array(zod.z.string()),
-    reasoning: zod.z.string()
-  }),
-  responseAppropriateness: zod.z.object({
-    score: scoreSchema2,
-    formatAlignment: zod.z.boolean(),
-    toneAlignment: zod.z.boolean(),
-    reasoning: zod.z.string()
-  }),
-  overallAssessment: zod.z.string()
-});
+var analyzeOutputSchema5 = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "intentAlignment": {
+      "type": "object",
+      "properties": {
+        "score": {
+          "type": "number"
+        },
+        "primaryIntent": {
+          "type": "string"
+        },
+        "isAddressed": {
+          "type": "boolean"
+        },
+        "reasoning": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "score",
+        "primaryIntent",
+        "isAddressed",
+        "reasoning"
+      ]
+    },
+    "requirementsFulfillment": {
+      "type": "object",
+      "properties": {
+        "requirements": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "requirement": {
+                "type": "string"
+              },
+              "isFulfilled": {
+                "type": "boolean"
+              },
+              "reasoning": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "requirement",
+              "isFulfilled",
+              "reasoning"
+            ]
+          }
+        },
+        "overallScore": {
+          "type": "number"
+        }
+      },
+      "required": [
+        "requirements",
+        "overallScore"
+      ]
+    },
+    "completeness": {
+      "type": "object",
+      "properties": {
+        "score": {
+          "type": "number"
+        },
+        "missingElements": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "reasoning": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "score",
+        "missingElements",
+        "reasoning"
+      ]
+    },
+    "responseAppropriateness": {
+      "type": "object",
+      "properties": {
+        "score": {
+          "type": "number"
+        },
+        "formatAlignment": {
+          "type": "boolean"
+        },
+        "toneAlignment": {
+          "type": "boolean"
+        },
+        "reasoning": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "score",
+        "formatAlignment",
+        "toneAlignment",
+        "reasoning"
+      ]
+    },
+    "overallAssessment": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "intentAlignment",
+    "requirementsFulfillment",
+    "completeness",
+    "responseAppropriateness",
+    "overallAssessment"
+  ]
+};
 var SCORING_WEIGHTS = {
   USER: {
     INTENT_ALIGNMENT: 0.4,
@@ -2719,19 +3198,64 @@ Provide a single, concise sentence explaining why this score was given.
 };
 
 // src/scorers/llm/trajectory/index.ts
-var analyzeOutputSchema6 = zod.z.object({
-  stepEvaluations: zod.z.array(
-    zod.z.object({
-      stepName: zod.z.string().describe("Name of the step (tool name or action)"),
-      wasNecessary: zod.z.boolean().describe("Whether this step was necessary for the task"),
-      wasInOrder: zod.z.boolean().describe("Whether this step was in a logical position in the sequence"),
-      reasoning: zod.z.string().describe("Brief explanation of the evaluation")
-    })
-  ),
-  missingSteps: zod.z.array(zod.z.string()).optional().describe("Steps that should have been taken but were not"),
-  extraSteps: zod.z.array(zod.z.string()).optional().describe("Steps that were unnecessary or redundant"),
-  overallAssessment: zod.z.string().describe("Brief overall assessment of the trajectory quality")
-});
+var analyzeOutputSchema6 = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "stepEvaluations": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "stepName": {
+            "type": "string",
+            "description": "Name of the step (tool name or action)"
+          },
+          "wasNecessary": {
+            "type": "boolean",
+            "description": "Whether this step was necessary for the task"
+          },
+          "wasInOrder": {
+            "type": "boolean",
+            "description": "Whether this step was in a logical position in the sequence"
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Brief explanation of the evaluation"
+          }
+        },
+        "required": [
+          "stepName",
+          "wasNecessary",
+          "wasInOrder",
+          "reasoning"
+        ]
+      }
+    },
+    "missingSteps": {
+      "description": "Steps that should have been taken but were not",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "extraSteps": {
+      "description": "Steps that were unnecessary or redundant",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "overallAssessment": {
+      "type": "string",
+      "description": "Brief overall assessment of the trajectory quality"
+    }
+  },
+  "required": [
+    "stepEvaluations",
+    "overallAssessment"
+  ]
+};
 function formatStepDetails(step) {
   switch (step.stepType) {
     case "tool_call":