@mastra/evals 0.13.8-alpha.1 → 0.13.8

package/dist/scorers/llm/bias/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/bias/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAW3F,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,gBAAgB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAAE,KAAK,EAAE,aAAa,CAAC;IAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;CAAE;;;;;;;6FA+CzG"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/bias/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAW3F,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,gBAAgB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAAE,KAAK,EAAE,aAAa,CAAC;IAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;CAAE;;;;;;;6FAgDzG"}

package/dist/scorers/llm/context-precision/index.d.ts

@@ -8,7 +8,7 @@ export interface ContextPrecisionMetricOptions {
 export declare function createContextPrecisionScorer({ model, options, }: {
     model: MastraLanguageModel;
     options: ContextPrecisionMetricOptions;
-}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"analyzeStepResult", {
+}): import("@mastra/core/scores").MastraScorer<"Context Precision Scorer", ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"analyzeStepResult", {
     verdicts: {
         verdict: string;
         reason: string;

package/dist/scorers/llm/context-precision/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-precision/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAS3F,MAAM,WAAW,6BAA6B;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,EAAE,CAAC;CACjG;AAYD,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,EAAE,6BAA6B,CAAC;CACxC;;;;;;6FAkGA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-precision/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAS3F,MAAM,WAAW,6BAA6B;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,EAAE,CAAC;CACjG;AAYD,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,EAAE,6BAA6B,CAAC;CACxC;;;;;;6FAmGA"}

package/dist/scorers/llm/context-relevance/index.d.ts

@@ -13,7 +13,7 @@ export interface ContextRelevanceOptions {
 export declare function createContextRelevanceScorerLLM({ model, options, }: {
     model: MastraLanguageModel;
     options: ContextRelevanceOptions;
-}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"analyzeStepResult", {
+}): import("@mastra/core/scores").MastraScorer<"Context Relevance (LLM)", ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"analyzeStepResult", {
     evaluations: {
         reasoning: string;
         context_index: number;

package/dist/scorers/llm/context-relevance/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-relevance/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAM3F,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,EAAE,CAAC;IAChG,SAAS,CAAC,EAAE;QACV,0BAA0B,CAAC,EAAE,MAAM,CAAC;QACpC,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;KACnC,CAAC;CACH;AAuBD,wBAAgB,+BAA+B,CAAC,EAC9C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,EAAE,uBAAuB,CAAC;CAClC;;;;;;;;;;6FA4IA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-relevance/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAM3F,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,EAAE,CAAC;IAChG,SAAS,CAAC,EAAE;QACV,0BAA0B,CAAC,EAAE,MAAM,CAAC;QACpC,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;KACnC,CAAC;CACH;AAuBD,wBAAgB,+BAA+B,CAAC,EAC9C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,EAAE,uBAAuB,CAAC;CAClC;;;;;;;;;;6FA6IA"}

package/dist/scorers/llm/faithfulness/index.d.ts

@@ -1,5 +1,4 @@
 import type { LanguageModel } from '@mastra/core/llm';
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
 export interface FaithfulnessMetricOptions {
     scale?: number;
     context?: string[];
@@ -7,7 +6,7 @@ export interface FaithfulnessMetricOptions {
 export declare function createFaithfulnessScorer({ model, options, }: {
     model: LanguageModel;
     options?: FaithfulnessMetricOptions;
-}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", string[]> & Record<"analyzeStepResult", {
+}): import("@mastra/core/scores").MastraScorer<"Faithfulness Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", string[]> & Record<"analyzeStepResult", {
     verdicts: {
         verdict: string;
         reason: string;

package/dist/scorers/llm/faithfulness/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/faithfulness/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAU3F,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,wBAAgB,wBAAwB,CAAC,EACvC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACrC;;;;;6FAgEA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/faithfulness/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAWtD,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,wBAAgB,wBAAwB,CAAC,EACvC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACrC;;;;;6FAiEA"}

package/dist/scorers/llm/hallucination/index.d.ts

@@ -1,5 +1,4 @@
 import type { LanguageModel } from '@mastra/core/llm';
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
 export interface HallucinationMetricOptions {
     scale?: number;
     context: string[];
@@ -7,7 +6,7 @@ export interface HallucinationMetricOptions {
 export declare function createHallucinationScorer({ model, options, }: {
     model: LanguageModel;
     options?: HallucinationMetricOptions;
-}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+}): import("@mastra/core/scores").MastraScorer<"Hallucination Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     claims: string[];
 }> & Record<"analyzeStepResult", {
     verdicts: {

package/dist/scorers/llm/hallucination/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/hallucination/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAW3F,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,wBAAgB,yBAAyB,CAAC,EACxC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,CAAC,EAAE,0BAA0B,CAAC;CACtC;;;;;;;;6FA0DA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/hallucination/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAYtD,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,wBAAgB,yBAAyB,CAAC,EACxC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,CAAC,EAAE,0BAA0B,CAAC;CACtC;;;;;;;;6FA2DA"}

package/dist/scorers/llm/index.cjs

@@ -222,7 +222,8 @@ function createAnswerRelevancyScorer({
     judge: {
       model,
       instructions: ANSWER_RELEVANCY_AGENT_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).preprocess({
     description: "Extract relevant statements from the LLM output",
     outputSchema: extractOutputSchema,
@@ -438,7 +439,8 @@ function createAnswerSimilarityScorer({
     judge: {
       model,
       instructions: ANSWER_SIMILARITY_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).preprocess({
     description: "Extract semantic units from output and ground truth",
     outputSchema: extractOutputSchema2,
@@ -694,7 +696,8 @@ function createFaithfulnessScorer({
     judge: {
       model,
       instructions: FAITHFULNESS_AGENT_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).preprocess({
     description: "Extract relevant statements from the LLM output",
     outputSchema: zod.z.array(zod.z.string()),
@@ -851,7 +854,8 @@ function createBiasScorer({ model, options }) {
     judge: {
       model,
       instructions: BIAS_AGENT_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).preprocess({
     description: "Extract relevant statements from the LLM output",
     outputSchema: zod.z.object({
@@ -1084,7 +1088,8 @@ function createHallucinationScorer({
     judge: {
       model,
       instructions: HALLUCINATION_AGENT_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).preprocess({
     description: "Extract all claims from the given output",
     outputSchema: zod.z.object({
@@ -1222,7 +1227,8 @@ function createToxicityScorer({ model, options }) {
     judge: {
       model,
       instructions: TOXICITY_AGENT_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).analyze({
     description: "Score the relevance of the statements to the input",
     outputSchema: zod.z.object({ verdicts: zod.z.array(zod.z.object({ verdict: zod.z.string(), reason: zod.z.string() })) }),
@@ -1370,7 +1376,8 @@ function createToolCallAccuracyScorerLLM({ model, availableTools }) {
     judge: {
       model,
       instructions: TOOL_SELECTION_ACCURACY_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;
     const isOutputInvalid = !run.output || run.output.length === 0;
@@ -1607,7 +1614,8 @@ function createContextRelevanceScorerLLM({
     judge: {
       model,
       instructions: CONTEXT_RELEVANCE_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).analyze({
     description: "Analyze the relevance and utility of provided context",
     outputSchema: analyzeOutputSchema3,
@@ -1829,7 +1837,8 @@ function createContextPrecisionScorer({
     judge: {
       model,
       instructions: CONTEXT_PRECISION_AGENT_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).analyze({
     description: "Evaluate the relevance of each context piece for generating the expected output",
     outputSchema: contextRelevanceOutputSchema,
@@ -2120,7 +2129,8 @@ function createNoiseSensitivityScorerLLM({
     judge: {
       model,
       instructions: NOISE_SENSITIVITY_INSTRUCTIONS
-    }
+    },
+    type: "agent"
   }).analyze({
     description: "Analyze the impact of noise on agent response quality",
     outputSchema: analyzeOutputSchema4,