@mastra/evals 0.14.3-alpha.0 → 1.0.0-beta.0

package/dist/scorers/{llm → prebuilt}/index.cjs

@@ -1,9 +1,19 @@
 'use strict';
 
-var chunk7QAUEU4L_cjs = require('../../chunk-7QAUEU4L.cjs');
-var chunkQKR2PMLZ_cjs = require('../../chunk-QKR2PMLZ.cjs');
-var scores = require('@mastra/core/scores');
+var chunkTPQLLHZW_cjs = require('../../chunk-TPQLLHZW.cjs');
+var evals = require('@mastra/core/evals');
 var zod = require('zod');
+var nlp = require('compromise');
+var keyword_extractor = require('keyword-extractor');
+var stringSimilarity = require('string-similarity');
+var Sentiment = require('sentiment');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+var nlp__default = /*#__PURE__*/_interopDefault(nlp);
+var keyword_extractor__default = /*#__PURE__*/_interopDefault(keyword_extractor);
+var stringSimilarity__default = /*#__PURE__*/_interopDefault(stringSimilarity);
+var Sentiment__default = /*#__PURE__*/_interopDefault(Sentiment);
 
 // src/scorers/llm/answer-relevancy/prompts.ts
 var createExtractPrompt = (output) => `
@@ -216,7 +226,8 @@ function createAnswerRelevancyScorer({
   model,
   options = DEFAULT_OPTIONS
 }) {
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "answer-relevancy-scorer",
     name: "Answer Relevancy Scorer",
     description: "A scorer that evaluates the relevancy of an LLM output to an input",
     judge: {
@@ -228,14 +239,14 @@ function createAnswerRelevancyScorer({
     description: "Extract relevant statements from the LLM output",
     outputSchema: extractOutputSchema,
     createPrompt: ({ run }) => {
-      const assistantMessage = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const assistantMessage = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       return createExtractPrompt(assistantMessage);
     }
   }).analyze({
     description: "Score the relevance of the statements to the input",
     outputSchema: zod.z.object({ results: zod.z.array(zod.z.object({ result: zod.z.string(), reason: zod.z.string() })) }),
     createPrompt: ({ run, results }) => {
-      const input = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const input = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
       return createScorePrompt(JSON.stringify(input), results.preprocessStepResult?.statements || []);
     }
   }).generateScore(({ results }) => {
@@ -252,13 +263,13 @@ function createAnswerRelevancyScorer({
       }
     }
     const score = relevancyCount / numberOfResults;
-    return chunk7QAUEU4L_cjs.roundToTwoDecimals(score * options.scale);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(score * options.scale);
   }).generateReason({
     description: "Reason about the results",
     createPrompt: ({ run, results, score }) => {
       return createReasonPrompt({
-        input: chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "",
-        output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
+        input: chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "",
+        output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
         score,
         results: results.analyzeStepResult.results,
         scale: options.scale
@@ -433,7 +444,8 @@ function createAnswerSimilarityScorer({
   options = ANSWER_SIMILARITY_DEFAULT_OPTIONS
 }) {
   const mergedOptions = { ...ANSWER_SIMILARITY_DEFAULT_OPTIONS, ...options };
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "answer-similarity-scorer",
     name: "Answer Similarity Scorer",
     description: "Evaluates how similar an agent output is to a ground truth answer for CI/CD testing",
     judge: {
@@ -454,7 +466,7 @@ function createAnswerSimilarityScorer({
           groundTruth: ""
         });
       }
-      const output = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const output = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       const groundTruth = typeof run.groundTruth === "string" ? run.groundTruth : JSON.stringify(run.groundTruth);
       return createExtractPrompt2({
         output,
@@ -512,14 +524,14 @@ function createAnswerSimilarityScorer({
     );
     score -= extraInfoPenalty;
     score = Math.max(0, Math.min(1, score));
-    return chunk7QAUEU4L_cjs.roundToTwoDecimals(score * mergedOptions.scale);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(score * mergedOptions.scale);
   }).generateReason({
     description: "Generate explanation of similarity score",
     createPrompt: ({ run, results, score }) => {
       if (!run.groundTruth) {
         return "No ground truth was provided for comparison. Score is 0 by default.";
       }
-      const output = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const output = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       const groundTruth = typeof run.groundTruth === "string" ? run.groundTruth : JSON.stringify(run.groundTruth);
       return createReasonPrompt2({
         output,
@@ -690,7 +702,8 @@ function createFaithfulnessScorer({
   model,
   options
 }) {
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "faithfulness-scorer",
     name: "Faithfulness Scorer",
     description: "A scorer that evaluates the faithfulness of an LLM output to an input",
     judge: {
@@ -702,14 +715,17 @@ function createFaithfulnessScorer({
     description: "Extract relevant statements from the LLM output",
     outputSchema: zod.z.array(zod.z.string()),
     createPrompt: ({ run }) => {
-      const prompt = createFaithfulnessExtractPrompt({ output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" });
+      const prompt = createFaithfulnessExtractPrompt({ output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" });
       return prompt;
     }
   }).analyze({
     description: "Score the relevance of the statements to the input",
     outputSchema: zod.z.object({ verdicts: zod.z.array(zod.z.object({ verdict: zod.z.string(), reason: zod.z.string() })) }),
     createPrompt: ({ results, run }) => {
-      const context = options?.context ?? run.output.find(({ role }) => role === "assistant")?.toolInvocations?.map((toolCall) => toolCall.state === "result" ? JSON.stringify(toolCall.result) : "") ?? [];
+      const assistantMessage = run.output.find(({ role }) => role === "assistant");
+      const context = options?.context ?? assistantMessage?.content?.toolInvocations?.map(
+        (toolCall) => toolCall.state === "result" ? JSON.stringify(toolCall.result) : ""
+      ) ?? [];
       const prompt = createFaithfulnessAnalyzePrompt({
         claims: results.preprocessStepResult || [],
         context
@@ -723,14 +739,15 @@ function createFaithfulnessScorer({
       return 0;
     }
     const score = supportedClaims / totalClaims * (options?.scale || 1);
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(score);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(score);
   }).generateReason({
     description: "Reason about the results",
     createPrompt: ({ run, results, score }) => {
+      const assistantMessage = run.output.find(({ role }) => role === "assistant");
       const prompt = createFaithfulnessReasonPrompt({
-        input: chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "",
-        output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
-        context: run.output.find(({ role }) => role === "assistant")?.toolInvocations?.map((toolCall) => JSON.stringify(toolCall)) || [],
+        input: chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "",
+        output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
+        context: assistantMessage?.content?.toolInvocations?.map((toolCall) => JSON.stringify(toolCall)) || [],
         score,
         scale: options?.scale || 1,
         verdicts: results.analyzeStepResult?.verdicts || []
@@ -848,7 +865,8 @@ ${biases.join("\n")}
 
 // src/scorers/llm/bias/index.ts
 function createBiasScorer({ model, options }) {
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "bias-scorer",
     name: "Bias Scorer",
     description: "A scorer that evaluates the bias of an LLM output to an input",
     judge: {
@@ -861,13 +879,13 @@ function createBiasScorer({ model, options }) {
     outputSchema: zod.z.object({
       opinions: zod.z.array(zod.z.string())
     }),
-    createPrompt: ({ run }) => createBiasExtractPrompt({ output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" })
+    createPrompt: ({ run }) => createBiasExtractPrompt({ output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" })
   }).analyze({
     description: "Score the relevance of the statements to the input",
     outputSchema: zod.z.object({ results: zod.z.array(zod.z.object({ result: zod.z.string(), reason: zod.z.string() })) }),
     createPrompt: ({ run, results }) => {
       const prompt = createBiasAnalyzePrompt({
-        output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
+        output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
         opinions: results.preprocessStepResult?.opinions || []
       });
       return prompt;
@@ -878,7 +896,7 @@ function createBiasScorer({ model, options }) {
     }
     const biasedVerdicts = results.analyzeStepResult.results.filter((v) => v.result.toLowerCase() === "yes");
     const score = biasedVerdicts.length / results.analyzeStepResult.results.length;
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(score * (options?.scale || 1));
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(score * (options?.scale || 1));
   }).generateReason({
     description: "Reason about the results",
     createPrompt: ({ score, results }) => {
@@ -1082,7 +1100,8 @@ function createHallucinationScorer({
   model,
   options
 }) {
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "hallucination-scorer",
     name: "Hallucination Scorer",
     description: "A scorer that evaluates the hallucination of an LLM output to an input",
     judge: {
@@ -1096,7 +1115,7 @@ function createHallucinationScorer({
       claims: zod.z.array(zod.z.string())
     }),
     createPrompt: ({ run }) => {
-      const prompt = createHallucinationExtractPrompt({ output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" });
+      const prompt = createHallucinationExtractPrompt({ output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "" });
       return prompt;
     }
   }).analyze({
@@ -1118,13 +1137,13 @@ function createHallucinationScorer({
       return 0;
     }
     const score = contradictedStatements / totalStatements * (options?.scale || 1);
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(score);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(score);
   }).generateReason({
     description: "Reason about the results",
     createPrompt: ({ run, results, score }) => {
       const prompt = createHallucinationReasonPrompt({
-        input: chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "",
-        output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
+        input: chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "",
+        output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "",
         context: options?.context || [],
         score,
         scale: options?.scale || 1,
@@ -1224,7 +1243,8 @@ function createToxicityScorer({
   model,
   options
 }) {
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "toxicity-scorer",
     name: "Toxicity Scorer",
     description: "A scorer that evaluates the toxicity of an LLM output to an input",
     judge: {
@@ -1237,8 +1257,8 @@ function createToxicityScorer({
     outputSchema: zod.z.object({ verdicts: zod.z.array(zod.z.object({ verdict: zod.z.string(), reason: zod.z.string() })) }),
     createPrompt: ({ run }) => {
       const prompt = createToxicityAnalyzePrompt({
-        input: chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "",
-        output: chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? ""
+        input: chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "",
+        output: chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? ""
       });
       return prompt;
     }
@@ -1254,7 +1274,7 @@ function createToxicityScorer({
       }
     }
     const score = toxicityCount / numberOfVerdicts;
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(score * (options?.scale || 1));
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(score * (options?.scale || 1));
   }).generateReason({
     description: "Reason about the results",
     createPrompt: ({ results, score }) => {
@@ -1373,7 +1393,8 @@ var analyzeOutputSchema2 = zod.z.object({
 });
 function createToolCallAccuracyScorerLLM({ model, availableTools }) {
   const toolDefinitions = availableTools.map((tool) => `${tool.id}: ${tool.description}`).join("\n");
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "llm-tool-call-accuracy-scorer",
     name: "Tool Call Accuracy (LLM)",
     description: "Evaluates whether an agent selected appropriate tools for the given task using LLM analysis",
     judge: {
@@ -1387,7 +1408,7 @@ function createToolCallAccuracyScorerLLM({ model, availableTools }) {
     if (isInputInvalid || isOutputInvalid) {
       throw new Error("Input and output messages cannot be null or empty");
     }
-    const { tools: actualTools, toolCallInfos } = chunkQKR2PMLZ_cjs.extractToolCalls(run.output);
+    const { tools: actualTools, toolCallInfos } = chunkTPQLLHZW_cjs.extractToolCalls(run.output);
     return {
       actualTools,
       hasToolCalls: actualTools.length > 0,
@@ -1397,8 +1418,8 @@ function createToolCallAccuracyScorerLLM({ model, availableTools }) {
     description: "Analyze the appropriateness of tool selections",
     outputSchema: analyzeOutputSchema2,
     createPrompt: ({ run, results }) => {
-      const userInput = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
-      const agentResponse = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const userInput = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const agentResponse = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       const toolsCalled = results.preprocessStepResult?.actualTools || [];
       return createAnalyzePrompt2({
         userInput,
@@ -1415,11 +1436,11 @@ function createToolCallAccuracyScorerLLM({ model, availableTools }) {
     }
     const appropriateToolCalls = evaluations.filter((e) => e.wasAppropriate).length;
     const totalToolCalls = evaluations.length;
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(appropriateToolCalls / totalToolCalls);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(appropriateToolCalls / totalToolCalls);
   }).generateReason({
     description: "Generate human-readable explanation of tool selection evaluation",
     createPrompt: ({ run, results, score }) => {
-      const userInput = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const userInput = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
       const evaluations = results.analyzeStepResult?.evaluations || [];
       const missingTools = results.analyzeStepResult?.missingTools || [];
       return createReasonPrompt3({
@@ -1611,7 +1632,8 @@ function createContextRelevanceScorerLLM({
   if (options.context && options.context.length === 0) {
     throw new Error("Context array cannot be empty if provided");
   }
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "context-relevance-scorer",
     name: "Context Relevance (LLM)",
     description: "Evaluates how relevant and useful the provided context was for generating the agent response",
     judge: {
@@ -1623,8 +1645,8 @@ function createContextRelevanceScorerLLM({
     description: "Analyze the relevance and utility of provided context",
     outputSchema: analyzeOutputSchema3,
     createPrompt: ({ run }) => {
-      const userQuery = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
-      const agentResponse = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const userQuery = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const agentResponse = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
       if (context.length === 0) {
         return createAnalyzePrompt3({
@@ -1672,11 +1694,11 @@ function createContextRelevanceScorerLLM({
     const missingContextPenalty = Math.min(missingContext.length * missingPenaltyRate, maxMissingPenalty);
     const finalScore = Math.max(0, relevanceScore - usagePenalty - missingContextPenalty);
     const scaledScore = finalScore * (options.scale || 1);
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(scaledScore);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(scaledScore);
   }).generateReason({
     description: "Generate human-readable explanation of context relevance evaluation",
     createPrompt: ({ run, results, score }) => {
-      const userQuery = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const userQuery = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
       const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
       if (context.length === 0) {
         return `No context was available for evaluation. The agent response was generated without any supporting context. Score: ${score}`;
@@ -1834,7 +1856,8 @@ function createContextPrecisionScorer({
   if (options.context && options.context.length === 0) {
     throw new Error("Context array cannot be empty if provided");
   }
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "context-precision-scorer",
     name: "Context Precision Scorer",
     description: "A scorer that evaluates the relevance and precision of retrieved context nodes for generating expected outputs",
     judge: {
@@ -1846,8 +1869,8 @@ function createContextPrecisionScorer({
     description: "Evaluate the relevance of each context piece for generating the expected output",
     outputSchema: contextRelevanceOutputSchema,
     createPrompt: ({ run }) => {
-      const input = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
-      const output = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const input = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const output = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
       if (context.length === 0) {
         throw new Error("No context available for evaluation");
@@ -1880,12 +1903,12 @@ function createContextPrecisionScorer({
     }
     const map = sumPrecision / relevantCount;
     const score = map * (options.scale || 1);
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(score);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(score);
   }).generateReason({
     description: "Reason about the context precision results",
     createPrompt: ({ run, results, score }) => {
-      const input = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
-      const output = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const input = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const output = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
       return createContextPrecisionReasonPrompt({
         input,
@@ -2126,7 +2149,8 @@ function createNoiseSensitivityScorerLLM({
   if (!options.baselineResponse || !options.noisyQuery) {
     throw new Error("Both baselineResponse and noisyQuery are required for Noise Sensitivity scoring");
   }
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "noise-sensitivity-scorer",
     name: "Noise Sensitivity (LLM)",
     description: "Evaluates how robust an agent is when exposed to irrelevant, distracting, or misleading information",
     judge: {
@@ -2138,8 +2162,8 @@ function createNoiseSensitivityScorerLLM({
     description: "Analyze the impact of noise on agent response quality",
     outputSchema: analyzeOutputSchema4,
     createPrompt: ({ run }) => {
-      const originalQuery = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
-      const noisyResponse = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const originalQuery = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const noisyResponse = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       if (!originalQuery || !noisyResponse) {
         throw new Error("Both original query and noisy response are required for evaluation");
       }
@@ -2182,11 +2206,11 @@ function createNoiseSensitivityScorerLLM({
     const majorIssues = analysisResult.majorIssues || [];
     const issuesPenalty = Math.min(majorIssues.length * majorIssuePenaltyRate, maxMajorIssuePenalty);
     finalScore = Math.max(0, finalScore - issuesPenalty);
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(finalScore);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(finalScore);
   }).generateReason({
     description: "Generate human-readable explanation of noise sensitivity evaluation",
     createPrompt: ({ run, results, score }) => {
-      const originalQuery = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const originalQuery = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
       const analysisResult = results.analyzeStepResult;
       if (!analysisResult) {
         throw new Error("Analysis step failed to produce results for reason generation");
@@ -2498,7 +2522,8 @@ function createPromptAlignmentScorerLLM({
 }) {
   const scale = options?.scale || 1;
   const evaluationMode = options?.evaluationMode || "both";
-  return scores.createScorer({
+  return evals.createScorer({
+    id: "prompt-alignment-scorer",
     name: "Prompt Alignment (LLM)",
     description: "Evaluates how well the agent response aligns with the intent and requirements of the user prompt",
     judge: {
@@ -2509,9 +2534,9 @@ function createPromptAlignmentScorerLLM({
     description: "Analyze prompt-response alignment across multiple dimensions",
     outputSchema: analyzeOutputSchema5,
     createPrompt: ({ run }) => {
-      const userPrompt = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
-      const systemPrompt = chunkQKR2PMLZ_cjs.getCombinedSystemPrompt(run.input) ?? "";
-      const agentResponse = chunkQKR2PMLZ_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
+      const userPrompt = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const systemPrompt = chunkTPQLLHZW_cjs.getCombinedSystemPrompt(run.input) ?? "";
+      const agentResponse = chunkTPQLLHZW_cjs.getAssistantMessageFromRunOutput(run.output) ?? "";
       if (evaluationMode === "user" && !userPrompt) {
         throw new Error("User prompt is required for user prompt alignment scoring");
       }
@@ -2547,12 +2572,12 @@ function createPromptAlignmentScorerLLM({
       weightedScore = userScore * SCORING_WEIGHTS.BOTH.USER_WEIGHT + systemScore * SCORING_WEIGHTS.BOTH.SYSTEM_WEIGHT;
     }
     const finalScore = weightedScore * scale;
-    return chunkQKR2PMLZ_cjs.roundToTwoDecimals(finalScore);
+    return chunkTPQLLHZW_cjs.roundToTwoDecimals(finalScore);
   }).generateReason({
     description: "Generate human-readable explanation of prompt alignment evaluation",
     createPrompt: ({ run, results, score }) => {
-      const userPrompt = chunkQKR2PMLZ_cjs.getUserMessageFromRunInput(run.input) ?? "";
-      const systemPrompt = chunkQKR2PMLZ_cjs.getCombinedSystemPrompt(run.input) ?? "";
+      const userPrompt = chunkTPQLLHZW_cjs.getUserMessageFromRunInput(run.input) ?? "";
+      const systemPrompt = chunkTPQLLHZW_cjs.getCombinedSystemPrompt(run.input) ?? "";
       const analysis = results.analyzeStepResult;
       if (!analysis) {
         return `Unable to analyze prompt alignment. Score: ${score}`;
@@ -2568,6 +2593,392 @@ function createPromptAlignmentScorerLLM({
     }
   });
 }
+function normalizeString(str) {
+  return str.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase();
+}
+function extractElements(doc) {
+  const nouns = doc.nouns().out("array") || [];
+  const verbs = doc.verbs().toInfinitive().out("array") || [];
+  const topics = doc.topics().out("array") || [];
+  const terms = doc.terms().out("array") || [];
+  const cleanAndSplitTerm = (term) => {
+    const normalized = normalizeString(term);
+    return normalized.replace(/([a-z])([A-Z])/g, "$1 $2").replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter((word) => word.length > 0);
+  };
+  const processedTerms = [
+    ...nouns.flatMap(cleanAndSplitTerm),
+    ...verbs.flatMap(cleanAndSplitTerm),
+    ...topics.flatMap(cleanAndSplitTerm),
+    ...terms.flatMap(cleanAndSplitTerm)
+  ];
+  return [...new Set(processedTerms)];
+}
+function calculateCoverage({ original, simplified }) {
+  if (original.length === 0) {
+    return simplified.length === 0 ? 1 : 0;
+  }
+  const covered = original.filter(
+    (element) => simplified.some((s) => {
+      const elem = normalizeString(element);
+      const simp = normalizeString(s);
+      if (elem.length <= 3) {
+        return elem === simp;
+      }
+      const longer = elem.length > simp.length ? elem : simp;
+      const shorter = elem.length > simp.length ? simp : elem;
+      if (longer.includes(shorter)) {
+        return shorter.length / longer.length > 0.6;
+      }
+      return false;
+    })
+  );
+  return covered.length / original.length;
+}
+function createCompletenessScorer() {
+  return evals.createScorer({
+    id: "completeness-scorer",
+    name: "Completeness Scorer",
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const isInputInvalid = !run.input || run.input.inputMessages.some((i) => {
+      const content = chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i);
+      return content === null || content === void 0;
+    });
+    const isOutputInvalid = !run.output || run.output.some((i) => {
+      const content = chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i);
+      return content === null || content === void 0;
+    });
+    if (isInputInvalid || isOutputInvalid) {
+      throw new Error("Inputs cannot be null or undefined");
+    }
+    const input = run.input?.inputMessages.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    const output = run.output?.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    const inputToProcess = input;
+    const outputToProcess = output;
+    const inputDoc = nlp__default.default(inputToProcess.trim());
+    const outputDoc = nlp__default.default(outputToProcess.trim());
+    const inputElements = extractElements(inputDoc);
+    const outputElements = extractElements(outputDoc);
+    return {
+      inputElements,
+      outputElements,
+      missingElements: inputElements.filter((e) => !outputElements.includes(e)),
+      elementCounts: {
+        input: inputElements.length,
+        output: outputElements.length
+      }
+    };
+  }).generateScore(({ results }) => {
+    const inputElements = results.preprocessStepResult?.inputElements;
+    const outputElements = results.preprocessStepResult?.outputElements;
+    return calculateCoverage({
+      original: inputElements,
+      simplified: outputElements
+    });
+  });
+}
+function calculateRatio(input, output) {
+  if (input === output) {
+    return 1;
+  }
+  if (input.length === 0 || output.length === 0) {
+    return 0;
+  }
+  const matches = longestCommonSubsequence(input, output);
+  const total = input.length + output.length;
+  return total > 0 ? 2 * matches / total : 0;
+}
+function longestCommonSubsequence(str1, str2) {
+  const m = str1.length;
+  const n = str2.length;
+  const dp = [];
+  for (let i = 0; i <= m; i++) {
+    dp[i] = [];
+    for (let j = 0; j <= n; j++) {
+      dp[i][j] = 0;
+    }
+  }
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      if (str1[i - 1] === str2[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1] + 1;
+      } else {
+        dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
+      }
+    }
+  }
+  return dp[m][n];
+}
+function countChanges(input, output) {
+  const inputNormalized = input.replace(/\s+/g, " ").trim();
+  const outputNormalized = output.replace(/\s+/g, " ").trim();
+  if (inputNormalized === outputNormalized) {
+    if (input !== output) {
+      const inputWords2 = input.split(/\s+/).filter((w) => w.length > 0);
+      const outputWords2 = output.split(/\s+/).filter((w) => w.length > 0);
+      return Math.abs(inputWords2.length - outputWords2.length) || 1;
+    }
+    return 0;
+  }
+  const inputWords = inputNormalized.split(/\s+/).filter((w) => w.length > 0);
+  const outputWords = outputNormalized.split(/\s+/).filter((w) => w.length > 0);
+  if (inputWords.length === 0 && outputWords.length === 0) {
+    return 0;
+  }
+  if (inputWords.length === 0) {
+    return outputWords.length;
+  }
+  if (outputWords.length === 0) {
+    return inputWords.length;
+  }
+  const matchingWords = findCommonWords(inputWords, outputWords);
+  const maxLength = Math.max(inputWords.length, outputWords.length);
+  const changes = maxLength - matchingWords;
+  return changes;
+}
+function findCommonWords(arr1, arr2) {
+  let matches = 0;
+  const used = /* @__PURE__ */ new Set();
+  for (let i = 0; i < arr1.length; i++) {
+    for (let j = 0; j < arr2.length; j++) {
+      if (!used.has(j) && arr1[i] === arr2[j]) {
+        matches++;
+        used.add(j);
+        break;
+      }
+    }
+  }
+  return matches;
+}
+function createTextualDifferenceScorer() {
+  return evals.createScorer({
+    id: "textual-difference-scorer",
+    name: "Textual Difference Scorer",
+    description: "Calculate textual difference between input and output using sequence matching algorithms.",
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const input = run.input?.inputMessages?.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    const output = run.output?.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    const ratio = calculateRatio(input, output);
+    const changes = countChanges(input, output);
+    const maxLength = Math.max(input.length, output.length);
+    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
+    const confidence = 1 - lengthDiff;
+    return {
+      ratio,
+      confidence,
+      changes,
+      lengthDiff
+    };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.ratio;
+  });
+}
+function createKeywordCoverageScorer() {
+  return evals.createScorer({
+    id: "keyword-coverage-scorer",
+    name: "Keyword Coverage Scorer",
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const input = run.input?.inputMessages?.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    const output = run.output?.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    if (!input && !output) {
+      return {
+        result: {
+          referenceKeywords: /* @__PURE__ */ new Set(),
+          responseKeywords: /* @__PURE__ */ new Set()
+        }
+      };
+    }
+    const extractKeywords = (text) => {
+      return keyword_extractor__default.default.extract(text, {
+        language: "english",
+        remove_digits: true,
+        return_changed_case: true,
+        remove_duplicates: true
+      });
+    };
+    const referenceKeywords = new Set(extractKeywords(input));
+    const responseKeywords = new Set(extractKeywords(output));
+    return {
+      referenceKeywords,
+      responseKeywords
+    };
+  }).analyze(async ({ results }) => {
+    if (!results.preprocessStepResult?.referenceKeywords?.size && !results.preprocessStepResult?.responseKeywords?.size) {
+      return {
+        totalKeywordsLength: 0,
+        matchedKeywordsLength: 0
+      };
+    }
+    const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(
+      (k) => results.preprocessStepResult?.responseKeywords?.has(k)
+    );
+    return {
+      totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,
+      matchedKeywordsLength: matchedKeywords.length ?? 0
+    };
+  }).generateScore(({ results }) => {
+    if (!results.analyzeStepResult?.totalKeywordsLength) {
+      return 1;
+    }
+    const totalKeywords = results.analyzeStepResult?.totalKeywordsLength;
+    const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength;
+    return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;
+  });
+}
+function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { ignoreCase: true, ignoreWhitespace: true }) {
+  return evals.createScorer({
+    id: "content-similarity-scorer",
+    name: "Content Similarity Scorer",
+    description: "Calculates content similarity between input and output messages using string comparison algorithms.",
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let processedInput = run.input?.inputMessages.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    let processedOutput = run.output.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    if (ignoreCase) {
+      processedInput = processedInput.toLowerCase();
+      processedOutput = processedOutput.toLowerCase();
+    }
+    if (ignoreWhitespace) {
+      processedInput = processedInput.replace(/\s+/g, " ").trim();
+      processedOutput = processedOutput.replace(/\s+/g, " ").trim();
+    }
+    return {
+      processedInput,
+      processedOutput
+    };
+  }).generateScore(({ results }) => {
+    const similarity = stringSimilarity__default.default.compareTwoStrings(
+      results.preprocessStepResult?.processedInput,
+      results.preprocessStepResult?.processedOutput
+    );
+    return similarity;
+  });
+}
+function createToneScorer(config = {}) {
+  const { referenceTone } = config;
+  return evals.createScorer({
+    id: "tone-scorer",
+    name: "Tone Scorer",
+    description: "Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.",
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const sentiment = new Sentiment__default.default();
+    const agentMessage = run.output?.map((i) => chunkTPQLLHZW_cjs.getTextContentFromMastraDBMessage(i)).join(", ") || "";
+    const responseSentiment = sentiment.analyze(agentMessage);
+    if (referenceTone) {
+      const referenceSentiment = sentiment.analyze(referenceTone);
+      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);
+      const normalizedScore = Math.max(0, 1 - sentimentDiff);
+      return {
+        score: normalizedScore,
+        responseSentiment: responseSentiment.comparative,
+        referenceSentiment: referenceSentiment.comparative,
+        difference: sentimentDiff
+      };
+    }
+    const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];
+    const sentiments = sentences.map((s) => sentiment.analyze(s).comparative);
+    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;
+    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;
+    const stability = Math.max(0, 1 - variance);
+    return {
+      score: stability,
+      avgSentiment,
+      sentimentVariance: variance
+    };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.score;
+  });
+}
+function checkToolOrder(actualTools, expectedOrder, strictMode = false) {
+  if (strictMode) {
+    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);
+  }
+  const expectedIndices = [];
+  for (const expectedTool of expectedOrder) {
+    const index = actualTools.indexOf(expectedTool);
+    if (index === -1) {
+      return false;
+    }
+    expectedIndices.push(index);
+  }
+  for (let i = 1; i < expectedIndices.length; i++) {
+    const currentIndex = expectedIndices[i];
+    const prevIndex = expectedIndices[i - 1];
+    if (currentIndex !== void 0 && prevIndex !== void 0 && currentIndex <= prevIndex) {
+      return false;
+    }
+  }
+  return true;
+}
+function calculateAccuracy({
+  expectedTool,
+  actualTools,
+  strictMode = false,
+  expectedToolOrder
+}) {
+  if (actualTools.length === 0) {
+    return 0;
+  }
+  if (expectedToolOrder && expectedToolOrder.length > 0) {
+    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;
+  }
+  if (!expectedTool) {
+    return 0;
+  }
+  if (strictMode) {
+    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;
+  }
+  return actualTools.includes(expectedTool) ? 1 : 0;
+}
+function createToolCallAccuracyScorerCode(options) {
+  const { expectedTool, strictMode = false, expectedToolOrder } = options;
+  if (!expectedTool && !expectedToolOrder) {
+    throw new Error("Either expectedTool or expectedToolOrder must be provided");
+  }
+  const getDescription = () => {
+    return expectedToolOrder ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(", ")}]` : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;
+  };
+  return evals.createScorer({
+    id: "code-tool-call-accuracy-scorer",
+    name: "Tool Call Accuracy Scorer",
+    description: getDescription(),
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;
+    const isOutputInvalid = !run.output || run.output.length === 0;
+    if (isInputInvalid || isOutputInvalid) {
+      throw new Error("Input and output messages cannot be null or empty");
+    }
+    const { tools: actualTools, toolCallInfos } = chunkTPQLLHZW_cjs.extractToolCalls(run.output);
+    const correctToolCalled = expectedTool ? strictMode ? actualTools.length === 1 && actualTools[0] === expectedTool : actualTools.includes(expectedTool) : false;
+    return {
+      expectedTool,
+      actualTools,
+      strictMode,
+      expectedToolOrder,
+      hasToolCalls: actualTools.length > 0,
+      correctToolCalled,
+      toolCallInfos,
+      correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null
+    };
+  }).generateScore(({ results }) => {
+    const preprocessResult = results.preprocessStepResult;
+    if (!preprocessResult) {
+      return 0;
+    }
+    return calculateAccuracy({
+      expectedTool: preprocessResult.expectedTool,
+      actualTools: preprocessResult.actualTools,
+      strictMode: preprocessResult.strictMode,
+      expectedToolOrder: preprocessResult.expectedToolOrder
+    });
+  });
+}
 
 exports.ANSWER_RELEVANCY_AGENT_INSTRUCTIONS = ANSWER_RELEVANCY_AGENT_INSTRUCTIONS;
 exports.ANSWER_SIMILARITY_DEFAULT_OPTIONS = ANSWER_SIMILARITY_DEFAULT_OPTIONS;
@@ -2576,12 +2987,18 @@ exports.DEFAULT_OPTIONS = DEFAULT_OPTIONS;
 exports.createAnswerRelevancyScorer = createAnswerRelevancyScorer;
 exports.createAnswerSimilarityScorer = createAnswerSimilarityScorer;
 exports.createBiasScorer = createBiasScorer;
+exports.createCompletenessScorer = createCompletenessScorer;
+exports.createContentSimilarityScorer = createContentSimilarityScorer;
 exports.createContextPrecisionScorer = createContextPrecisionScorer;
 exports.createContextRelevanceScorerLLM = createContextRelevanceScorerLLM;
 exports.createFaithfulnessScorer = createFaithfulnessScorer;
 exports.createHallucinationScorer = createHallucinationScorer;
+exports.createKeywordCoverageScorer = createKeywordCoverageScorer;
 exports.createNoiseSensitivityScorerLLM = createNoiseSensitivityScorerLLM;
 exports.createPromptAlignmentScorerLLM = createPromptAlignmentScorerLLM;
+exports.createTextualDifferenceScorer = createTextualDifferenceScorer;
+exports.createToneScorer = createToneScorer;
+exports.createToolCallAccuracyScorerCode = createToolCallAccuracyScorerCode;
 exports.createToolCallAccuracyScorerLLM = createToolCallAccuracyScorerLLM;
 exports.createToxicityScorer = createToxicityScorer;
 //# sourceMappingURL=index.cjs.map