@mastra/evals 1.5.1 → 1.6.0-alpha.0

package/dist/scorers/prebuilt/index.cjs

@@ -2306,6 +2306,266 @@ function createContextPrecisionScorer({
   });
 }
 
+// src/scorers/llm/context-recall/prompts.ts
+var CONTEXT_RECALL_AGENT_INSTRUCTIONS = `You are a precise context recall evaluator. Your job is to determine if the retrieved context contains enough information to support all the claims in a ground-truth reference answer.
+
+Key Principles:
+1. First extract all atomic claims/sentences from the ground-truth answer
+2. Then verify each claim against the provided retrieval context
+3. A claim is attributable if any part of the retrieval context directly supports it
+4. A claim is not attributable if no part of the retrieval context mentions or implies it
+5. Focus on information coverage, not exact wording
+6. Be strict in attribution - the context must actually contain supporting information
+7. Never use prior knowledge in judgments - only use the provided context`;
+function createClaimExtractionPrompt({ groundTruth }) {
+  return `Extract all atomic claims from the given ground-truth answer. A claim is any single statement that asserts one piece of information.
+
+Guidelines for claim extraction:
+- Break down compound statements into individual claims
+- Each claim should be self-contained and verifiable independently
+- Include all factual assertions, including numbers, dates, and quantities
+- Keep relationships between entities intact within each claim
+- Exclude questions, commands, and purely stylistic text
+- Preserve the original meaning without adding interpretation
+
+Example:
+Text: "Albert Einstein was born on 14 March 1879 in Ulm, Germany. He developed the theory of relativity and won the Nobel Prize in Physics in 1921."
+
+{
+    "claims": [
+        "Albert Einstein was born on 14 March 1879",
+        "Albert Einstein was born in Ulm, Germany",
+        "Albert Einstein developed the theory of relativity",
+        "Albert Einstein won the Nobel Prize in Physics",
+        "Albert Einstein won the Nobel Prize in 1921"
+    ]
+}
+
+Please return only JSON format with "claims" array.
+Return empty list for empty input.
+
+Ground-Truth Answer:
+${groundTruth}
+
+JSON:
+`;
+}
+function createClaimAttributionPrompt({ claims, context }) {
+  return `Determine if each claim from the ground-truth answer can be attributed to the provided retrieval context. A claim is attributable if any part of the context directly supports or contains the information in that claim.
+
+Retrieval Context:
+${context.map((ctx, index) => `[${index}] ${ctx}`).join("\n")}
+
+Number of claims: ${claims.length}
+
+Claims to verify:
+${claims.map((claim, index) => `[${index}] ${claim}`).join("\n")}
+
+For each claim, provide a verdict and reasoning. The verdict must be one of:
+- "yes" if the claim can be attributed to information in the retrieval context
+- "no" if the claim cannot be attributed to any part of the retrieval context
+
+The number of verdicts MUST MATCH the number of claims exactly.
+
+Format:
+{
+    "verdicts": [
+        {
+            "verdict": "yes/no",
+            "reason": "explanation of why this claim is or isn't attributable to the context"
+        }
+    ]
+}
+
+Rules:
+- Only use information from the provided retrieval context
+- Mark claims as "yes" if the context contains supporting information, even if the wording differs
+- Mark claims as "no" if the context does not mention or imply the information
+- Never use prior knowledge in your judgment
+- Provide clear reasoning that references specific context pieces
+
+Example:
+Context:
+[0] "Albert Einstein was born on 14 March 1879 in Ulm, Germany."
+[1] "Einstein published his theory of special relativity in 1905."
+
+Claims:
+[0] "Albert Einstein was born on 14 March 1879"
+[1] "Albert Einstein won the Nobel Prize in 1921"
+
+{
+    "verdicts": [
+        {
+            "verdict": "yes",
+            "reason": "Context node [0] explicitly states Einstein was born on 14 March 1879"
+        },
+        {
+            "verdict": "no",
+            "reason": "No part of the retrieval context mentions Einstein winning the Nobel Prize"
+        }
+    ]
+}`;
+}
+function createContextRecallReasonPrompt({
+  groundTruth,
+  context,
+  score,
+  scale,
+  verdicts
+}) {
+  return `Explain the context recall score for the retrieval context based on how well it covers the claims in the ground-truth answer.
+
+Ground-Truth Answer:
+${groundTruth}
+
+Retrieval Context:
+${context.map((ctx, index) => `[${index}] ${ctx}`).join("\n")}
+
+Score: ${score} out of ${scale}
+Verdicts:
+${JSON.stringify(verdicts, null, 2)}
+
+Context Recall measures what fraction of the ground-truth answer's claims are supported by the retrieved context. The score is calculated as:
+- Extract all claims from the ground-truth answer
+- Check each claim against the retrieval context
+- Score = (attributed claims / total claims) \xD7 scale
+
+Rules for explanation:
+- Explain which claims were and were not found in the context
+- Mention specific context pieces that supported attributed claims
+- Highlight what information was missing for unattributed claims
+- Keep explanation concise and focused on coverage gaps
+- Use the given score, don't recalculate
+
+Format:
+"The score is ${score} because {explanation of context recall}"
+
+Example responses:
+"The score is 0.67 because 2 out of 3 ground-truth claims are supported by the retrieval context. The claims about Einstein's birthdate and birthplace are covered by context node [0], but the claim about the Nobel Prize is not mentioned in any context node."
+"The score is 1.0 because all claims in the ground-truth answer are fully supported by the retrieval context."
+"The score is 0.0 because none of the ground-truth claims could be attributed to the retrieval context."`;
+}
+
+// src/scorers/llm/context-recall/index.ts
+var claimExtractionOutputSchema = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "claims": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": [
+    "claims"
+  ]
+};
+var claimAttributionOutputSchema = {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "verdicts": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "verdict": {
+            "type": "string"
+          },
+          "reason": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "verdict",
+          "reason"
+        ]
+      }
+    }
+  },
+  "required": [
+    "verdicts"
+  ]
+};
+var getContext3 = ({
+  input,
+  output,
+  options
+}) => {
+  if (options.contextExtractor && chunkIZLA36WC_cjs.isScorerRunInputForAgent(input) && chunkIZLA36WC_cjs.isScorerRunOutputForAgent(output)) {
+    return options.contextExtractor(input, output);
+  }
+  return options.context ?? [];
+};
+function createContextRecallScorer({
+  model,
+  options
+}) {
+  if (!options.context && !options.contextExtractor) {
+    throw new Error("Either context or contextExtractor is required for Context Recall scoring");
+  }
+  if (options.context && options.context.length === 0) {
+    throw new Error("Context array cannot be empty if provided");
+  }
+  return evals.createScorer({
+    id: "context-recall-scorer",
+    name: "Context Recall Scorer",
+    description: "A scorer that evaluates how well retrieved context covers the claims in a ground-truth reference answer",
+    judge: {
+      model,
+      instructions: CONTEXT_RECALL_AGENT_INSTRUCTIONS
+    },
+    type: "agent"
+  }).preprocess({
+    description: "Extract atomic claims from the ground-truth answer",
+    outputSchema: claimExtractionOutputSchema,
+    createPrompt: ({ run }) => {
+      if (!run.groundTruth) {
+        return createClaimExtractionPrompt({ groundTruth: "" });
+      }
+      const groundTruth = typeof run.groundTruth === "string" ? run.groundTruth : JSON.stringify(run.groundTruth);
+      return createClaimExtractionPrompt({ groundTruth });
+    }
+  }).analyze({
+    description: "Check if each ground-truth claim is attributable to the retrieval context",
+    outputSchema: claimAttributionOutputSchema,
+    createPrompt: ({ results, run }) => {
+      const context = getContext3({ input: run.input, output: run.output, options });
+      return createClaimAttributionPrompt({
+        claims: results.preprocessStepResult?.claims || [],
+        context
+      });
+    }
+  }).generateScore(({ results, run }) => {
+    if (!run.groundTruth) {
+      return 0;
+    }
+    const totalClaims = results.preprocessStepResult?.claims?.length ?? 0;
+    if (totalClaims === 0) {
+      return 0;
+    }
+    const verdicts = results.analyzeStepResult?.verdicts || [];
+    const attributedClaims = verdicts.filter((v) => v.verdict.toLowerCase().trim() === "yes").length;
+    const score = Math.min(1, attributedClaims / totalClaims) * (options.scale || 1);
+    return chunkIZLA36WC_cjs.roundToTwoDecimals(score);
+  }).generateReason({
+    description: "Explain the context recall evaluation results",
+    createPrompt: ({ run, results, score }) => {
+      const groundTruth = run.groundTruth ? typeof run.groundTruth === "string" ? run.groundTruth : JSON.stringify(run.groundTruth) : "";
+      const context = getContext3({ input: run.input, output: run.output, options });
+      return createContextRecallReasonPrompt({
+        groundTruth,
+        context,
+        score,
+        scale: options.scale || 1,
+        verdicts: results.analyzeStepResult?.verdicts || []
+      });
+    }
+  });
+}
+
 // src/scorers/llm/noise-sensitivity/prompts.ts
 var NOISE_SENSITIVITY_INSTRUCTIONS = `You are an expert noise sensitivity evaluator. Your job is to analyze how much irrelevant, distracting, or misleading information (noise) affected the agent's response quality and accuracy.
 
@@ -4361,6 +4621,7 @@ exports.createBiasScorer = createBiasScorer;
 exports.createCompletenessScorer = createCompletenessScorer;
 exports.createContentSimilarityScorer = createContentSimilarityScorer;
 exports.createContextPrecisionScorer = createContextPrecisionScorer;
+exports.createContextRecallScorer = createContextRecallScorer;
 exports.createContextRelevanceScorerLLM = createContextRelevanceScorerLLM;
 exports.createFaithfulnessScorer = createFaithfulnessScorer;
 exports.createHallucinationScorer = createHallucinationScorer;