@mastra/evals 0.10.1 → 0.10.2-alpha.0

package/dist/metrics/llm/index.cjs

@@ -1806,9 +1806,9 @@ function generateEvaluatePrompt9({
   output,
   context
 }) {
-  return `For EACH sentence in the given expected output below, determine whether the sentence can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: \`verdict\` and \`reason\`.
-The "verdict" key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the sentence can be attributed to any parts of the retrieval context, else answer 'no'.
-The "reason" key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said sentence. You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
+  return `For EACH context node provided below, determine whether the information in that node was used in the given output. Please generate a list of JSON with two keys: \`verdict\` and \`reason\`.
+The "verdict" key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the context node was used in the output, else answer 'no'.
+The "reason" key should provide a brief explanation for the verdict. If the context was used, quote the specific part of the output that relates to this context node, keeping it concise and using an ellipsis if needed.
 
 **
 IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: \`verdict\` and \`reason\`.
@@ -1823,17 +1823,18 @@ IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' k
     ]  
 }}
 
-Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of sentences in of \`expected output\`.
+The number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of context nodes provided.
 **
 
 input:
 ${input}
 
-Expected Output:
+Output to evaluate:
 ${output}
 
-Retrieval Context:
-${context}
+Context Nodes:
+${context.map((node, i) => `--- Node ${i + 1} ---
+${node}`).join("\n\n")}
 `;
 }
 function generateReasonPrompt8({

package/dist/metrics/llm/index.js

@@ -1804,9 +1804,9 @@ function generateEvaluatePrompt9({
   output,
   context
 }) {
-  return `For EACH sentence in the given expected output below, determine whether the sentence can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: \`verdict\` and \`reason\`.
-The "verdict" key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the sentence can be attributed to any parts of the retrieval context, else answer 'no'.
-The "reason" key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said sentence. You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
+  return `For EACH context node provided below, determine whether the information in that node was used in the given output. Please generate a list of JSON with two keys: \`verdict\` and \`reason\`.
+The "verdict" key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the context node was used in the output, else answer 'no'.
+The "reason" key should provide a brief explanation for the verdict. If the context was used, quote the specific part of the output that relates to this context node, keeping it concise and using an ellipsis if needed.
 
 **
 IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: \`verdict\` and \`reason\`.
@@ -1821,17 +1821,18 @@ IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' k
     ]  
 }}
 
-Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of sentences in of \`expected output\`.
+The number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of context nodes provided.
 **
 
 input:
 ${input}
 
-Expected Output:
+Output to evaluate:
 ${output}
 
-Retrieval Context:
-${context}
+Context Nodes:
+${context.map((node, i) => `--- Node ${i + 1} ---
+${node}`).join("\n\n")}
 `;
 }
 function generateReasonPrompt8({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/evals",
-  "version": "0.10.1",
+  "version": "0.10.2-alpha.0",
   "description": "",
   "type": "module",
   "files": [
@@ -81,7 +81,7 @@
     "typescript": "^5.8.2",
     "vitest": "^3.1.2",
     "@internal/lint": "0.0.7",
-    "@mastra/core": "0.10.1"
+    "@mastra/core": "0.10.2-alpha.1"
   },
   "scripts": {
     "check": "tsc --noEmit",