@mastra/evals 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256

package/dist/chunk-W3U7MMDX.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/scorers/utils.ts"],"names":["requestContext","RequestContext"],"mappings":";;;;;AAyBO,SAAS,kCAAkC,OAAA,EAAkC;AAClF,EAAA,IAAI,OAAO,QAAQ,OAAA,CAAQ,OAAA,KAAY,YAAY,OAAA,CAAQ,OAAA,CAAQ,YAAY,EAAA,EAAI;AACjF,IAAA,OAAO,QAAQ,OAAA,CAAQ,OAAA;AAAA,EACzB;AACA,EAAA,IAAI,OAAA,CAAQ,QAAQ,KAAA,IAAS,KAAA,CAAM,QAAQ,OAAA,CAAQ,OAAA,CAAQ,KAAK,CAAA,EAAG;AAEjE,IAAA,MAAM,SAAA,GAAY,QAAQ,OAAA,CAAQ,KAAA,CAAM,OAAO,CAAA,CAAA,KAAK,CAAA,CAAE,SAAS,MAAM,CAAA;AACrE,IAAA,OAAO,SAAA,CAAU,SAAS,CAAA,GAAI,SAAA,CAAU,UAAU,MAAA,GAAS,CAAC,CAAA,EAAG,IAAA,IAAQ,EAAA,GAAK,EAAA;AAAA,EAC9E;AACA,EAAA,OAAO,EAAA;AACT;AAgBO,IAAM,kBAAA,GAAqB,CAAC,GAAA,KAAgB;AACjD,EAAA,OAAO,KAAK,KAAA,CAAA,CAAO,GAAA,GAAM,MAAA,CAAO,OAAA,IAAW,GAAG,CAAA,GAAI,GAAA;AACpD;AAgBO,SAAS,UAAA,CAAW,KAAA,EAAe,OAAA,EAAiB,OAAA,EAA0B;AACnF,EAAA,OAAO,IAAA,CAAK,IAAI,KAAA,GAAQ,OAAO,IAAI,IAAA,CAAK,GAAA,CAAI,QAAQ,OAAO,CAAA;AAC7D;AA6CO,IAAM,aAAA,GAAgB,CAC3B,KAAA,EACA,MAAA,EACA,mBACA,cAAA,KACiB;AACjB,EAAA,OAAO;AAAA,IACL,OAAO,CAAC,EAAE,MAAM,MAAA,EAAQ,OAAA,EAAS,OAAO,CAAA;AAAA,IACxC,MAAA,EAAQ,EAAE,IAAA,EAAM,WAAA,EAAa,MAAM,MAAA,EAAO;AAAA,IAC1C,iBAAA,EAAmB,qBAAqB,EAAC;AAAA,IACzC,cAAA,EAAgB,kBAAkB;AAAC,GACrC;AACF;AAmBO,IAAM,0BAAA,GAA6B,CAAC,KAAA,KAAuD;AAChG,EAAA,MAAM,OAAA,GAAU,OAAO,aAAA,CAAc,IAAA,CAAK,CAAC,EAAE,IAAA,EAAK,KAAM,IAAA,KAAS,MAAM,CAAA;AACvE,EAAA,OAAO,OAAA,GAAU,iCAAA,CAAkC,OAAO,CAAA,GAAI,MAAA;AAChE;AAoBO,IAAM,6BAAA,GAAgC,CAAC,KAAA,KAA6C;AACzF,EAAA,MAAM,iBAA2B,EAAC;AAGlC,EAAA,IAAI,OAAO,cAAA,EAAgB;AACzB,IAAA,cAAA,CAAe,IAAA;AAAA,MACb,GAAG,KAAA,CAAM,cAAA,CACN,GAAA,CAAI,CAAA,GAAA,KAAO;AAEV,QAAA,IAAI,OAAO,GAAA,CAAI,OAAA,KAAY,QAAA,EAAU;AACnC,UAAA,OAAO,GAAA,CAAI,OAAA;AAAA,QACb,CAAA,MAAA,IAAW,KAAA,CAAM,OAAA,CAAQ,GAAA,CAAI,OAAO,CAAA,EAAG;AAErC,UAAA,OAAO,IAAI,OAAA,CACR,MAAA,CAAO,CAAC,IAAA,KAAc,KAAK,IAAA,KAAS,MAAM,CAAA,CAC1C,GAAA,CAAI,CAAC,IAAA,KAAc,IAAA,CAAK,QAAQ,EAAE,CAAA,CAClC,KAAK,GAAG,CAAA;AAAA,QACb;AACA,QAAA,OAAO,EAAA;AAAA,MACT,CAAC,CAAA,CACA,MAAA,CAAO,CAAA,OAAA,KAAW,OAAO;AAAA,KAC9B;AAAA,EACF;AAGA,EAAA,IAAI,OAAO,oBAAA,EAAsB;AAC/B,IAAA,MAAA,CAAO,MAAA,CAAO,KAAA,CAAM,oBAAoB,CAAA,CAAE,QAAQ,CAAA,QAAA,KAAY;AAC5D,MAAA,QAAA,CAAS,QAAQ,CAAA,GAAA,KAAO;AACtB,QAAA,IAAI,OAAO,GAAA,CAAI,OAAA,KAAY,QAAA,EAAU;AACnC,UAAA,cAAA,CAAe,IAAA,CAAK,IAAI,OAAO,CAAA;AAAA,QACjC;AAAA,MACF,CAAC,CAAA;AAAA,IACH,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,OAAO,cAAA;AACT;AAmBO,IAAM,uBAAA,GAA0B,CAAC,KAAA,KAA2C;AACjF,EAAA,MAAM,cAAA,GAAiB,8BAA8B,KAAK,CAAA;AAC1D,EAAA,OAAO,cAAA,CAAe,KAAK,MAAM,CAAA;AACnC;AAmBO,IAAM,gCAAA,GAAmC,CAAC,MAAA,KAAqC;AACpF,EAAA,MAAM,OAAA,GAAU,QAAQ,IAAA,CAAK,CAAC,EAAE,IAAA,EAAK,KAAM,SAAS,WAAW,CAAA;AAC/D,EAAA,OAAO,OAAA,GAAU,iCAAA,CAAkC,OAAO,CAAA,GAAI,MAAA;AAChE;AAiCO,IAAM,yBAAA,GAA4B,CAAC,MAAA,KAAyD;AACjG,EAAA,IAAI,CAAC,QAAQ,OAAO,MAAA;AAEpB,EAAA,MAAM,OAAA,GAAU,OAAO,IAAA,CAAK,CAAC,EAAE,IAAA,EAAK,KAAM,SAAS,WAAW,CAAA;AAC9D,EAAA,IAAI,CAAC,SAAS,OAAO,MAAA;AAGrB,EAAA,IAAI,OAAA,CAAQ,QAAQ,SAAA,EAAW;AAC7B,IAAA,OAAO,QAAQ,OAAA,CAAQ,SAAA;AAAA,EACzB;AAIA,EAAA,MAAM,cAAA,GAAiB,QAAQ,OAAA,CAAQ,KAAA,EAAO,OAAO,CAAC,CAAA,KAAW,CAAA,CAAE,IAAA,KAAS,WAAW,CAAA;AACvF,EAAA,IAAI,cAAA,IAAkB,cAAA,CAAe,MAAA,GAAS,CAAA,EAAG;AAC/C,IAAA,MAAM,cAAA,GAAiB,cAAA,CACpB,GAAA,CAAI,CAAC,CAAA,KAAW;AAEf,MAAA,IAAI,EAAE,OAAA,IAAW,KAAA,CAAM,OAAA,CAAQ,CAAA,CAAE,OAAO,CAAA,EAAG;AACzC,QAAA,OAAO,EAAE,OAAA,CACN,MAAA,CAAO,CAAC,CAAA,KAAW,EAAE,IAAA,KAAS,MAAM,CAAA,CACpC,GAAA,CAAI,CAAC,CAAA,KAAW,CAAA,CAAE,IAAI,CAAA,CACtB,KAAK,EAAE,CAAA;AAAA,MACZ;AACA,MAAA,OAAO,EAAE,SAAA,IAAa,EAAA;AAAA,IACxB,CAAC,CAAA,CACA,MAAA,CAAO,OAAO,CAAA;AAEjB,IAAA,OAAO,eAAe,MAAA,GAAS,CAAA,GAAI,cAAA,CAAe,IAAA,CAAK,IAAI,CAAA,GAAI,MAAA;AAAA,EACjE;AAEA,EAAA,OAAO,MAAA;AACT;AAuBO,IAAM,uBAAuB,CAAC;AAAA,EACnC,UAAA;AAAA,EACA,QAAA;AAAA,EACA,IAAA;AAAA,EACA,MAAA;AAAA,EACA,KAAA,GAAQ;AACV,CAAA,KAMuH;AACrH,EAAA,OAAO;AAAA,IACL,UAAA;AAAA,IACA,QAAA;AAAA,IACA,IAAA;AAAA,IACA,MAAA;AAAA,IACA;AAAA,GACF;AACF;AAmCO,SAAS,iBAAA,CAAkB;AAAA,EAChC,OAAA;AAAA,EACA,IAAA;AAAA,EACA,EAAA,GAAK,cAAA;AAAA,EACL,kBAAkB;AACpB,CAAA,EAWoB;AAClB,EAAA,OAAO;AAAA,IACL,EAAA;AAAA,IACA,IAAA;AAAA,IACA,OAAA,EAAS;AAAA,MACP,MAAA,EAAQ,CAAA;AAAA,MACR,OAAO,CAAC,EAAE,MAAM,MAAA,EAAQ,IAAA,EAAM,SAAS,CAAA;AAAA,MACvC,OAAA;AAAA,MACA,GAAI,eAAA,CAAgB,MAAA,GAAS,CAAA,IAAK;AAAA,QAChC,eAAA,EAAiB,eAAA,CAAgB,GAAA,CAAI,CAAA,EAAA,MAAO;AAAA,UAC1C,YAAY,EAAA,CAAG,UAAA;AAAA,UACf,UAAU,EAAA,CAAG,QAAA;AAAA,UACb,MAAM,EAAA,CAAG,IAAA;AAAA,UACT,QAAQ,EAAA,CAAG,MAAA;AAAA,UACX,OAAO,EAAA,CAAG;AAAA,SACZ,CAAE;AAAA;AACJ,KACF;AAAA,IACA,SAAA,sBAAe,IAAA;AAAK,GACtB;AACF;AA+BO,IAAM,qBAAqB,CAAC;AAAA,EACjC,gBAAgB,EAAC;AAAA,EACjB,MAAA;AAAA,EACA,qBAAqB,EAAC;AAAA,EACtB,iBAAiB,EAAC;AAAA,EAClB,uBAAuB,EAAC;AAAA,kBACxBA,gBAAA,GAAiB,IAAIC,6BAAA,EAAe;AAAA,EACpC,KAAA,GAAQ,OAAO,UAAA;AACjB,CAAA,KAaK;AACH,EAAA,OAAO;AAAA,IACL,KAAA,EAAO;AAAA,MACL,aAAA;AAAA,MACA,kBAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACF;AAAA,IACA,MAAA;AAAA,oBACAD,gBAAA;AAAA,IACA;AAAA,GACF;AACF;AAqCO,SAAS,iBAAiB,MAAA,EAAqF;AACpH,EAAA,MAAM,YAAsB,EAAC;AAC7B,EAAA,MAAM,gBAAgC,EAAC;AAEvC,EAAA,KAAA,IAAS,YAAA,GAAe,CAAA,EAAG,YAAA,GAAe,MAAA,CAAO,QAAQ,YAAA,EAAA,EAAgB;AACvE,IAAA,MAAM,OAAA,GAAU,OAAO,YAAY,CAAA;AAEnC,IAAA,IAAI,OAAA,EAAS,SAAS,eAAA,EAAiB;AACrC,MAAA,KAAA,IAAS,kBAAkB,CAAA,EAAG,eAAA,GAAkB,QAAQ,OAAA,CAAQ,eAAA,CAAgB,QAAQ,eAAA,EAAA,EAAmB;AACzG,QAAA,MAAM,UAAA,GAAa,OAAA,CAAQ,OAAA,CAAQ,eAAA,CAAgB,eAAe,CAAA;AAClE,QAAA,IAAI,UAAA,IAAc,WAAW,QAAA,KAAa,UAAA,CAAW,UAAU,QAAA,IAAY,UAAA,CAAW,UAAU,MAAA,CAAA,EAAS;AACvG,UAAA,SAAA,CAAU,IAAA,CAAK,WAAW,QAAQ,CAAA;AAClC,UAAA,aAAA,CAAc,IAAA,CAAK;AAAA,YACjB,UAAU,UAAA,CAAW,QAAA;AAAA,YACrB,YAAY,UAAA,CAAW,UAAA,IAAc,CAAA,EAAG,YAAY,IAAI,eAAe,CAAA,CAAA;AAAA,YACvE,YAAA;AAAA,YACA;AAAA,WACD,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,KAAA,EAAO,SAAA,EAAW,aAAA,EAAc;AAC3C;AAiBO,IAAM,oBAAA,GAAuB,CAAC,QAAA,KAA2D;AAC9F,EAAA,OAAO,QAAA,EAAU,eAAe,GAAA,CAAI,CAAA,GAAA,KAAO,kCAAkC,GAAG,CAAC,KAAK,EAAC;AACzF;AAmBO,IAAM,4BAAA,GAA+B,CAAC,SAAA,KAAiD;AAC5F,EAAA,OAAO,SAAA,CAAU,MAAA,CAAO,CAAA,GAAA,KAAO,GAAA,CAAI,IAAA,KAAS,WAAW,CAAA,CAAE,GAAA,CAAI,CAAA,GAAA,KAAO,iCAAA,CAAkC,GAAG,CAAC,CAAA;AAC5G;AAyCO,SAAS,mBAAmB,MAAA,EAAmD;AACpF,EAAA,MAAM,UAA4B,EAAC;AAEnC,EAAA,KAAA,MAAW,WAAW,MAAA,EAAQ;AAC5B,IAAA,MAAM,eAAA,GAAkB,SAAS,OAAA,EAAS,eAAA;AAC1C,IAAA,IAAI,CAAC,eAAA,EAAiB;AAEtB,IAAA,KAAA,MAAW,cAAc,eAAA,EAAiB;AACxC,MAAA,IAAI,UAAA,CAAW,KAAA,KAAU,QAAA,IAAY,UAAA,CAAW,WAAW,MAAA,EAAW;AACpE,QAAA,OAAA,CAAQ,IAAA,CAAK;AAAA,UACX,UAAU,UAAA,CAAW,QAAA;AAAA,UACrB,UAAA,EAAY,WAAW,UAAA,IAAc,EAAA;AAAA,UACrC,IAAA,EAAM,UAAA,CAAW,IAAA,IAAQ,EAAC;AAAA,UAC1B,QAAQ,UAAA,CAAW;AAAA,SACpB,CAAA;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,OAAA;AACT","file":"chunk-W3U7MMDX.cjs","sourcesContent":["import type { MastraDBMessage } from '@mastra/core/agent';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent, ScoringInput } from '@mastra/core/evals';\nimport { RequestContext } from '@mastra/core/request-context';\n\n/**\n * Extracts text content from a MastraDBMessage.\n *\n * This function matches the logic used in `MessageList.mastraDBMessageToAIV4UIMessage`.\n * It first checks for a string `content.content` field, then falls back to extracting\n * text from the `parts` array (returning only the last text part, like AI SDK does).\n *\n * @param message - The MastraDBMessage to extract text from\n * @returns The extracted text content, or an empty string if no text is found\n *\n * @example\n * ```ts\n * const message: MastraDBMessage = {\n *   id: 'msg-1',\n *   role: 'assistant',\n *   content: { format: 2, parts: [{ type: 'text', text: 'Hello!' }] },\n *   createdAt: new Date(),\n * };\n * const text = getTextContentFromMastraDBMessage(message); // 'Hello!'\n * ```\n */\nexport function getTextContentFromMastraDBMessage(message: MastraDBMessage): string {\n  if (typeof message.content.content === 'string' && message.content.content !== '') {\n    return message.content.content;\n  }\n  if (message.content.parts && Array.isArray(message.content.parts)) {\n    // Return only the last text part like AI SDK does\n    const textParts = message.content.parts.filter(p => p.type === 'text');\n    return textParts.length > 0 ? textParts[textParts.length - 1]?.text || '' : '';\n  }\n  return '';\n}\n\n/**\n * Rounds a number to two decimal places.\n *\n * Uses `Number.EPSILON` to handle floating-point precision issues.\n *\n * @param num - The number to round\n * @returns The number rounded to two decimal places\n *\n * @example\n * ```ts\n * roundToTwoDecimals(0.1 + 0.2); // 0.3\n * roundToTwoDecimals(1.005); // 1.01\n * ```\n */\nexport const roundToTwoDecimals = (num: number) => {\n  return Math.round((num + Number.EPSILON) * 100) / 100;\n};\n\n/**\n * Determines if a value is closer to the first target than the second.\n *\n * @param value - The value to compare\n * @param target1 - The first target value\n * @param target2 - The second target value\n * @returns `true` if `value` is closer to `target1` than `target2`\n *\n * @example\n * ```ts\n * isCloserTo(0.6, 1, 0); // true (0.6 is closer to 1)\n * isCloserTo(0.3, 1, 0); // false (0.3 is closer to 0)\n * ```\n */\nexport function isCloserTo(value: number, target1: number, target2: number): boolean {\n  return Math.abs(value - target1) < Math.abs(value - target2);\n}\n\n/**\n * Represents a test case for scorer evaluation.\n */\nexport type TestCase = {\n  /** The input text to evaluate */\n  input: string;\n  /** The output text to evaluate */\n  output: string;\n  /** The expected result of the evaluation */\n  expectedResult: {\n    /** The expected score */\n    score: number;\n    /** The optional expected reason */\n    reason?: string;\n  };\n};\n\n/**\n * Represents a test case with additional context for scorer evaluation.\n */\nexport type TestCaseWithContext = TestCase & {\n  /** Additional context strings for the evaluation */\n  context: string[];\n};\n\n/**\n * Creates a scoring input object for testing purposes.\n *\n * @param input - The user input text\n * @param output - The assistant output text\n * @param additionalContext - Optional additional context data\n * @param requestContext - Optional request context data\n * @returns A ScoringInput object ready for use in scorer tests\n *\n * @example\n * ```ts\n * const run = createTestRun(\n *   'What is 2+2?',\n *   'The answer is 4.',\n *   { topic: 'math' }\n * );\n * ```\n */\nexport const createTestRun = (\n  input: string,\n  output: string,\n  additionalContext?: Record<string, any>,\n  requestContext?: Record<string, any>,\n): ScoringInput => {\n  return {\n    input: [{ role: 'user', content: input }],\n    output: { role: 'assistant', text: output },\n    additionalContext: additionalContext ?? {},\n    requestContext: requestContext ?? {},\n  };\n};\n\n/**\n * Extracts the user message text from a scorer run input.\n *\n * Finds the first message with role 'user' and extracts its text content.\n *\n * @param input - The scorer run input containing input messages\n * @returns The user message text, or `undefined` if no user message is found\n *\n * @example\n * ```ts\n * const scorer = createScorer({ ... })\n *   .preprocess(({ run }) => {\n *     const userText = getUserMessageFromRunInput(run.input);\n *     return { userText };\n *   });\n * ```\n */\nexport const getUserMessageFromRunInput = (input?: ScorerRunInputForAgent): string | undefined => {\n  const message = input?.inputMessages.find(({ role }) => role === 'user');\n  return message ? getTextContentFromMastraDBMessage(message) : undefined;\n};\n\n/**\n * Extracts all system messages from a scorer run input.\n *\n * Collects text from both standard system messages and tagged system messages\n * (specialized system prompts like memory instructions).\n *\n * @param input - The scorer run input containing system messages\n * @returns An array of system message strings\n *\n * @example\n * ```ts\n * const scorer = createScorer({ ... })\n *   .preprocess(({ run }) => {\n *     const systemMessages = getSystemMessagesFromRunInput(run.input);\n *     return { systemPrompt: systemMessages.join('\\n') };\n *   });\n * ```\n */\nexport const getSystemMessagesFromRunInput = (input?: ScorerRunInputForAgent): string[] => {\n  const systemMessages: string[] = [];\n\n  // Add standard system messages\n  if (input?.systemMessages) {\n    systemMessages.push(\n      ...input.systemMessages\n        .map(msg => {\n          // Handle different content types - extract text if it's an array of parts\n          if (typeof msg.content === 'string') {\n            return msg.content;\n          } else if (Array.isArray(msg.content)) {\n            // Extract text from parts array\n            return msg.content\n              .filter((part: any) => part.type === 'text')\n              .map((part: any) => part.text || '')\n              .join(' ');\n          }\n          return '';\n        })\n        .filter(content => content),\n    );\n  }\n\n  // Add tagged system messages (these are specialized system prompts)\n  if (input?.taggedSystemMessages) {\n    Object.values(input.taggedSystemMessages).forEach(messages => {\n      messages.forEach(msg => {\n        if (typeof msg.content === 'string') {\n          systemMessages.push(msg.content);\n        }\n      });\n    });\n  }\n\n  return systemMessages;\n};\n\n/**\n * Combines all system messages into a single prompt string.\n *\n * Joins all system messages (standard and tagged) with double newlines.\n *\n * @param input - The scorer run input containing system messages\n * @returns A combined system prompt string\n *\n * @example\n * ```ts\n * const scorer = createScorer({ ... })\n *   .preprocess(({ run }) => {\n *     const systemPrompt = getCombinedSystemPrompt(run.input);\n *     return { systemPrompt };\n *   });\n * ```\n */\nexport const getCombinedSystemPrompt = (input?: ScorerRunInputForAgent): string => {\n  const systemMessages = getSystemMessagesFromRunInput(input);\n  return systemMessages.join('\\n\\n');\n};\n\n/**\n * Extracts the assistant message text from a scorer run output.\n *\n * Finds the first message with role 'assistant' and extracts its text content.\n *\n * @param output - The scorer run output (array of MastraDBMessage)\n * @returns The assistant message text, or `undefined` if no assistant message is found\n *\n * @example\n * ```ts\n * const scorer = createScorer({ ... })\n *   .preprocess(({ run }) => {\n *     const response = getAssistantMessageFromRunOutput(run.output);\n *     return { response };\n *   });\n * ```\n */\nexport const getAssistantMessageFromRunOutput = (output?: ScorerRunOutputForAgent) => {\n  const message = output?.find(({ role }) => role === 'assistant');\n  return message ? getTextContentFromMastraDBMessage(message) : undefined;\n};\n\n/**\n * Extracts reasoning text from a scorer run output.\n *\n * This function extracts reasoning content from assistant messages, which is\n * produced by reasoning models like `deepseek-reasoner`. The reasoning can be\n * stored in two places:\n * 1. `content.reasoning` - a string field on the message content\n * 2. `content.parts` - as parts with `type: 'reasoning'` containing `details`\n *\n * @param output - The scorer run output (array of MastraDBMessage)\n * @returns The reasoning text, or `undefined` if no reasoning is present\n *\n * @example\n * ```ts\n * const reasoningScorer = createScorer({\n *   id: 'reasoning-scorer',\n *   name: 'Reasoning Quality',\n *   description: 'Evaluates the quality of model reasoning',\n *   type: 'agent',\n * })\n *   .preprocess(({ run }) => {\n *     const reasoning = getReasoningFromRunOutput(run.output);\n *     const response = getAssistantMessageFromRunOutput(run.output);\n *     return { reasoning, response };\n *   })\n *   .generateScore(({ results }) => {\n *     // Score based on reasoning quality\n *     return results.preprocessStepResult?.reasoning ? 1 : 0;\n *   });\n * ```\n */\nexport const getReasoningFromRunOutput = (output?: ScorerRunOutputForAgent): string | undefined => {\n  if (!output) return undefined;\n\n  const message = output.find(({ role }) => role === 'assistant');\n  if (!message) return undefined;\n\n  // Check for reasoning in content.reasoning (string format)\n  if (message.content.reasoning) {\n    return message.content.reasoning;\n  }\n\n  // Check for reasoning in parts with type 'reasoning'\n  // Reasoning models store reasoning in parts as { type: 'reasoning', details: [{ type: 'text', text: '...' }] }\n  const reasoningParts = message.content.parts?.filter((p: any) => p.type === 'reasoning');\n  if (reasoningParts && reasoningParts.length > 0) {\n    const reasoningTexts = reasoningParts\n      .map((p: any) => {\n        // The reasoning text can be in p.reasoning or in p.details[].text\n        if (p.details && Array.isArray(p.details)) {\n          return p.details\n            .filter((d: any) => d.type === 'text')\n            .map((d: any) => d.text)\n            .join('');\n        }\n        return p.reasoning || '';\n      })\n      .filter(Boolean);\n\n    return reasoningTexts.length > 0 ? reasoningTexts.join('\\n') : undefined;\n  }\n\n  return undefined;\n};\n\n/**\n * Creates a tool invocation object for testing purposes.\n *\n * @param options - The tool invocation configuration\n * @param options.toolCallId - Unique identifier for the tool call\n * @param options.toolName - Name of the tool being called\n * @param options.args - Arguments passed to the tool\n * @param options.result - Result returned by the tool\n * @param options.state - State of the invocation (default: 'result')\n * @returns A tool invocation object\n *\n * @example\n * ```ts\n * const invocation = createToolInvocation({\n *   toolCallId: 'call-123',\n *   toolName: 'weatherTool',\n *   args: { location: 'London' },\n *   result: { temperature: 20, condition: 'sunny' },\n * });\n * ```\n */\nexport const createToolInvocation = ({\n  toolCallId,\n  toolName,\n  args,\n  result,\n  state = 'result',\n}: {\n  toolCallId: string;\n  toolName: string;\n  args: Record<string, any>;\n  result: Record<string, any>;\n  state?: 'call' | 'partial-call' | 'result';\n}): { toolCallId: string; toolName: string; args: Record<string, any>; result: Record<string, any>; state: string } => {\n  return {\n    toolCallId,\n    toolName,\n    args,\n    result,\n    state,\n  };\n};\n\n/**\n * Creates a MastraDBMessage object for testing purposes.\n *\n * Supports optional tool invocations for testing tool call scenarios.\n *\n * @param options - The message configuration\n * @param options.content - The text content of the message\n * @param options.role - The role of the message sender ('user', 'assistant', or 'system')\n * @param options.id - Optional message ID (default: 'test-message')\n * @param options.toolInvocations - Optional array of tool invocations\n * @returns A MastraDBMessage object\n *\n * @example\n * ```ts\n * const message = createTestMessage({\n *   content: 'Hello, how can I help?',\n *   role: 'assistant',\n * });\n *\n * // With tool invocations\n * const messageWithTools = createTestMessage({\n *   content: 'Let me check the weather.',\n *   role: 'assistant',\n *   toolInvocations: [{\n *     toolCallId: 'call-1',\n *     toolName: 'weatherTool',\n *     args: { location: 'Paris' },\n *     result: { temp: 22 },\n *     state: 'result',\n *   }],\n * });\n * ```\n */\nexport function createTestMessage({\n  content,\n  role,\n  id = 'test-message',\n  toolInvocations = [],\n}: {\n  content: string;\n  role: 'user' | 'assistant' | 'system';\n  id?: string;\n  toolInvocations?: Array<{\n    toolCallId: string;\n    toolName: string;\n    args: Record<string, any>;\n    result: Record<string, any>;\n    state: any;\n  }>;\n}): MastraDBMessage {\n  return {\n    id,\n    role,\n    content: {\n      format: 2,\n      parts: [{ type: 'text', text: content }],\n      content,\n      ...(toolInvocations.length > 0 && {\n        toolInvocations: toolInvocations.map(ti => ({\n          toolCallId: ti.toolCallId,\n          toolName: ti.toolName,\n          args: ti.args,\n          result: ti.result,\n          state: ti.state,\n        })),\n      }),\n    },\n    createdAt: new Date(),\n  };\n}\n\n/**\n * Creates a complete agent test run object for testing scorers.\n *\n * Provides a convenient way to construct the full run object that scorers receive,\n * including input messages, output, system messages, and request context.\n *\n * @param options - The test run configuration\n * @param options.inputMessages - Array of input messages (default: [])\n * @param options.output - The output messages (required)\n * @param options.rememberedMessages - Array of remembered messages from memory (default: [])\n * @param options.systemMessages - Array of system messages (default: [])\n * @param options.taggedSystemMessages - Tagged system messages map (default: {})\n * @param options.requestContext - Request context (default: new RequestContext())\n * @param options.runId - Unique run ID (default: random UUID)\n * @returns A complete test run object\n *\n * @example\n * ```ts\n * const testRun = createAgentTestRun({\n *   inputMessages: [createTestMessage({ content: 'Hello', role: 'user' })],\n *   output: [createTestMessage({ content: 'Hi there!', role: 'assistant' })],\n * });\n *\n * const result = await scorer.run({\n *   input: testRun.input,\n *   output: testRun.output,\n * });\n * ```\n */\nexport const createAgentTestRun = ({\n  inputMessages = [],\n  output,\n  rememberedMessages = [],\n  systemMessages = [],\n  taggedSystemMessages = {},\n  requestContext = new RequestContext(),\n  runId = crypto.randomUUID(),\n}: {\n  inputMessages?: ScorerRunInputForAgent['inputMessages'];\n  output: ScorerRunOutputForAgent;\n  rememberedMessages?: ScorerRunInputForAgent['rememberedMessages'];\n  systemMessages?: ScorerRunInputForAgent['systemMessages'];\n  taggedSystemMessages?: ScorerRunInputForAgent['taggedSystemMessages'];\n  requestContext?: RequestContext;\n  runId?: string;\n}): {\n  input: ScorerRunInputForAgent;\n  output: ScorerRunOutputForAgent;\n  requestContext: RequestContext;\n  runId: string;\n} => {\n  return {\n    input: {\n      inputMessages,\n      rememberedMessages,\n      systemMessages,\n      taggedSystemMessages,\n    },\n    output,\n    requestContext,\n    runId,\n  };\n};\n\n/**\n * Information about a tool call extracted from scorer output.\n */\nexport type ToolCallInfo = {\n  /** Name of the tool that was called */\n  toolName: string;\n  /** Unique identifier for the tool call */\n  toolCallId: string;\n  /** Index of the message containing this tool call */\n  messageIndex: number;\n  /** Index of the invocation within the message's tool invocations */\n  invocationIndex: number;\n};\n\n/**\n * Extracts all tool calls from a scorer run output.\n *\n * Iterates through all messages and their tool invocations to collect\n * information about tools that were called (with state 'result' or 'call').\n *\n * @param output - The scorer run output (array of MastraDBMessage)\n * @returns An object containing tool names and detailed tool call info\n *\n * @example\n * ```ts\n * const scorer = createScorer({ ... })\n *   .preprocess(({ run }) => {\n *     const { tools, toolCallInfos } = extractToolCalls(run.output);\n *     return {\n *       toolsUsed: tools,\n *       toolCount: tools.length,\n *     };\n *   });\n * ```\n */\nexport function extractToolCalls(output: ScorerRunOutputForAgent): { tools: string[]; toolCallInfos: ToolCallInfo[] } {\n  const toolCalls: string[] = [];\n  const toolCallInfos: ToolCallInfo[] = [];\n\n  for (let messageIndex = 0; messageIndex < output.length; messageIndex++) {\n    const message = output[messageIndex];\n    // Tool invocations are now nested under content\n    if (message?.content?.toolInvocations) {\n      for (let invocationIndex = 0; invocationIndex < message.content.toolInvocations.length; invocationIndex++) {\n        const invocation = message.content.toolInvocations[invocationIndex];\n        if (invocation && invocation.toolName && (invocation.state === 'result' || invocation.state === 'call')) {\n          toolCalls.push(invocation.toolName);\n          toolCallInfos.push({\n            toolName: invocation.toolName,\n            toolCallId: invocation.toolCallId || `${messageIndex}-${invocationIndex}`,\n            messageIndex,\n            invocationIndex,\n          });\n        }\n      }\n    }\n  }\n\n  return { tools: toolCalls, toolCallInfos };\n}\n\n/**\n * Extracts text content from all input messages.\n *\n * @param runInput - The scorer run input\n * @returns An array of text strings from each input message\n *\n * @example\n * ```ts\n * const scorer = createScorer({ ... })\n *   .preprocess(({ run }) => {\n *     const messages = extractInputMessages(run.input);\n *     return { allUserMessages: messages.join('\\n') };\n *   });\n * ```\n */\nexport const extractInputMessages = (runInput: ScorerRunInputForAgent | undefined): string[] => {\n  return runInput?.inputMessages?.map(msg => getTextContentFromMastraDBMessage(msg)) || [];\n};\n\n/**\n * Extracts text content from all assistant response messages.\n *\n * Filters for messages with role 'assistant' and extracts their text content.\n *\n * @param runOutput - The scorer run output (array of MastraDBMessage)\n * @returns An array of text strings from each assistant message\n *\n * @example\n * ```ts\n * const scorer = createScorer({ ... })\n *   .preprocess(({ run }) => {\n *     const responses = extractAgentResponseMessages(run.output);\n *     return { allResponses: responses.join('\\n') };\n *   });\n * ```\n */\nexport const extractAgentResponseMessages = (runOutput: ScorerRunOutputForAgent): string[] => {\n  return runOutput.filter(msg => msg.role === 'assistant').map(msg => getTextContentFromMastraDBMessage(msg));\n};\n\n/**\n * Information about a tool result extracted from scorer output.\n */\nexport type ToolResultInfo = {\n  /** Name of the tool that was called */\n  toolName: string;\n  /** Unique identifier for the tool call */\n  toolCallId: string;\n  /** Arguments passed to the tool */\n  args: Record<string, any>;\n  /** Result returned by the tool */\n  result: any;\n};\n\n/**\n * Extracts tool results from a scorer run output.\n *\n * Returns structured objects that can be used with the hallucination scorer's\n * `getContext` hook or for other scorer logic.\n *\n * @param output - The scorer run output (array of MastraDBMessage)\n * @returns An array of ToolResultInfo objects\n *\n * @example\n * ```ts\n * import { extractToolResults } from '@mastra/evals/scorers';\n * import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt';\n *\n * const scorer = createHallucinationScorer({\n *   model: openai('gpt-4o'),\n *   options: {\n *     getContext: (run) => {\n *       const toolResults = extractToolResults(run.output);\n *       return toolResults.map(t => JSON.stringify({ tool: t.toolName, result: t.result }));\n *     },\n *   },\n * });\n * ```\n */\nexport function extractToolResults(output: ScorerRunOutputForAgent): ToolResultInfo[] {\n  const results: ToolResultInfo[] = [];\n\n  for (const message of output) {\n    const toolInvocations = message?.content?.toolInvocations;\n    if (!toolInvocations) continue;\n\n    for (const invocation of toolInvocations) {\n      if (invocation.state === 'result' && invocation.result !== undefined) {\n        results.push({\n          toolName: invocation.toolName,\n          toolCallId: invocation.toolCallId || '',\n          args: invocation.args || {},\n          result: invocation.result,\n        });\n      }\n    }\n  }\n\n  return results;\n}\n"]}

package/dist/docs/SKILL.md

@@ -0,0 +1,43 @@
+---
+name: mastra-evals
+description: Documentation for @mastra/evals. Use when working with @mastra/evals APIs, configuration, or implementation.
+metadata:
+  package: "@mastra/evals"
+  version: "0.0.0-execa-dynamic-import-20260304221256"
+---
+
+## When to use
+
+Use this skill whenever you are working with @mastra/evals to obtain the domain-specific knowledge.
+
+## How to use
+
+Read the individual reference documents for detailed explanations and code examples.
+
+### Docs
+
+- [Built-in Scorers](references/docs-evals-built-in-scorers.md) - Overview of Mastra's ready-to-use scorers for evaluating AI outputs across quality, safety, and performance dimensions.
+- [Scorers overview](references/docs-evals-overview.md) - Overview of scorers in Mastra, detailing their capabilities for evaluating AI outputs and measuring performance.
+
+### Reference
+
+- [Reference: Answer Relevancy Scorer](references/reference-evals-answer-relevancy.md) - Documentation for the Answer Relevancy Scorer in Mastra, which evaluates how well LLM outputs address the input query.
+- [Reference: Answer Similarity Scorer](references/reference-evals-answer-similarity.md) - Documentation for the Answer Similarity Scorer in Mastra, which compares agent outputs against ground truth answers for CI/CD testing.
+- [Reference: Bias Scorer](references/reference-evals-bias.md) - Documentation for the Bias Scorer in Mastra, which evaluates LLM outputs for various forms of bias, including gender, political, racial/ethnic, or geographical bias.
+- [Reference: Completeness Scorer](references/reference-evals-completeness.md) - Documentation for the Completeness Scorer in Mastra, which evaluates how thoroughly LLM outputs cover key elements present in the input.
+- [Reference: Content Similarity Scorer](references/reference-evals-content-similarity.md) - Documentation for the Content Similarity Scorer in Mastra, which measures textual similarity between strings and provides a matching score.
+- [Reference: Context Precision Scorer](references/reference-evals-context-precision.md) - Documentation for the Context Precision Scorer in Mastra. Evaluates the relevance and precision of retrieved context for generating expected outputs using Mean Average Precision.
+- [Reference: Context Relevance Scorer](references/reference-evals-context-relevance.md) - Documentation for the Context Relevance Scorer in Mastra. Evaluates the relevance and utility of provided context for generating agent responses using weighted relevance scoring.
+- [Reference: Faithfulness Scorer](references/reference-evals-faithfulness.md) - Documentation for the Faithfulness Scorer in Mastra, which evaluates the factual accuracy of LLM outputs compared to the provided context.
+- [Reference: Hallucination Scorer](references/reference-evals-hallucination.md) - Documentation for the Hallucination Scorer in Mastra, which evaluates the factual correctness of LLM outputs by identifying contradictions with provided context.
+- [Reference: Keyword Coverage Scorer](references/reference-evals-keyword-coverage.md) - Documentation for the Keyword Coverage Scorer in Mastra, which evaluates how well LLM outputs cover important keywords from the input.
+- [Reference: Noise Sensitivity Scorer](references/reference-evals-noise-sensitivity.md) - Documentation for the Noise Sensitivity Scorer in Mastra. A CI/testing scorer that evaluates agent robustness by comparing responses between clean and noisy inputs in controlled test environments.
+- [Reference: Prompt Alignment Scorer](references/reference-evals-prompt-alignment.md) - Documentation for the Prompt Alignment Scorer in Mastra. Evaluates how well agent responses align with user prompt intent, requirements, completeness, and appropriateness using multi-dimensional analysis.
+- [Reference: Scorer Utils](references/reference-evals-scorer-utils.md) - Utility functions for extracting data from scorer run inputs and outputs, including text content, reasoning, system messages, and tool calls.
+- [Reference: Textual Difference Scorer](references/reference-evals-textual-difference.md) - Documentation for the Textual Difference Scorer in Mastra, which measures textual differences between strings using sequence matching.
+- [Reference: Tone Consistency Scorer](references/reference-evals-tone-consistency.md) - Documentation for the Tone Consistency Scorer in Mastra, which evaluates emotional tone and sentiment consistency in text.
+- [Reference: Tool Call Accuracy Scorers](references/reference-evals-tool-call-accuracy.md) - Documentation for the Tool Call Accuracy Scorers in Mastra, which evaluate whether LLM outputs call the correct tools from available options.
+- [Reference: Toxicity Scorer](references/reference-evals-toxicity.md) - Documentation for the Toxicity Scorer in Mastra, which evaluates LLM outputs for racist, biased, or toxic elements.
+
+
+Read [assets/SOURCE_MAP.json](assets/SOURCE_MAP.json) for source code references.

package/dist/docs/assets/SOURCE_MAP.json

@@ -0,0 +1,6 @@
+{
+  "version": "0.0.0-execa-dynamic-import-20260304221256",
+  "package": "@mastra/evals",
+  "exports": {},
+  "modules": {}
+}

package/dist/docs/references/docs-evals-built-in-scorers.md

@@ -0,0 +1,47 @@
+# Built-in Scorers
+
+Mastra provides a comprehensive set of built-in scorers for evaluating AI outputs. These scorers are optimized for common evaluation scenarios and are ready to use in your agents and workflows.
+
+To create your own scorers, see the [Custom Scorers](https://mastra.ai/docs/evals/custom-scorers) guide.
+
+## Available scorers
+
+### Accuracy and reliability
+
+These scorers evaluate how correct, truthful, and complete your agent's answers are:
+
+- [`answer-relevancy`](https://mastra.ai/reference/evals/answer-relevancy): Evaluates how well responses address the input query (`0-1`, higher is better)
+- [`answer-similarity`](https://mastra.ai/reference/evals/answer-similarity): Compares agent outputs against ground-truth answers for CI/CD testing using semantic analysis (`0-1`, higher is better)
+- [`faithfulness`](https://mastra.ai/reference/evals/faithfulness): Measures how accurately responses represent provided context (`0-1`, higher is better)
+- [`hallucination`](https://mastra.ai/reference/evals/hallucination): Detects factual contradictions and unsupported claims (`0-1`, lower is better)
+- [`completeness`](https://mastra.ai/reference/evals/completeness): Checks if responses include all necessary information (`0-1`, higher is better)
+- [`content-similarity`](https://mastra.ai/reference/evals/content-similarity): Measures textual similarity using character-level matching (`0-1`, higher is better)
+- [`textual-difference`](https://mastra.ai/reference/evals/textual-difference): Measures textual differences between strings (`0-1`, higher means more similar)
+- [`tool-call-accuracy`](https://mastra.ai/reference/evals/tool-call-accuracy): Evaluates whether the LLM selects the correct tool from available options (`0-1`, higher is better)
+- [`prompt-alignment`](https://mastra.ai/reference/evals/prompt-alignment): Measures how well agent responses align with user prompt intent, requirements, completeness, and format (`0-1`, higher is better)
+
+### Context quality
+
+These scorers evaluate the quality and relevance of context used in generating responses:
+
+- [`context-precision`](https://mastra.ai/reference/evals/context-precision): Evaluates context relevance and ranking using Mean Average Precision, rewarding early placement of relevant context (`0-1`, higher is better)
+- [`context-relevance`](https://mastra.ai/reference/evals/context-relevance): Measures context utility with nuanced relevance levels, usage tracking, and missing context detection (`0-1`, higher is better)
+
+> tip Context Scorer Selection
+>
+> - Use **Context Precision** when context ordering matters and you need standard IR metrics (ideal for RAG ranking evaluation)
+> - Use **Context Relevance** when you need detailed relevance assessment and want to track context usage and identify gaps
+>
+> Both context scorers support:
+>
+> - **Static context**: Pre-defined context arrays
+> - **Dynamic context extraction**: Extract context from runs using custom functions (ideal for RAG systems, vector databases, etc.)
+
+### Output quality
+
+These scorers evaluate adherence to format, style, and safety requirements:
+
+- [`tone-consistency`](https://mastra.ai/reference/evals/tone-consistency): Measures consistency in formality, complexity, and style (`0-1`, higher is better)
+- [`toxicity`](https://mastra.ai/reference/evals/toxicity): Detects harmful or inappropriate content (`0-1`, lower is better)
+- [`bias`](https://mastra.ai/reference/evals/bias): Detects potential biases in the output (`0-1`, lower is better)
+- [`keyword-coverage`](https://mastra.ai/reference/evals/keyword-coverage): Assesses technical terminology usage (`0-1`, higher is better)

package/dist/docs/references/docs-evals-overview.md

@@ -0,0 +1,141 @@
+# Scorers overview
+
+While traditional software tests have clear pass/fail conditions, AI outputs are non-deterministic — they can vary with the same input. **Scorers** help bridge this gap by providing quantifiable metrics for measuring agent quality.
+
+Scorers are automated tests that evaluate Agents outputs using model-graded, rule-based, and statistical methods. Scorers return **scores**: numerical values (typically between 0 and 1) that quantify how well an output meets your evaluation criteria. These scores enable you to objectively track performance, compare different approaches, and identify areas for improvement in your AI systems. Scorers can be customized with your own prompts and scoring functions.
+
+Scorers can be run in the cloud, capturing real-time results. But scorers can also be part of your CI/CD pipeline, allowing you to test and monitor your agents over time.
+
+## Types of Scorers
+
+There are different kinds of scorers, each serving a specific purpose. Here are some common types:
+
+1. **Textual Scorers**: Evaluate accuracy, reliability, and context understanding of agent responses
+2. **Classification Scorers**: Measure accuracy in categorizing data based on predefined categories
+3. **Prompt Engineering Scorers**: Explore impact of different instructions and input formats
+
+## Installation
+
+To access Mastra's scorers feature install the `@mastra/evals` package.
+
+**npm**:
+
+```bash
+npm install @mastra/evals@latest
+```
+
+**pnpm**:
+
+```bash
+pnpm add @mastra/evals@latest
+```
+
+**Yarn**:
+
+```bash
+yarn add @mastra/evals@latest
+```
+
+**Bun**:
+
+```bash
+bun add @mastra/evals@latest
+```
+
+## Live evaluations
+
+**Live evaluations** allow you to automatically score AI outputs in real-time as your agents and workflows operate. Instead of running evaluations manually or in batches, scorers run asynchronously alongside your AI systems, providing continuous quality monitoring.
+
+### Adding scorers to agents
+
+You can add built-in scorers to your agents to automatically evaluate their outputs. See the [full list of built-in scorers](https://mastra.ai/docs/evals/built-in-scorers) for all available options.
+
+```typescript
+import { Agent } from '@mastra/core/agent'
+import { createAnswerRelevancyScorer, createToxicityScorer } from '@mastra/evals/scorers/prebuilt'
+
+export const evaluatedAgent = new Agent({
+  scorers: {
+    relevancy: {
+      scorer: createAnswerRelevancyScorer({ model: 'openai/gpt-4.1-nano' }),
+      sampling: { type: 'ratio', rate: 0.5 },
+    },
+    safety: {
+      scorer: createToxicityScorer({ model: 'openai/gpt-4.1-nano' }),
+      sampling: { type: 'ratio', rate: 1 },
+    },
+  },
+})
+```
+
+### Adding scorers to workflow steps
+
+You can also add scorers to individual workflow steps to evaluate outputs at specific points in your process:
+
+```typescript
+import { createWorkflow, createStep } from "@mastra/core/workflows";
+import { z } from "zod";
+import { customStepScorer } from "../scorers/custom-step-scorer";
+
+const contentStep = createStep({
+  scorers: {
+    customStepScorer: {
+      scorer: customStepScorer(),
+      sampling: {
+        type: "ratio",
+        rate: 1, // Score every step execution
+      }
+    }
+  },
+});
+
+export const contentWorkflow = createWorkflow({ ... })
+  .then(contentStep)
+  .commit();
+```
+
+### How live evaluations work
+
+**Asynchronous execution**: Live evaluations run in the background without blocking your agent responses or workflow execution. This ensures your AI systems maintain their performance while still being monitored.
+
+**Sampling control**: The `sampling.rate` parameter (0-1) controls what percentage of outputs get scored:
+
+- `1.0`: Score every single response (100%)
+- `0.5`: Score half of all responses (50%)
+- `0.1`: Score 10% of responses
+- `0.0`: Disable scoring
+
+**Automatic storage**: All scoring results are automatically stored in the `mastra_scorers` table in your configured database, allowing you to analyze performance trends over time.
+
+## Trace evaluations
+
+In addition to live evaluations, you can use scorers to evaluate historical traces from your agent interactions and workflows. This is particularly useful for analyzing past performance, debugging issues, or running batch evaluations.
+
+> **Observability Required:** To score traces, you must first configure observability in your Mastra instance to collect trace data. See [Tracing documentation](https://mastra.ai/docs/observability/tracing/overview) for setup instructions.
+
+### Scoring traces with Studio
+
+To score traces, you first need to register your scorers with your Mastra instance:
+
+```typescript
+const mastra = new Mastra({
+  scorers: {
+    answerRelevancy: myAnswerRelevancyScorer,
+    responseQuality: myResponseQualityScorer,
+  },
+})
+```
+
+Once registered, you can score traces interactively within Studio under the Observability section. This provides a user-friendly interface for running scorers against historical traces.
+
+## Testing scorers locally
+
+Mastra provides a CLI command `mastra dev` to test your scorers. Studio includes a scorers section where you can run individual scorers against test inputs and view detailed results.
+
+For more details, see [Studio](https://mastra.ai/docs/getting-started/studio) docs.
+
+## Next steps
+
+- Learn how to create your own scorers in the [Creating Custom Scorers](https://mastra.ai/docs/evals/custom-scorers) guide
+- Explore built-in scorers in the [Built-in Scorers](https://mastra.ai/docs/evals/built-in-scorers) section
+- Test scorers with [Studio](https://mastra.ai/docs/getting-started/studio)

package/dist/docs/references/reference-evals-answer-relevancy.md

@@ -0,0 +1,105 @@
+# Answer Relevancy Scorer
+
+The `createAnswerRelevancyScorer()` function accepts a single options object with the following properties:
+
+## Parameters
+
+**model:** (`LanguageModel`): Configuration for the model used to evaluate relevancy.
+
+**uncertaintyWeight:** (`number`): Weight given to 'unsure' verdicts in scoring (0-1). (Default: `0.3`)
+
+**scale:** (`number`): Maximum score value. (Default: `1`)
+
+This function returns an instance of the MastraScorer class. The `.run()` method accepts the same input as other scorers (see the [MastraScorer reference](https://mastra.ai/reference/evals/mastra-scorer)), but the return value includes LLM-specific fields as documented below.
+
+## .run() Returns
+
+**runId:** (`string`): The id of the run (optional).
+
+**score:** (`number`): Relevancy score (0 to scale, default 0-1)
+
+**preprocessPrompt:** (`string`): The prompt sent to the LLM for the preprocess step (optional).
+
+**preprocessStepResult:** (`object`): Object with extracted statements: { statements: string\[] }
+
+**analyzePrompt:** (`string`): The prompt sent to the LLM for the analyze step (optional).
+
+**analyzeStepResult:** (`object`): Object with results: { results: Array<{ result: 'yes' | 'unsure' | 'no', reason: string }> }
+
+**generateReasonPrompt:** (`string`): The prompt sent to the LLM for the reason step (optional).
+
+**reason:** (`string`): Explanation of the score.
+
+## Scoring Details
+
+The scorer evaluates relevancy through query-answer alignment, considering completeness and detail level, but not factual correctness.
+
+### Scoring Process
+
+1. **Statement Preprocess:**
+   - Breaks output into meaningful statements while preserving context.
+
+2. **Relevance Analysis:**
+
+   - Each statement is evaluated as:
+
+     - "yes": Full weight for direct matches
+     - "unsure": Partial weight (default: 0.3) for approximate matches
+     - "no": Zero weight for irrelevant content
+
+3. **Score Calculation:**
+   - `((direct + uncertainty * partial) / total_statements) * scale`
+
+### Score Interpretation
+
+A relevancy score between 0 and 1:
+
+- **1.0**: The response fully answers the query with relevant and focused information.
+- **0.7–0.9**: The response mostly answers the query but may include minor unrelated content.
+- **0.4–0.6**: The response partially answers the query, mixing relevant and unrelated information.
+- **0.1–0.3**: The response includes minimal relevant content and largely misses the intent of the query.
+- **0.0**: The response is entirely unrelated and does not answer the query.
+
+## Example
+
+Evaluate agent responses for relevancy across different scenarios:
+
+```typescript
+import { runEvals } from '@mastra/core/evals'
+import { createAnswerRelevancyScorer } from '@mastra/evals/scorers/prebuilt'
+import { myAgent } from './agent'
+
+const scorer = createAnswerRelevancyScorer({ model: 'openai/gpt-4o' })
+
+const result = await runEvals({
+  data: [
+    {
+      input: 'What are the health benefits of regular exercise?',
+    },
+    {
+      input: 'What should a healthy breakfast include?',
+    },
+    {
+      input: 'What are the benefits of meditation?',
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
+    })
+  },
+})
+
+console.log(result.scores)
+```
+
+For more details on `runEvals`, see the [runEvals reference](https://mastra.ai/reference/evals/run-evals).
+
+To add this scorer to an agent, see the [Scorers overview](https://mastra.ai/docs/evals/overview) guide.
+
+## Related
+
+- [Faithfulness Scorer](https://mastra.ai/reference/evals/faithfulness)

package/dist/docs/references/reference-evals-answer-similarity.md

@@ -0,0 +1,99 @@
+# Answer Similarity Scorer
+
+The `createAnswerSimilarityScorer()` function creates a scorer that evaluates how similar an agent's output is to a ground truth answer. This scorer is specifically designed for CI/CD testing scenarios where you have expected answers and want to ensure consistency over time.
+
+## Parameters
+
+**model:** (`LanguageModel`): The language model used to evaluate semantic similarity between outputs and ground truth.
+
+**options:** (`AnswerSimilarityOptions`): Configuration options for the scorer.
+
+### AnswerSimilarityOptions
+
+**requireGroundTruth:** (`boolean`): Whether to require ground truth for evaluation. If false, missing ground truth returns score 0. (Default: `true`)
+
+**semanticThreshold:** (`number`): Weight for semantic matches vs exact matches (0-1). (Default: `0.8`)
+
+**exactMatchBonus:** (`number`): Additional score bonus for exact matches (0-1). (Default: `0.2`)
+
+**missingPenalty:** (`number`): Penalty per missing key concept from ground truth. (Default: `0.15`)
+
+**contradictionPenalty:** (`number`): Penalty for contradictory information. High value ensures wrong answers score near 0. (Default: `1.0`)
+
+**extraInfoPenalty:** (`number`): Mild penalty for extra information not present in ground truth (capped at 0.2). (Default: `0.05`)
+
+**scale:** (`number`): Score scaling factor. (Default: `1`)
+
+This function returns an instance of the MastraScorer class. The `.run()` method accepts the same input as other scorers (see the [MastraScorer reference](https://mastra.ai/reference/evals/mastra-scorer)), but **requires ground truth** to be provided in the run object.
+
+## .run() Returns
+
+**runId:** (`string`): The id of the run (optional).
+
+**score:** (`number`): Similarity score between 0-1 (or 0-scale if custom scale used). Higher scores indicate better similarity to ground truth.
+
+**reason:** (`string`): Human-readable explanation of the score with actionable feedback.
+
+**preprocessStepResult:** (`object`): Extracted semantic units from output and ground truth.
+
+**analyzeStepResult:** (`object`): Detailed analysis of matches, contradictions, and extra information.
+
+**preprocessPrompt:** (`string`): The prompt used for semantic unit extraction.
+
+**analyzePrompt:** (`string`): The prompt used for similarity analysis.
+
+**generateReasonPrompt:** (`string`): The prompt used for generating the explanation.
+
+## Scoring Details
+
+The scorer uses a multi-step process:
+
+1. **Extract**: Breaks down output and ground truth into semantic units
+2. **Analyze**: Compares units and identifies matches, contradictions, and gaps
+3. **Score**: Calculates weighted similarity with penalties for contradictions
+4. **Reason**: Generates human-readable explanation
+
+Score calculation: `max(0, base_score - contradiction_penalty - missing_penalty - extra_info_penalty) × scale`
+
+## Example
+
+Evaluate agent responses for similarity to ground truth across different scenarios:
+
+```typescript
+import { runEvals } from '@mastra/core/evals'
+import { createAnswerSimilarityScorer } from '@mastra/evals/scorers/prebuilt'
+import { myAgent } from './agent'
+
+const scorer = createAnswerSimilarityScorer({ model: 'openai/gpt-4o' })
+
+const result = await runEvals({
+  data: [
+    {
+      input: 'What is 2+2?',
+      groundTruth: '4',
+    },
+    {
+      input: 'What is the capital of France?',
+      groundTruth: 'The capital of France is Paris',
+    },
+    {
+      input: 'What are the primary colors?',
+      groundTruth: 'The primary colors are red, blue, and yellow',
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
+    })
+  },
+})
+
+console.log(result.scores)
+```
+
+For more details on `runEvals`, see the [runEvals reference](https://mastra.ai/reference/evals/run-evals).
+
+To add this scorer to an agent, see the [Scorers overview](https://mastra.ai/docs/evals/overview) guide.

package/dist/docs/references/reference-evals-bias.md

@@ -0,0 +1,120 @@
+# Bias Scorer
+
+The `createBiasScorer()` function accepts a single options object with the following properties:
+
+## Parameters
+
+**model:** (`LanguageModel`): Configuration for the model used to evaluate bias.
+
+**scale:** (`number`): Maximum score value. (Default: `1`)
+
+This function returns an instance of the MastraScorer class. The `.run()` method accepts the same input as other scorers (see the [MastraScorer reference](https://mastra.ai/reference/evals/mastra-scorer)), but the return value includes LLM-specific fields as documented below.
+
+## .run() Returns
+
+**runId:** (`string`): The id of the run (optional).
+
+**preprocessStepResult:** (`object`): Object with extracted opinions: { opinions: string\[] }
+
+**preprocessPrompt:** (`string`): The prompt sent to the LLM for the preprocess step (optional).
+
+**analyzeStepResult:** (`object`): Object with results: { results: Array<{ result: 'yes' | 'no', reason: string }> }
+
+**analyzePrompt:** (`string`): The prompt sent to the LLM for the analyze step (optional).
+
+**score:** (`number`): Bias score (0 to scale, default 0-1). Higher scores indicate more bias.
+
+**reason:** (`string`): Explanation of the score.
+
+**generateReasonPrompt:** (`string`): The prompt sent to the LLM for the generateReason step (optional).
+
+## Bias Categories
+
+The scorer evaluates several types of bias:
+
+1. **Gender Bias**: Discrimination or stereotypes based on gender
+2. **Political Bias**: Prejudice against political ideologies or beliefs
+3. **Racial/Ethnic Bias**: Discrimination based on race, ethnicity, or national origin
+4. **Geographical Bias**: Prejudice based on location or regional stereotypes
+
+## Scoring Details
+
+The scorer evaluates bias through opinion analysis based on:
+
+- Opinion identification and extraction
+- Presence of discriminatory language
+- Use of stereotypes or generalizations
+- Balance in perspective presentation
+- Loaded or prejudicial terminology
+
+### Scoring Process
+
+1. Extracts opinions from text:
+
+   - Identifies subjective statements
+   - Excludes factual claims
+   - Includes cited opinions
+
+2. Evaluates each opinion:
+
+   - Checks for discriminatory language
+   - Assesses stereotypes and generalizations
+   - Analyzes perspective balance
+
+Final score: `(biased_opinions / total_opinions) * scale`
+
+### Score interpretation
+
+A bias score between 0 and 1:
+
+- **1.0**: Contains explicit discriminatory or stereotypical statements.
+- **0.7–0.9**: Includes strong prejudiced assumptions or generalizations.
+- **0.4–0.6**: Mixes reasonable points with subtle bias or stereotypes.
+- **0.1–0.3**: Mostly neutral with minor biased language or assumptions.
+- **0.0**: Completely objective and free from bias.
+
+## Example
+
+Evaluate agent responses for bias across different types of questions:
+
+```typescript
+import { runEvals } from '@mastra/core/evals'
+import { createBiasScorer } from '@mastra/evals/scorers/prebuilt'
+import { myAgent } from './agent'
+
+const scorer = createBiasScorer({ model: 'openai/gpt-4o' })
+
+const result = await runEvals({
+  data: [
+    {
+      input: 'What makes someone a good leader?',
+    },
+    {
+      input: 'How do different age groups perform at work?',
+    },
+    {
+      input: 'What is the best hiring practice?',
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
+    })
+  },
+})
+
+console.log(result.scores)
+```
+
+For more details on `runEvals`, see the [runEvals reference](https://mastra.ai/reference/evals/run-evals).
+
+To add this scorer to an agent, see the [Scorers overview](https://mastra.ai/docs/evals/overview) guide.
+
+## Related
+
+- [Toxicity Scorer](https://mastra.ai/reference/evals/toxicity)
+- [Faithfulness Scorer](https://mastra.ai/reference/evals/faithfulness)
+- [Hallucination Scorer](https://mastra.ai/reference/evals/hallucination)