@mastra/mcp-docs-server 0.13.39 → 1.0.0-beta.1

package/.docs/raw/reference/{scorers → evals}/create-scorer.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Reference: createScorer | Scorers | Mastra Docs"
+title: "Reference: createScorer | Evals | Mastra Docs"
 description: Documentation for creating custom scorers in Mastra, allowing users to define their own evaluation logic using either JavaScript functions or LLM-based prompts.
 ---
 
@@ -12,8 +12,11 @@ Mastra provides a unified `createScorer` factory that allows you to define custo
 Use the `createScorer` factory to define your scorer with a name, description, and optional judge configuration. Then chain step methods to build your evaluation pipeline. You must provide at least a `generateScore` step.
 
 ```typescript
+import { createScorer } from "@mastra/core/evals";
+
 const scorer = createScorer({
-  name: "My Custom Scorer",
+  id: "my-custom-scorer",
+  name: "My Custom Scorer", // Optional, defaults to id
   description: "Evaluates responses based on custom criteria",
   type: "agent", // Optional: for agent evaluation with automatic typing
   judge: {
@@ -39,29 +42,35 @@ const scorer = createScorer({
 
 <PropertiesTable
   content={[
+    {
+      name: "id",
+      type: "string",
+      isOptional: false,
+      description: "Unique identifier for the scorer. Used as the name if `name` is not provided.",
+    },
     {
       name: "name",
       type: "string",
-      required: true,
-      description: "Name of the scorer.",
+      isOptional: true,
+      description: "Name of the scorer. Defaults to `id` if not provided.",
     },
     {
       name: "description",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Description of what the scorer does.",
     },
     {
       name: "judge",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
         "Optional judge configuration for LLM-based steps. See Judge Object section below.",
     },
     {
       name: "type",
       type: "string",
-      required: false,
+      isOptional: true,
       description:
         "Type specification for input/output. Use 'agent' for automatic agent types. For custom types, use the generic approach instead.",
     },
@@ -77,13 +86,13 @@ This function returns a scorer builder that you can chain step methods onto. See
     {
       name: "model",
       type: "LanguageModel",
-      required: true,
+      isOptional: false,
       description: "The LLM model instance to use for evaluation.",
     },
     {
       name: "instructions",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "System prompt/instructions for the LLM.",
     },
   ]}
@@ -98,7 +107,7 @@ You can specify input/output types when creating scorers for better type inferen
 For evaluating agents, use `type: 'agent'` to automatically get the correct types for agent input/output:
 
 ```typescript
-import { createScorer } from "@mastra/core/scorers";
+import { createScorer } from "@mastra/core/evals";
 
 // Agent scorer with automatic typing
 const agentScorer = createScorer({
@@ -108,7 +117,7 @@ const agentScorer = createScorer({
 })
   .preprocess(({ run }) => {
     // run.input is automatically typed as ScorerRunInputForAgent
-    const userMessage = run.input.inputMessages[0]?.content;
+    const userMessage = run.inputData.inputMessages[0]?.content;
     return { userMessage };
   })
   .generateScore(({ run, results }) => {
@@ -123,7 +132,7 @@ const agentScorer = createScorer({
 For custom input/output types, use the generic approach:
 
 ```typescript
-import { createScorer } from "@mastra/core/scorers";
+import { createScorer } from "@mastra/core/evals";
 
 type CustomInput = { query: string; context: string[] };
 type CustomOutput = { answer: string; confidence: number };
@@ -156,7 +165,7 @@ const agentTraceScorer = createScorer({
   type: "agent",
 }).generateScore(({ run }) => {
   // Trace data is automatically transformed to agent format
-  const userMessages = run.input.inputMessages;
+  const userMessages = run.inputData.inputMessages;
   const agentResponse = run.output[0]?.content;
 
   // Score based on response length
@@ -185,34 +194,34 @@ Function: `({ run, results }) => any`
     {
       name: "run.input",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
     },
     {
       name: "run.output",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
     },
     {
       name: "run.runId",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Unique identifier for this scoring run.",
     },
     {
-      name: "run.runtimeContext",
+      name: "run.requestContext",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
-        "Runtime context from the agent or workflow step being evaluated (optional).",
+        "Request Context from the agent or workflow step being evaluated (optional).",
     },
     {
       name: "results",
       type: "object",
-      required: true,
+      isOptional: false,
       description: "Empty object (no previous steps).",
     },
   ]}
@@ -228,26 +237,26 @@ The method can return any value. The returned value will be available to subsequ
     {
       name: "description",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Description of what this preprocessing step does.",
     },
     {
       name: "outputSchema",
       type: "ZodSchema",
-      required: true,
+      isOptional: false,
       description: "Zod schema for the expected output of the preprocess step.",
     },
     {
       name: "createPrompt",
       type: "function",
-      required: true,
+      isOptional: false,
       description:
         "Function: ({ run, results }) => string. Returns the prompt for the LLM.",
     },
     {
       name: "judge",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
         "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
     },
@@ -266,34 +275,34 @@ Function: `({ run, results }) => any`
     {
       name: "run.input",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
     },
     {
       name: "run.output",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
     },
     {
       name: "run.runId",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Unique identifier for this scoring run.",
     },
     {
-      name: "run.runtimeContext",
+      name: "run.requestContext",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
-        "Runtime context from the agent or workflow step being evaluated (optional).",
+        "Request Context from the agent or workflow step being evaluated (optional).",
     },
     {
       name: "results.preprocessStepResult",
       type: "any",
-      required: false,
+      isOptional: true,
       description: "Result from preprocess step, if defined (optional).",
     },
   ]}
@@ -309,26 +318,26 @@ The method can return any value. The returned value will be available to subsequ
     {
       name: "description",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Description of what this analysis step does.",
     },
     {
       name: "outputSchema",
       type: "ZodSchema",
-      required: true,
+      isOptional: false,
       description: "Zod schema for the expected output of the analyze step.",
     },
     {
       name: "createPrompt",
       type: "function",
-      required: true,
+      isOptional: false,
       description:
         "Function: ({ run, results }) => string. Returns the prompt for the LLM.",
     },
     {
       name: "judge",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
         "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
     },
@@ -347,40 +356,40 @@ Function: `({ run, results }) => number`
     {
       name: "run.input",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
     },
     {
       name: "run.output",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
     },
     {
       name: "run.runId",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Unique identifier for this scoring run.",
     },
     {
-      name: "run.runtimeContext",
+      name: "run.requestContext",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
-        "Runtime context from the agent or workflow step being evaluated (optional).",
+        "Request Context from the agent or workflow step being evaluated (optional).",
     },
     {
       name: "results.preprocessStepResult",
       type: "any",
-      required: false,
+      isOptional: true,
       description: "Result from preprocess step, if defined (optional).",
     },
     {
       name: "results.analyzeStepResult",
       type: "any",
-      required: false,
+      isOptional: true,
       description: "Result from analyze step, if defined (optional).",
     },
   ]}
@@ -396,27 +405,27 @@ The method must return a numerical score.
     {
       name: "description",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Description of what this scoring step does.",
     },
     {
       name: "outputSchema",
       type: "ZodSchema",
-      required: true,
+      isOptional: false,
       description:
         "Zod schema for the expected output of the generateScore step.",
     },
     {
       name: "createPrompt",
       type: "function",
-      required: true,
+      isOptional: false,
       description:
         "Function: ({ run, results }) => string. Returns the prompt for the LLM.",
     },
     {
       name: "judge",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
         "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
     },
@@ -430,7 +439,7 @@ When using prompt object mode, you must also provide a `calculateScore` function
     {
       name: "calculateScore",
       type: "function",
-      required: true,
+      isOptional: false,
       description:
         "Function: ({ run, results, analyzeStepResult }) => number. Converts the LLM's structured output into a numerical score.",
     },
@@ -449,46 +458,46 @@ Function: `({ run, results, score }) => string`
     {
       name: "run.input",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
     },
     {
       name: "run.output",
       type: "any",
-      required: true,
+      isOptional: false,
       description:
         "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
     },
     {
       name: "run.runId",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Unique identifier for this scoring run.",
     },
     {
-      name: "run.runtimeContext",
+      name: "run.requestContext",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
-        "Runtime context from the agent or workflow step being evaluated (optional).",
+        "Request Context from the agent or workflow step being evaluated (optional).",
     },
     {
       name: "results.preprocessStepResult",
       type: "any",
-      required: false,
+      isOptional: true,
       description: "Result from preprocess step, if defined (optional).",
     },
     {
       name: "results.analyzeStepResult",
       type: "any",
-      required: false,
+      isOptional: true,
       description: "Result from analyze step, if defined (optional).",
     },
     {
       name: "score",
       type: "number",
-      required: true,
+      isOptional: false,
       description: "Score computed by the generateScore step.",
     },
   ]}
@@ -504,20 +513,20 @@ The method must return a string explaining the score.
     {
       name: "description",
       type: "string",
-      required: true,
+      isOptional: false,
       description: "Description of what this reasoning step does.",
     },
     {
       name: "createPrompt",
       type: "function",
-      required: true,
+      isOptional: false,
       description:
         "Function: ({ run, results, score }) => string. Returns the prompt for the LLM.",
     },
     {
       name: "judge",
       type: "object",
-      required: false,
+      isOptional: true,
       description:
         "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
     },