create-mastra 0.17.1-alpha.0 → 0.17.1-alpha.1

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # create-mastra
 
+## 0.17.1-alpha.1
+
+### Patch Changes
+
+- Add scorers to the default weather agent in the create command. ([#9042](https://github.com/mastra-ai/mastra/pull/9042))
+
+- Fix tool result in playground ([#9087](https://github.com/mastra-ai/mastra/pull/9087))
+
 ## 0.17.1-alpha.0
 
 ## 0.17.0

package/dist/index.js

@@ -724,13 +724,18 @@ const format = (open, close) => {
 		// Handle nested colors.
 
 		// We could have done this, but it's too slow (as of Node.js 22).
-		// return openCode + string.replaceAll(closeCode, openCode) + closeCode;
+		// return openCode + string.replaceAll(closeCode, (close === 22 ? closeCode : '') + openCode) + closeCode;
 
 		let result = openCode;
 		let lastIndex = 0;
 
+		// SGR 22 resets both bold (1) and dim (2). When we encounter a nested
+		// close for styles that use 22, we need to re-open the outer style.
+		const reopenOnNestedClose = close === 22;
+		const replaceCode = (reopenOnNestedClose ? closeCode : '') + openCode;
+
 		while (index !== -1) {
-			result += string.slice(lastIndex, index) + openCode;
+			result += string.slice(lastIndex, index) + replaceCode;
 			lastIndex = index + closeCode.length;
 			index = string.indexOf(closeCode, lastIndex);
 		}
@@ -1503,7 +1508,7 @@ var getModelIdentifier = (llmProvider) => {
     return `'mistral/mistral-medium-2508'`;
   }
 };
-async function writeAgentSample(llmProvider, destPath, addExampleTool) {
+async function writeAgentSample(llmProvider, destPath, addExampleTool, addScorers) {
   const modelString = getModelIdentifier(llmProvider);
   const instructions = `
       You are a helpful weather assistant that provides accurate weather information and can help planning activities based on the weather.
@@ -1524,12 +1529,36 @@ import { Agent } from '@mastra/core/agent';
 import { Memory } from '@mastra/memory';
 import { LibSQLStore } from '@mastra/libsql';
 ${addExampleTool ? `import { weatherTool } from '../tools/weather-tool';` : ""}
+${addScorers ? `import { scorers } from '../scorers/weather-scorer';` : ""}
 
 export const weatherAgent = new Agent({
   name: 'Weather Agent',
   instructions: \`${instructions}\`,
   model: ${modelString},
   ${addExampleTool ? "tools: { weatherTool }," : ""}
+  ${addScorers ? `scorers: {
+    toolCallAppropriateness: {
+      scorer: scorers.toolCallAppropriatenessScorer,
+      sampling: {
+        type: 'ratio',
+        rate: 1,
+      },
+    },
+    completeness: {
+      scorer: scorers.completenessScorer,
+      sampling: {
+        type: 'ratio',
+        rate: 1,
+      },
+    },
+    translation: {
+      scorer: scorers.translationScorer,
+      sampling: {
+        type: 'ratio',
+        rate: 1,
+      },
+    },
+  },` : ""}
   memory: new Memory({
     storage: new LibSQLStore({
       url: "file:../mastra.db", // path is relative to the .mastra/output directory
@@ -1741,14 +1770,106 @@ async function writeToolSample(destPath) {
   const fileService = new FileService();
   await fileService.copyStarterFile("tools.ts", destPath);
 }
+async function writeScorersSample(destPath) {
+  const content = `import { z } from 'zod';
+import { createToolCallAccuracyScorerCode } from '@mastra/evals/scorers/code';
+import { createCompletenessScorer } from '@mastra/evals/scorers/code';
+import { createScorer } from '@mastra/core/scores';
+
+export const toolCallAppropriatenessScorer = createToolCallAccuracyScorerCode({
+  expectedTool: 'weatherTool',
+  strictMode: false,
+});
+
+export const completenessScorer = createCompletenessScorer();
+
+// Custom LLM-judged scorer: evaluates if non-English locations are translated appropriately
+export const translationScorer = createScorer({
+  name: 'Translation Quality',
+  description: 'Checks that non-English location names are translated and used correctly',
+  type: 'agent',
+  judge: {
+    model: 'openai/gpt-4o-mini',
+    instructions:
+      'You are an expert evaluator of translation quality for geographic locations. ' +
+      'Determine whether the user text mentions a non-English location and whether the assistant correctly uses an English translation of that location. ' +
+      'Be lenient with transliteration differences and diacritics. ' +
+      'Return only the structured JSON matching the provided schema.',
+  },
+})
+  .preprocess(({ run }) => {
+    const userText = (run.input?.inputMessages?.[0]?.content as string) || '';
+    const assistantText = (run.output?.[0]?.content as string) || '';
+    return { userText, assistantText };
+  })
+  .analyze({
+    description: 'Extract location names and detect language/translation adequacy',
+    outputSchema: z.object({
+      nonEnglish: z.boolean(),
+      translated: z.boolean(),
+      confidence: z.number().min(0).max(1).default(1),
+      explanation: z.string().default(''),
+    }),
+    createPrompt: ({ results }) => \`
+            You are evaluating if a weather assistant correctly handled translation of a non-English location.
+            User text:
+            """
+            \${results.preprocessStepResult.userText}
+            """
+            Assistant response:
+            """
+            \${results.preprocessStepResult.assistantText}
+            """
+            Tasks:
+            1) Identify if the user mentioned a location that appears non-English.
+            2) If non-English, check whether the assistant used a correct English translation of that location in its response.
+            3) Be lenient with transliteration differences (e.g., accents/diacritics).
+            Return JSON with fields:
+            {
+            "nonEnglish": boolean,
+            "translated": boolean,
+            "confidence": number, // 0-1
+            "explanation": string
+            }
+        \`,
+  })
+  .generateScore(({ results }) => {
+    const r = (results as any)?.analyzeStepResult || {};
+    if (!r.nonEnglish) return 1; // If not applicable, full credit
+    if (r.translated) return Math.max(0, Math.min(1, 0.7 + 0.3 * (r.confidence ?? 1)));
+    return 0; // Non-English but not translated
+  })
+  .generateReason(({ results, score }) => {
+    const r = (results as any)?.analyzeStepResult || {};
+    return \`Translation scoring: nonEnglish=\${r.nonEnglish ?? false}, translated=\${r.translated ?? false}, confidence=\${r.confidence ?? 0}. Score=\${score}. \${r.explanation ?? ''}\`;
+  });
+
+export const scorers = {
+  toolCallAppropriatenessScorer,
+  completenessScorer,
+  translationScorer,
+};`;
+  const formattedContent = await prettier.format(content, {
+    parser: "typescript",
+    singleQuote: true
+  });
+  await fs4.writeFile(destPath, formattedContent);
+}
 async function writeCodeSampleForComponents(llmprovider, component, destPath, importComponents) {
   switch (component) {
     case "agents":
-      return writeAgentSample(llmprovider, destPath, importComponents.includes("tools"));
+      return writeAgentSample(
+        llmprovider,
+        destPath,
+        importComponents.includes("tools"),
+        importComponents.includes("scorers")
+      );
     case "tools":
       return writeToolSample(destPath);
     case "workflows":
       return writeWorkflowSample(destPath);
+    case "scorers":
+      return writeScorersSample(destPath);
     default:
       return "";
   }
@@ -1761,7 +1882,8 @@ var writeIndexFile = async ({
   dirPath,
   addAgent,
   addExample,
-  addWorkflow
+  addWorkflow,
+  addScorers
 }) => {
   const indexPath = dirPath + "/index.ts";
   const destPath = path.join(indexPath);
@@ -1769,7 +1891,8 @@ var writeIndexFile = async ({
     await fs4.writeFile(destPath, "");
     const filteredExports = [
       addWorkflow ? `workflows: { weatherWorkflow },` : "",
-      addAgent ? `agents: { weatherAgent },` : ""
+      addAgent ? `agents: { weatherAgent },` : "",
+      addScorers ? `scorers: { toolCallAppropriatenessScorer, completenessScorer, translationScorer },` : ""
     ].filter(Boolean);
     if (!addExample) {
       await fs4.writeFile(
@@ -1790,6 +1913,7 @@ import { PinoLogger } from '@mastra/loggers';
 import { LibSQLStore } from '@mastra/libsql';
 ${addWorkflow ? `import { weatherWorkflow } from './workflows/weather-workflow';` : ""}
 ${addAgent ? `import { weatherAgent } from './agents/weather-agent';` : ""}
+${addScorers ? `import { toolCallAppropriatenessScorer, completenessScorer, translationScorer } from './scorers/weather-scorer';` : ""}
 
 export const mastra = new Mastra({
   ${filteredExports.join("\n  ")}
@@ -2198,7 +2322,8 @@ var init = async ({
         dirPath,
         addExample,
         addWorkflow: components.includes("workflows"),
-        addAgent: components.includes("agents")
+        addAgent: components.includes("agents"),
+        addScorers: components.includes("scorers")
       }),
       ...components.map((component) => createComponentsDir(dirPath, component)),
       writeAPIKey({ provider: llmProvider, apiKey: llmApiKey })
@@ -2222,6 +2347,10 @@ var init = async ({
       if (needsLoggers) {
         await depService.installPackages(["@mastra/loggers"]);
       }
+      const needsEvals = components.includes(`scorers`) && await depService.checkDependencies(["@mastra/evals"]) !== `ok`;
+      if (needsEvals) {
+        await depService.installPackages(["@mastra/evals"]);
+      }
     }
     const key = await getAPIKey(llmProvider || "openai");
     if (configureEditorWithDocsMCP) {
@@ -2451,7 +2580,7 @@ var create = async (args2) => {
     await init({
       ...result,
       llmApiKey: result?.llmApiKey,
-      components: ["agents", "tools", "workflows"],
+      components: ["agents", "tools", "workflows", "scorers"],
       addExample: true
     });
     postCreate({ projectName });
@@ -2673,7 +2802,7 @@ program.version(`${version}`, "-v, --version").description(`create-mastra ${vers
 program.name("create-mastra").description("Create a new Mastra project").argument("[project-name]", "Directory name of the project").option(
   "-p, --project-name <string>",
   "Project name that will be used in package.json and as the project directory name."
-).option("--default", "Quick start with defaults (src, OpenAI, examples)").option("-c, --components <components>", "Comma-separated list of components (agents, tools, workflows)").option("-l, --llm <model-provider>", "Default model provider (openai, anthropic, groq, google, or cerebras)").option("-k, --llm-api-key <api-key>", "API key for the model provider").option("-e, --example", "Include example code").option("-n, --no-example", "Do not include example code").option("-t, --timeout [timeout]", "Configurable timeout for package installation, defaults to 60000 ms").option("-d, --dir <directory>", "Target directory for Mastra source code (default: src/)").option("-m, --mcp <mcp>", "MCP Server for code editor (cursor, cursor-global, windsurf, vscode)").option(
+).option("--default", "Quick start with defaults (src, OpenAI, examples)").option("-c, --components <components>", "Comma-separated list of components (agents, tools, workflows, scorers)").option("-l, --llm <model-provider>", "Default model provider (openai, anthropic, groq, google, or cerebras)").option("-k, --llm-api-key <api-key>", "API key for the model provider").option("-e, --example", "Include example code").option("-n, --no-example", "Do not include example code").option("-t, --timeout [timeout]", "Configurable timeout for package installation, defaults to 60000 ms").option("-d, --dir <directory>", "Target directory for Mastra source code (default: src/)").option("-m, --mcp <mcp>", "MCP Server for code editor (cursor, cursor-global, windsurf, vscode)").option(
   "--template [template-name]",
   "Create project from a template (use template name, public GitHub URL, or leave blank to select from list)"
 ).action(async (projectNameArg, args) => {
@@ -2681,7 +2810,7 @@ program.name("create-mastra").description("Create a new Mastra project").argumen
   const timeout = args?.timeout ? args?.timeout === true ? 6e4 : parseInt(args?.timeout, 10) : void 0;
   if (args.default) {
     await create({
-      components: ["agents", "tools", "workflows"],
+      components: ["agents", "tools", "workflows", "scorers"],
       llmProvider: "openai",
       addExample: true,
       createVersionTag,