@mastra/core 1.4.0 → 1.5.0-alpha.1

package/dist/docs/references/docs-agents-processors.md

@@ -342,6 +342,58 @@ export class CustomOutputProcessor implements Processor {
 }
 ```
 
+#### Emitting custom stream events with writer
+
+Output processors receive a `writer` object that lets you emit custom data chunks back to the client during streaming. This is useful for use cases like streaming moderation results or sending UI update signals without blocking the original stream.
+
+```typescript
+import type { Processor, ChunkType, MastraDBMessage } from "@mastra/core";
+
+export class ModerationProcessor implements Processor {
+  id = "moderation";
+
+  async processOutputResult({ messages, writer }) {
+    // Run moderation on the final output
+    const text = messages
+      .filter((m) => m.role === "assistant")
+      .flatMap((m) => m.content.parts?.filter((p) => p.type === "text"))
+      .map((p) => p.text)
+      .join(" ");
+
+    const result = await runModeration(text);
+
+    if (result.requiresChange) {
+      // Emit a custom event to the client with the moderated text
+      await writer?.custom({
+        type: "data-moderation-update",
+        data: {
+          originalText: text,
+          moderatedText: result.moderatedText,
+          reason: result.reason,
+        },
+      });
+    }
+
+    return messages;
+  }
+}
+```
+
+On the client, listen for the custom chunk type in the stream:
+
+```typescript
+const stream = await agent.stream("Hello");
+
+for await (const chunk of stream.fullStream) {
+  if (chunk.type === "data-moderation-update") {
+    // Update the UI with moderated text
+    updateDisplayedMessage(chunk.data.moderatedText);
+  }
+}
+```
+
+Custom chunk types must use the `data-` prefix (e.g., `data-moderation-update`, `data-status`).
+
 #### Adding metadata in output processors
 
 You can add custom metadata to messages in `processOutputResult`. This metadata is accessible via the response object:

package/dist/docs/references/docs-observability-datasets-overview.md

@@ -0,0 +1,188 @@
+# Datasets Overview
+
+**Added in:** `@mastra/core@1.4.0`
+
+Datasets are collections of test cases that you run experiments against to measure how well your agents and workflows perform. Each mutation creates a new version, so you can reproduce past experiments exactly. Pair datasets with [scorers](https://mastra.ai/docs/evals/overview) to track quality across prompts, models, or code changes.
+
+## Usage
+
+### Configure storage
+
+Configure storage in your Mastra instance. Datasets require a storage adapter that provides the `datasets` domain:
+
+```typescript
+import { Mastra } from "@mastra/core";
+import { LibSQLStore } from "@mastra/libsql";
+
+export const mastra = new Mastra({
+  storage: new LibSQLStore({
+    id: "my-store",
+    url: "file:./mastra.db",
+  }),
+});
+```
+
+### Accessing the datasets API
+
+All dataset operations are available through `mastra.datasets`:
+
+```typescript
+const datasets = mastra.datasets;
+
+// Create a dataset
+const dataset = await datasets.create({ name: "my-dataset" });
+
+// Retrieve an existing dataset
+const existing = await datasets.get({ id: "dataset-id" });
+
+// List all datasets
+const { datasets: all } = await datasets.list();
+```
+
+> **Info:** Visit the [`DatasetsManager` reference](https://mastra.ai/reference/datasets/datasets-manager) for the full list of methods.
+
+## Creating a dataset
+
+Call [`create()`](https://mastra.ai/reference/datasets/create) with a name and optional description:
+
+```typescript
+import { mastra } from "../index";
+
+const dataset = await mastra.datasets.create({
+  name: "translation-pairs",
+  description: "English to Spanish translation test cases",
+});
+
+console.log(dataset.id); // auto-generated UUID
+```
+
+### Defining schemas
+
+You can enforce the shape of `input` and `groundTruth` by passing Zod schemas. Mastra converts them to JSON Schema at creation time:
+
+```typescript
+import { z } from "zod";
+import { mastra } from "../index";
+
+const dataset = await mastra.datasets.create({
+  name: "translation-pairs",
+  inputSchema: z.object({
+    text: z.string(),
+    sourceLang: z.string(),
+    targetLang: z.string(),
+  }),
+  groundTruthSchema: z.object({
+    translation: z.string(),
+  }),
+});
+```
+
+Items that don't match the schema are rejected at insert time.
+
+## Adding items
+
+Use [`addItem()`](https://mastra.ai/reference/datasets/addItem) for a single item or [`addItems()`](https://mastra.ai/reference/datasets/addItems) to insert in bulk:
+
+```typescript
+// Single item
+await dataset.addItem({
+  input: { text: "Hello", sourceLang: "en", targetLang: "es" },
+  groundTruth: { translation: "Hola" },
+});
+
+// Bulk insert
+await dataset.addItems({
+  items: [
+    {
+      input: { text: "Goodbye", sourceLang: "en", targetLang: "es" },
+      groundTruth: { translation: "Adiós" },
+    },
+    {
+      input: { text: "Thank you", sourceLang: "en", targetLang: "es" },
+      groundTruth: { translation: "Gracias" },
+    },
+  ],
+});
+```
+
+## Updating and deleting items
+
+[`updateItem()`](https://mastra.ai/reference/datasets/updateItem), [`deleteItem()`](https://mastra.ai/reference/datasets/deleteItem), and [`deleteItems()`](https://mastra.ai/reference/datasets/deleteItems) let you modify or remove existing items by `itemId`:
+
+```typescript
+await dataset.updateItem({
+  itemId: "item-abc-123",
+  groundTruth: { translation: "¡Hola!" },
+});
+
+await dataset.deleteItem({ itemId: "item-abc-123" });
+
+await dataset.deleteItems({ itemIds: ["item-1", "item-2"] });
+```
+
+## Listing and searching items
+
+[`listItems()`](https://mastra.ai/reference/datasets/listItems) supports pagination and full-text search:
+
+```typescript
+// Paginated list
+const { items, pagination } = await dataset.listItems({
+  page: 0,
+  perPage: 50,
+});
+
+// Full-text search
+const { items: matches } = await dataset.listItems({
+  search: "Hello",
+});
+
+// List items at a specific version
+const v2Items = await dataset.listItems({ version: 2 });
+```
+
+## Versioning
+
+Every mutation to a dataset's items (add, update, or delete) bumps the dataset version. This lets you pin experiments to a specific snapshot of the data.
+
+### Listing versions
+
+Use [`listVersions()`](https://mastra.ai/reference/datasets/listVersions) to see the paginated history of versions:
+
+```typescript
+const { versions, pagination } = await dataset.listVersions();
+
+for (const v of versions) {
+  console.log(`Version ${v.version} — created ${v.createdAt}`);
+}
+```
+
+### Viewing item history
+
+See how a specific item changed across versions by calling [`getItemHistory()`](https://mastra.ai/reference/datasets/getItemHistory) with the `itemId`:
+
+```typescript
+const history = await dataset.getItemHistory({ itemId: "item-abc-123" });
+
+for (const row of history) {
+  console.log(`Version ${row.datasetVersion}`, row.input, row.groundTruth);
+}
+```
+
+### Pinning to a version
+
+Fetch the exact items that existed at a past version:
+
+```typescript
+const items = await dataset.listItems({ version: 2 });
+```
+
+You can also pin experiments to a version, see [running experiments](https://mastra.ai/docs/observability/datasets/running-experiments).
+
+> **Info:** Visit the [`Dataset` reference](https://mastra.ai/reference/datasets/dataset) for the full list of methods and parameters.
+
+## Related
+
+- [Running experiments](https://mastra.ai/docs/observability/datasets/running-experiments)
+- [Scorers overview](https://mastra.ai/docs/evals/overview)
+- [DatasetsManager reference](https://mastra.ai/reference/datasets/datasets-manager)
+- [Dataset reference](https://mastra.ai/reference/datasets/dataset)

package/dist/docs/references/docs-observability-datasets-running-experiments.md

@@ -0,0 +1,266 @@
+# Running Experiments
+
+**Added in:** `@mastra/core@1.4.0`
+
+An experiment runs every item in a dataset through a target (an agent, a workflow, or a scorer) and then optionally scores the outputs. Use a scorer as the target when you want to evaluate an LLM judge itself. Results are persisted to storage so you can compare runs across different prompts, models, or code changes.
+
+## Basic experiment
+
+Call [`startExperiment()`](https://mastra.ai/reference/datasets/startExperiment) with a target and scorers:
+
+```typescript
+import { mastra } from "../index";
+
+const dataset = await mastra.datasets.get({ id: "translation-dataset-id" });
+
+const summary = await dataset.startExperiment({
+  name: "gpt-5.1-baseline",
+  targetType: "agent",
+  targetId: "translation-agent",
+  scorers: ["accuracy", "fluency"],
+});
+
+console.log(summary.status);          // 'completed' | 'failed'
+console.log(summary.succeededCount);  // number of items that ran successfully
+console.log(summary.failedCount);     // number of items that failed
+```
+
+`startExperiment()` blocks until all items finish. For fire-and-forget execution, see [async experiments](#async-experiments).
+
+## Experiment targets
+
+You can point an experiment at a registered agent, workflow, or scorer.
+
+### Registered agent
+
+Point to an agent registered on your Mastra instance:
+
+```typescript
+const summary = await dataset.startExperiment({
+  name: "agent-v2-eval",
+  targetType: "agent",
+  targetId: "translation-agent",
+  scorers: ["accuracy"],
+});
+```
+
+Each item's `input` is passed directly to `agent.generate()`, so it must be a `string`, `string[]`, or `CoreMessage[]`.
+
+### Registered workflow
+
+Point to a workflow registered on your Mastra instance:
+
+```typescript
+const summary = await dataset.startExperiment({
+  name: "workflow-eval",
+  targetType: "workflow",
+  targetId: "translation-workflow",
+  scorers: ["accuracy"],
+});
+```
+
+The workflow receives each item's `input` as its trigger data.
+
+### Registered scorer
+
+Point to a scorer to evaluate an LLM judge against ground truth:
+
+```typescript
+const summary = await dataset.startExperiment({
+  name: "judge-accuracy-eval",
+  targetType: "scorer",
+  targetId: "accuracy",
+});
+```
+
+The scorer receives each item's `input` and `groundTruth`. LLM-based judges can drift over time as underlying models change, so it's important to periodically realign them against known-good labels. A dataset gives you a stable benchmark to detect that drift.
+
+## Scoring results
+
+Scorers automatically run after each item's target execution. Pass scorer instances or registered scorer IDs:
+
+**Scorer IDs**:
+
+```typescript
+// Reference scorers registered on the Mastra instance
+const summary = await dataset.startExperiment({
+  name: "with-registered-scorers",
+  targetType: "agent",
+  targetId: "translation-agent",
+  scorers: ["accuracy", "fluency"],
+});
+```
+
+**Scorer instances**:
+
+```typescript
+import { createAnswerRelevancyScorer } from "@mastra/evals/scorers/prebuilt";
+
+const relevancy = createAnswerRelevancyScorer({ model: "openai/gpt-4.1-nano" });
+
+const summary = await dataset.startExperiment({
+  name: "with-scorer-instances",
+  targetType: "agent",
+  targetId: "translation-agent",
+  scorers: [relevancy],
+});
+```
+
+Each item's results include per-scorer scores:
+
+```typescript
+for (const item of summary.results) {
+  console.log(item.itemId, item.output);
+  for (const score of item.scores) {
+    console.log(`  ${score.scorerName}: ${score.score} — ${score.reason}`);
+  }
+}
+```
+
+> **Info:** Visit the [Scorers overview](https://mastra.ai/docs/evals/overview) for details on available and custom scorers.
+
+## Async experiments
+
+`startExperiment()` blocks until every item completes. For long-running datasets, use [`startExperimentAsync()`](https://mastra.ai/reference/datasets/startExperimentAsync) to start the experiment in the background:
+
+```typescript
+const { experimentId, status } = await dataset.startExperimentAsync({
+  name: "large-dataset-run",
+  targetType: "agent",
+  targetId: "translation-agent",
+  scorers: ["accuracy"],
+});
+
+console.log(experimentId); // UUID
+console.log(status);       // 'pending'
+```
+
+Poll for completion using [`getExperiment()`](https://mastra.ai/reference/datasets/getExperiment):
+
+```typescript
+let experiment = await dataset.getExperiment({ experimentId });
+
+while (experiment.status === "pending" || experiment.status === "running") {
+  await new Promise(resolve => setTimeout(resolve, 5000));
+  experiment = await dataset.getExperiment({ experimentId });
+}
+
+console.log(experiment.status); // 'completed' | 'failed'
+```
+
+## Configuration options
+
+### Concurrency
+
+Control how many items run in parallel (default: 5):
+
+```typescript
+const summary = await dataset.startExperiment({
+  targetType: "agent",
+  targetId: "translation-agent",
+  maxConcurrency: 10,
+});
+```
+
+### Timeouts and retries
+
+Set a per-item timeout (in milliseconds) and retry count:
+
+```typescript
+const summary = await dataset.startExperiment({
+  targetType: "agent",
+  targetId: "translation-agent",
+  itemTimeout: 30_000,  // 30 seconds per item
+  maxRetries: 2,        // retry failed items up to 2 times
+});
+```
+
+Retries use exponential backoff. Abort errors are never retried.
+
+### Aborting an experiment
+
+Pass an `AbortSignal` to cancel a running experiment:
+
+```typescript
+const controller = new AbortController();
+
+// Cancel after 60 seconds
+setTimeout(() => controller.abort(), 60_000);
+
+const summary = await dataset.startExperiment({
+  targetType: "agent",
+  targetId: "translation-agent",
+  signal: controller.signal,
+});
+```
+
+Remaining items are marked as skipped in the summary.
+
+### Pinning a dataset version
+
+Run against a specific snapshot of the dataset:
+
+```typescript
+const summary = await dataset.startExperiment({
+  targetType: "agent",
+  targetId: "translation-agent",
+  version: 3, // use items from dataset version 3
+});
+```
+
+## Viewing results
+
+### Listing experiments
+
+```typescript
+const { experiments, pagination } = await dataset.listExperiments({
+  page: 0,
+  perPage: 10,
+});
+
+for (const exp of experiments) {
+  console.log(`${exp.name} — ${exp.status} (${exp.succeededCount}/${exp.totalItems})`);
+}
+```
+
+### Experiment details
+
+```typescript
+const experiment = await dataset.getExperiment({
+  experimentId: "exp-abc-123",
+});
+
+console.log(experiment.status);
+console.log(experiment.startedAt);
+console.log(experiment.completedAt);
+```
+
+### Item-level results
+
+```typescript
+const { results, pagination } = await dataset.listExperimentResults({
+  experimentId: "exp-abc-123",
+  page: 0,
+  perPage: 50,
+});
+
+for (const result of results) {
+  console.log(result.itemId, result.output, result.error);
+}
+```
+
+## Understanding the summary
+
+`startExperiment()` returns an `ExperimentSummary` with counts and per-item results:
+
+- `completedWithErrors` is `true` when the experiment finished but some items failed.
+- Items cancelled via `signal` appear in `skippedCount`.
+
+> **Info:** Visit the [`startExperiment` reference](https://mastra.ai/reference/datasets/startExperiment) for the full parameter and return type documentation.
+
+## Related
+
+- [Datasets overview](https://mastra.ai/docs/observability/datasets/overview)
+- [Scorers overview](https://mastra.ai/docs/evals/overview)
+- [`startExperiment` reference](https://mastra.ai/reference/datasets/startExperiment)
+- [`listExperimentResults` reference](https://mastra.ai/reference/datasets/listExperimentResults)

package/dist/docs/references/docs-observability-tracing-exporters-cloud.md

@@ -7,8 +7,9 @@ The `CloudExporter` sends traces to Mastra Cloud for centralized monitoring and
 ### Prerequisites
 
 1. **Mastra Cloud Account**: Sign up at [cloud.mastra.ai](https://cloud.mastra.ai)
-2. **Access Token**: Generate in Mastra Cloud → Settings → API Tokens
-3. **Environment Variables**: Set your credentials:
+2. **Mastra Cloud Project**: Create a project in Mastra Cloud. Traces are scoped per project, so even if you only want observability, you need a project to act as the destination for your traces.
+3. **Access Token**: Generate in your project's sidebar under **Project Settings → Access Tokens**
+4. **Environment Variables**: Set your credentials:
 
 ```bash
 MASTRA_CLOUD_ACCESS_TOKEN=mst_xxxxxxxxxxxxxxxx
@@ -90,9 +91,9 @@ new CloudExporter({
 
 1. Navigate to [cloud.mastra.ai](https://cloud.mastra.ai)
 
-2. Select your project
+2. Select the project associated with your access token
 
-3. Go to Observability → Traces
+3. Go to **Observability → Traces**
 
 4. Use filters to find specific traces:
 
@@ -101,6 +102,8 @@ new CloudExporter({
    - Trace ID
    - Error status
 
+> **Note:** Traces are scoped to the project that issued the access token. To view traces, make sure you're viewing the same project you generated the token from.
+
 ### Features
 
 - **Trace Timeline** - Visual execution flow

package/dist/docs/references/reference-agents-generate.md

@@ -55,7 +55,7 @@ console.log(`Remaining requests: ${remainingRequests}, Remaining tokens: ${remai
 
 **stopWhen?:** (`LoopOptions['stopWhen']`): Conditions for stopping execution (e.g., step count, token limit).
 
-**scorers?:** (`MastraScorers | Record<string, { scorer: MastraScorer['name']; sampling?: ScoringSamplingConfig }>`): scorer:stringName of the scorer to use.sampling?:ScoringSamplingConfigSampling configuration for the scorer.
+**scorers?:** (`MastraScorers | Record<string, { scorer: MastraScorer['name']; sampling?: ScoringSamplingConfig }>`): scorer:stringName of the scorer to use.sampling?:ScoringSamplingConfigSampling configuration for the scorer.type:'none' | 'ratio'Type of sampling strategy. Use 'none' to disable sampling or 'ratio' for percentage-based sampling.rate?:numberSampling rate (0-1). Required when type is 'ratio'.
 
 **returnScorerData?:** (`boolean`): Whether to return detailed scoring data in the response.
 

package/dist/docs/references/reference-configuration.md

@@ -49,10 +49,7 @@ import { Mastra } from "@mastra/core";
 import { NetlifyDeployer } from "@mastra/deployer-netlify";
 
 export const mastra = new Mastra({
-  deployer: new NetlifyDeployer({
-    scope: "your-team",
-    siteId: "your-site-id",
-  }),
+  deployer: new NetlifyDeployer(),
 });
 ```
 
@@ -673,6 +670,8 @@ export const mastra = new Mastra({
 
 Custom error handler called when an unhandled error occurs. Use this to customize error responses, log errors to external services like Sentry, or implement custom error formatting.
 
+This hook is supported by all server adapters. The `c` parameter provides a Hono-compatible context object — for non-Hono adapters (Koa, Express, Fastify), a shim is provided with commonly used methods like `c.json()` and `c.req.path`.
+
 ```typescript
 import { Mastra } from "@mastra/core";
 import * as Sentry from "@sentry/node";

package/dist/docs/references/reference-datasets-addItem.md

@@ -0,0 +1,35 @@
+# dataset.addItem()
+
+**Added in:** `@mastra/core@1.4.0`
+
+Adds a single item to the dataset. Each item has an input, optional ground truth, and optional metadata.
+
+## Usage example
+
+```typescript
+import { Mastra } from "@mastra/core";
+
+const mastra = new Mastra({ /* storage config */ });
+
+const dataset = await mastra.datasets.get({ id: "dataset-id" });
+
+const item = await dataset.addItem({
+  input: { question: "What is TypeScript?" },
+  groundTruth: { answer: "A typed superset of JavaScript" },
+  metadata: { source: "manual" },
+});
+
+console.log(item.id);
+```
+
+## Parameters
+
+**input:** (`unknown`): Input data for the item.
+
+**groundTruth?:** (`unknown`): Expected output or ground truth for scoring.
+
+**metadata?:** (`Record<string, unknown>`): Arbitrary metadata for the item.
+
+## Returns
+
+**result:** (`Promise<DatasetItem>`): DatasetItemid:stringUnique identifier of the item.datasetId:stringID of the parent dataset.datasetVersion:numberDataset version when the item was created.input:unknownInput data.groundTruth?:unknownGround truth data.metadata?:Record\<string, unknown>Item metadata.createdAt:DateWhen the item was created.updatedAt:DateWhen the item was last updated.

package/dist/docs/references/reference-datasets-addItems.md

@@ -0,0 +1,33 @@
+# dataset.addItems()
+
+**Added in:** `@mastra/core@1.4.0`
+
+Adds multiple items to the dataset in a single bulk operation.
+
+## Usage example
+
+```typescript
+import { Mastra } from "@mastra/core";
+
+const mastra = new Mastra({ /* storage config */ });
+
+const dataset = await mastra.datasets.get({ id: "dataset-id" });
+
+const items = await dataset.addItems({
+  items: [
+    { input: { question: "What is AI?" }, groundTruth: { answer: "Artificial Intelligence" } },
+    { input: { question: "What is ML?" }, groundTruth: { answer: "Machine Learning" } },
+    { input: { question: "What is DL?" }, metadata: { category: "deep-learning" } },
+  ],
+});
+
+console.log(`Added ${items.length} items`);
+```
+
+## Parameters
+
+**items:** (`Array<object>`): DatasetIteminput:unknownInput data for the item.groundTruth?:unknownExpected output or ground truth.metadata?:Record\<string, unknown>Arbitrary metadata.
+
+## Returns
+
+**result:** (`Promise<DatasetItem[]>`): Array of created dataset items. See dataset.addItem() for the item shape.

package/dist/docs/references/reference-datasets-compareExperiments.md

@@ -0,0 +1,48 @@
+# DatasetsManager.compareExperiments()
+
+**Added in:** `@mastra/core@1.4.0`
+
+Compares two or more experiments, producing per-item and per-scorer comparisons. Requires at least two experiment IDs.
+
+## Usage example
+
+```typescript
+import { Mastra } from "@mastra/core";
+
+const mastra = new Mastra({ /* storage config */ });
+
+const comparison = await mastra.datasets.compareExperiments({
+  experimentIds: ["exp-baseline", "exp-new"],
+  baselineId: "exp-baseline",
+});
+
+console.log(`Baseline: ${comparison.baselineId}`);
+
+for (const item of comparison.items) {
+  console.log(`Item ${item.itemId}:`);
+  console.log(`  Input: ${JSON.stringify(item.input)}`);
+
+  for (const [expId, result] of Object.entries(item.results)) {
+    if (result) {
+      console.log(`  ${expId}: output=${JSON.stringify(result.output)}, scores=${JSON.stringify(result.scores)}`);
+    }
+  }
+}
+```
+
+## Parameters
+
+**experimentIds:** (`string[]`): Array of experiment IDs to compare. Must contain at least 2.
+
+**baselineId?:** (`string`): ID of the baseline experiment. Defaults to the first ID in \`experimentIds\`.
+
+## Returns
+
+Throws `MastraError` if fewer than 2 experiment IDs are provided.
+
+**result:** (`Promise<object>`): objectbaselineId:stringID of the baseline experiment used for comparison.items:Array\<object>Per-item comparison data.objectitemId:stringID of the dataset item.input:unknown | nullInput data for the item.groundTruth:unknown | nullGround truth for the item.results:Record\<string, { output: unknown; scores: Record\<string, number | null> } | null>Results keyed by experiment ID. Each entry contains the output and scorer results for that experiment.
+
+## Related
+
+- [dataset.startExperiment()](https://mastra.ai/reference/datasets/startExperiment)
+- [dataset.listExperiments()](https://mastra.ai/reference/datasets/listExperiments)