@mastra/server 1.81.0-alpha.1 → 1.81.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_types/@internal_core/dist/storage/index.d.ts +11 -0
- package/dist/{a2a-CSz0rw8E.js → a2a-UYs7rzN2.js} +2 -2
- package/dist/{a2a-CSz0rw8E.js.map → a2a-UYs7rzN2.js.map} +1 -1
- package/dist/{a2a-CmMIWPuz.cjs → a2a-eUB3oDRN.cjs} +2 -2
- package/dist/{a2a-CmMIWPuz.cjs.map → a2a-eUB3oDRN.cjs.map} +1 -1
- package/dist/{agent-builder-XhBB0Svr.js → agent-builder-5dpawFJf.js} +2 -2
- package/dist/{agent-builder-XhBB0Svr.js.map → agent-builder-5dpawFJf.js.map} +1 -1
- package/dist/{agent-builder-DIEX-as5.cjs → agent-builder-DwfHzUnt.cjs} +2 -2
- package/dist/{agent-builder-DIEX-as5.cjs.map → agent-builder-DwfHzUnt.cjs.map} +1 -1
- package/dist/{agents-D36tukip.cjs → agents-BjdRPzyV.cjs} +3 -2
- package/dist/agents-BjdRPzyV.cjs.map +1 -0
- package/dist/{agents-BVjnYswL.js → agents-BuI0JjjX.js} +3 -2
- package/dist/agents-BuI0JjjX.js.map +1 -0
- package/dist/{api-schema-manifest-BVPhF1ZS.js → api-schema-manifest-DafznVYs.js} +2 -2
- package/dist/{api-schema-manifest-BVPhF1ZS.js.map → api-schema-manifest-DafznVYs.js.map} +1 -1
- package/dist/{api-schema-manifest-BWId44SV.cjs → api-schema-manifest-GgyfF4dw.cjs} +2 -2
- package/dist/{api-schema-manifest-BWId44SV.cjs.map → api-schema-manifest-GgyfF4dw.cjs.map} +1 -1
- package/dist/{builder-registry-DWusXgse.cjs → builder-registry-CamBJItc.cjs} +2 -2
- package/dist/{builder-registry-DWusXgse.cjs.map → builder-registry-CamBJItc.cjs.map} +1 -1
- package/dist/{builder-registry-CtGv0Gj6.js → builder-registry-DHVK50Ba.js} +2 -2
- package/dist/{builder-registry-CtGv0Gj6.js.map → builder-registry-DHVK50Ba.js.map} +1 -1
- package/dist/{conversations-CJn0ioOZ.cjs → conversations-ORTpTqBg.cjs} +2 -2
- package/dist/{conversations-CJn0ioOZ.cjs.map → conversations-ORTpTqBg.cjs.map} +1 -1
- package/dist/{conversations-CABVTBQY.js → conversations-fdXV9K0J.js} +2 -2
- package/dist/{conversations-CABVTBQY.js.map → conversations-fdXV9K0J.js.map} +1 -1
- package/dist/{dist-BK1fCE4z.cjs → dist-BEtxPxkM.cjs} +1 -1
- package/dist/dist-BQ1uFC_1.js +1 -0
- package/dist/{dist-OpNBTIST.cjs → dist-CR3PolEz.cjs} +5 -4
- package/dist/{dist-OpNBTIST.cjs.map → dist-CR3PolEz.cjs.map} +1 -1
- package/dist/{dist-BLjE21MA.js → dist-CmG4sN60.js} +5 -4
- package/dist/{dist-BLjE21MA.js.map → dist-CmG4sN60.js.map} +1 -1
- package/dist/docs/SKILL.md +2 -2
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/reference-observability-tracing-advanced-queries.md +226 -86
- package/dist/docs/references/reference-server-routes.md +18 -14
- package/dist/{editor-builder-CeiISBIO.cjs → editor-builder-6l27BFzD.cjs} +2 -2
- package/dist/{editor-builder-CeiISBIO.cjs.map → editor-builder-6l27BFzD.cjs.map} +1 -1
- package/dist/{editor-builder-DeD6EVm8.js → editor-builder-DxGZ7IN5.js} +2 -2
- package/dist/{editor-builder-DeD6EVm8.js.map → editor-builder-DxGZ7IN5.js.map} +1 -1
- package/dist/{mcp-Bt3h1lO7.cjs → mcp-B7Q8qwjn.cjs} +118 -3
- package/dist/mcp-B7Q8qwjn.cjs.map +1 -0
- package/dist/{mcp-mpYrlkUA.cjs → mcp-CMw0sq0y.cjs} +177 -4
- package/dist/mcp-CMw0sq0y.cjs.map +1 -0
- package/dist/{mcp-D4Cprr8R.js → mcp-CQ_pR-JC.js} +77 -4
- package/dist/mcp-CQ_pR-JC.js.map +1 -0
- package/dist/{mcp-D3F3iYew.js → mcp-DN9mT9q4.js} +154 -5
- package/dist/mcp-DN9mT9q4.js.map +1 -0
- package/dist/{observability-DHuljMXO.cjs → observability-DRbDeN1X.cjs} +2 -2
- package/dist/{observability-DHuljMXO.cjs.map → observability-DRbDeN1X.cjs.map} +1 -1
- package/dist/{observability-new-endpoints-CaTskbSh.cjs → observability-new-endpoints-BrB2CKix.cjs} +15 -4
- package/dist/observability-new-endpoints-BrB2CKix.cjs.map +1 -0
- package/dist/{observability-new-endpoints-CsUP8y2j.js → observability-new-endpoints-CQNVjYYq.js} +16 -5
- package/dist/observability-new-endpoints-CQNVjYYq.js.map +1 -0
- package/dist/{observability-BgIU0otL.js → observability-pX3wmG8T.js} +2 -2
- package/dist/{observability-BgIU0otL.js.map → observability-pX3wmG8T.js.map} +1 -1
- package/dist/{plans-Dydti0sT.cjs → plans-BHQFsFRU.cjs} +2 -2
- package/dist/{plans-Dydti0sT.cjs.map → plans-BHQFsFRU.cjs.map} +1 -1
- package/dist/{plans-B5GgYqx-.js → plans-D9BGMVkL.js} +2 -2
- package/dist/{plans-B5GgYqx-.js.map → plans-D9BGMVkL.js.map} +1 -1
- package/dist/{probe-image-size-BqH96zcz.js → probe-image-size-BjNoJxWf.js} +2 -2
- package/dist/{probe-image-size-BqH96zcz.js.map → probe-image-size-BjNoJxWf.js.map} +1 -1
- package/dist/{probe-image-size-BIZUe8Bp.cjs → probe-image-size-jS0ECSlc.cjs} +2 -2
- package/dist/{probe-image-size-BIZUe8Bp.cjs.map → probe-image-size-jS0ECSlc.cjs.map} +1 -1
- package/dist/{responses-CxvSoOLt.cjs → responses-4_RCBNJB.cjs} +2 -2
- package/dist/{responses-CxvSoOLt.cjs.map → responses-4_RCBNJB.cjs.map} +1 -1
- package/dist/{responses-DbiqhxP3.js → responses-Cb-9JV9A.js} +2 -2
- package/dist/{responses-DbiqhxP3.js.map → responses-Cb-9JV9A.js.map} +1 -1
- package/dist/{routes-CBrxFxzL.js → routes-4X2kSYEG.js} +19 -15
- package/dist/{routes-CBrxFxzL.js.map → routes-4X2kSYEG.js.map} +1 -1
- package/dist/{routes-By3cLbPV.cjs → routes-BqWcvAcz.cjs} +19 -15
- package/dist/routes-BqWcvAcz.cjs.map +1 -0
- package/dist/{scores-CnozEyM4.cjs → scores-DnEcBe2i.cjs} +2 -2
- package/dist/{scores-CnozEyM4.cjs.map → scores-DnEcBe2i.cjs.map} +1 -1
- package/dist/{scores-BjclMqFa.js → scores-DqImg-o-.js} +2 -2
- package/dist/{scores-BjclMqFa.js.map → scores-DqImg-o-.js.map} +1 -1
- package/dist/server/handlers/a2a.cjs +1 -1
- package/dist/server/handlers/a2a.js +1 -1
- package/dist/server/handlers/agent-builder.cjs +1 -1
- package/dist/server/handlers/agent-builder.js +1 -1
- package/dist/server/handlers/agents.cjs +1 -1
- package/dist/server/handlers/agents.d.ts.map +1 -1
- package/dist/server/handlers/agents.js +1 -1
- package/dist/server/handlers/builder-registry.cjs +1 -1
- package/dist/server/handlers/builder-registry.js +1 -1
- package/dist/server/handlers/conversations.cjs +1 -1
- package/dist/server/handlers/conversations.js +1 -1
- package/dist/server/handlers/editor-builder.cjs +1 -1
- package/dist/server/handlers/editor-builder.js +1 -1
- package/dist/server/handlers/error.cjs +5 -4
- package/dist/server/handlers/error.cjs.map +1 -1
- package/dist/server/handlers/error.d.ts.map +1 -1
- package/dist/server/handlers/error.js +5 -4
- package/dist/server/handlers/error.js.map +1 -1
- package/dist/server/handlers/favorites-enrichment.cjs +1 -1
- package/dist/server/handlers/favorites-enrichment.js +1 -1
- package/dist/server/handlers/mcp.cjs +5 -1
- package/dist/server/handlers/mcp.d.ts +507 -245
- package/dist/server/handlers/mcp.d.ts.map +1 -1
- package/dist/server/handlers/mcp.js +1 -1
- package/dist/server/handlers/observability-new-endpoints.cjs +1 -1
- package/dist/server/handlers/observability-new-endpoints.d.ts +80 -16
- package/dist/server/handlers/observability-new-endpoints.d.ts.map +1 -1
- package/dist/server/handlers/observability-new-endpoints.js +1 -1
- package/dist/server/handlers/observability-shared.cjs +56 -0
- package/dist/server/handlers/observability-shared.cjs.map +1 -1
- package/dist/server/handlers/observability-shared.d.ts +29 -0
- package/dist/server/handlers/observability-shared.d.ts.map +1 -1
- package/dist/server/handlers/observability-shared.js +50 -1
- package/dist/server/handlers/observability-shared.js.map +1 -1
- package/dist/server/handlers/observability.cjs +2 -2
- package/dist/server/handlers/observability.js +2 -2
- package/dist/server/handlers/plans.cjs +1 -1
- package/dist/server/handlers/plans.js +1 -1
- package/dist/server/handlers/responses.cjs +1 -1
- package/dist/server/handlers/responses.js +1 -1
- package/dist/server/handlers/scores.cjs +1 -1
- package/dist/server/handlers/scores.js +1 -1
- package/dist/server/handlers/stored-agent-favorites.cjs +1 -1
- package/dist/server/handlers/stored-agent-favorites.js +1 -1
- package/dist/server/handlers/stored-agents.cjs +1 -1
- package/dist/server/handlers/stored-agents.js +1 -1
- package/dist/server/handlers/stored-skill-favorites.cjs +1 -1
- package/dist/server/handlers/stored-skill-favorites.js +1 -1
- package/dist/server/handlers/stored-skills.cjs +1 -1
- package/dist/server/handlers/stored-skills.js +1 -1
- package/dist/server/handlers/system.cjs +2 -2
- package/dist/server/handlers/system.d.ts +8 -0
- package/dist/server/handlers/system.d.ts.map +1 -1
- package/dist/server/handlers/system.js +2 -2
- package/dist/server/handlers/tools.cjs +1 -1
- package/dist/server/handlers/tools.js +1 -1
- package/dist/server/handlers/workspace.cjs +1 -1
- package/dist/server/handlers/workspace.d.ts +4 -0
- package/dist/server/handlers/workspace.d.ts.map +1 -1
- package/dist/server/handlers/workspace.js +1 -1
- package/dist/server/handlers.cjs +11 -11
- package/dist/server/handlers.js +11 -11
- package/dist/server/schemas/index.cjs +3 -3
- package/dist/server/schemas/index.js +3 -3
- package/dist/server/schemas/mcp.d.ts +141 -0
- package/dist/server/schemas/mcp.d.ts.map +1 -1
- package/dist/server/schemas/system.d.ts +8 -0
- package/dist/server/schemas/system.d.ts.map +1 -1
- package/dist/server/schemas/workspace.d.ts +2 -0
- package/dist/server/schemas/workspace.d.ts.map +1 -1
- package/dist/server/server-adapter/index.cjs +1 -1
- package/dist/server/server-adapter/index.js +1 -1
- package/dist/server/server-adapter/routes/mcp.d.ts +256 -0
- package/dist/server/server-adapter/routes/mcp.d.ts.map +1 -1
- package/dist/server/server-adapter/routes/observability.d.ts +42 -10
- package/dist/server/server-adapter/routes/observability.d.ts.map +1 -1
- package/dist/server/server-adapter/routes/system.d.ts +8 -0
- package/dist/server/server-adapter/routes/system.d.ts.map +1 -1
- package/dist/server/server-adapter/routes/workspace.d.ts +4 -0
- package/dist/server/server-adapter/routes/workspace.d.ts.map +1 -1
- package/dist/{system-BxHpBJiq.cjs → system-CHCn8IQ3.cjs} +5 -1
- package/dist/system-CHCn8IQ3.cjs.map +1 -0
- package/dist/{system-CxwzIMsy.js → system-DOwo7OGf.js} +5 -1
- package/dist/system-DOwo7OGf.js.map +1 -0
- package/dist/{tools-Ck35EVD7.js → tools-BFpuzQsz.js} +2 -2
- package/dist/{tools-Ck35EVD7.js.map → tools-BFpuzQsz.js.map} +1 -1
- package/dist/{tools-CeJrnYDf.cjs → tools-Dy_j6URC.cjs} +2 -2
- package/dist/{tools-CeJrnYDf.cjs.map → tools-Dy_j6URC.cjs.map} +1 -1
- package/dist/{workspace-CyH_iu8w.cjs → workspace-BU6sQpbV.cjs} +14 -10
- package/dist/{workspace-C9dE6D1U.js.map → workspace-BU6sQpbV.cjs.map} +1 -1
- package/dist/{workspace-KeHGZvNV.cjs → workspace-C5BBUatQ.cjs} +3 -1
- package/dist/workspace-C5BBUatQ.cjs.map +1 -0
- package/dist/{workspace-C9dE6D1U.js → workspace-CS-GYTve.js} +14 -10
- package/dist/workspace-CS-GYTve.js.map +1 -0
- package/dist/{workspace-muaSMxVS.js → workspace-DBDA3UQl.js} +3 -1
- package/dist/workspace-DBDA3UQl.js.map +1 -0
- package/package.json +4 -4
- package/dist/agents-BVjnYswL.js.map +0 -1
- package/dist/agents-D36tukip.cjs.map +0 -1
- package/dist/dist-_M1jEPcL.js +0 -1
- package/dist/mcp-Bt3h1lO7.cjs.map +0 -1
- package/dist/mcp-D3F3iYew.js.map +0 -1
- package/dist/mcp-D4Cprr8R.js.map +0 -1
- package/dist/mcp-mpYrlkUA.cjs.map +0 -1
- package/dist/observability-new-endpoints-CaTskbSh.cjs.map +0 -1
- package/dist/observability-new-endpoints-CsUP8y2j.js.map +0 -1
- package/dist/routes-By3cLbPV.cjs.map +0 -1
- package/dist/system-BxHpBJiq.cjs.map +0 -1
- package/dist/system-CxwzIMsy.js.map +0 -1
- package/dist/workspace-CyH_iu8w.cjs.map +0 -1
- package/dist/workspace-KeHGZvNV.cjs.map +0 -1
- package/dist/workspace-muaSMxVS.js.map +0 -1
package/dist/docs/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: mastra-server
|
|
|
3
3
|
description: Documentation for @mastra/server. Use when working with @mastra/server APIs, configuration, or implementation.
|
|
4
4
|
metadata:
|
|
5
5
|
package: "@mastra/server"
|
|
6
|
-
version: "1.81.0-alpha.
|
|
6
|
+
version: "1.81.0-alpha.2"
|
|
7
7
|
---
|
|
8
8
|
|
|
9
9
|
## When to use
|
|
@@ -24,7 +24,7 @@ Read the individual reference documents for detailed explanations and code examp
|
|
|
24
24
|
|
|
25
25
|
### Reference
|
|
26
26
|
|
|
27
|
-
- [Reference:
|
|
27
|
+
- [Reference: Query traces and spans](references/reference-observability-tracing-advanced-queries.md) - Find, count, and measure completed traces and spans with structured predicates. List matching traces or spans, aggregate them into grouped measures, paginate, poll, and discover query fields.
|
|
28
28
|
- [Reference: createRoute()](references/reference-server-create-route.md) - Use createRoute() to define type-safe Mastra server routes with Zod validation for requests and responses, plus handler context and OpenAPI metadata.
|
|
29
29
|
- [Reference: MastraServer](references/reference-server-mastra-server.md) - The MastraServer abstract class is the base for all server adapters. Extend this class to create adapters for frameworks other than Hono or Express.
|
|
30
30
|
- [Reference: Server routes](references/reference-server-routes.md) - Server adapters register these routes when you call server.init(). All routes are prefixed with the prefix option if configured.
|
|
@@ -2,43 +2,57 @@
|
|
|
2
2
|
|
|
3
3
|
> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
|
|
4
4
|
|
|
5
|
-
#
|
|
5
|
+
# Query traces and spans
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Query completed traces or spans with structured filters, or aggregate them into counts, error rates, latency percentiles, tokens, and cost. Each operation is available through the client SDK, HTTP API, and CLI:
|
|
8
8
|
|
|
9
|
-
|
|
|
10
|
-
|
|
|
11
|
-
| Query traces
|
|
12
|
-
|
|
|
13
|
-
| Aggregate
|
|
9
|
+
| Operation | Use it to | Client SDK | HTTP endpoint | CLI command |
|
|
10
|
+
| ------------------------------------- | -------------------------------------------------------------------------------- | ------------------- | ------------------------------------------ | ---------------------------- |
|
|
11
|
+
| [Query traces](#query-traces) | List traces matching root-span fields or conditions on related spans. | `queryTraces()` | `POST /api/observability/traces/query` | `mastra api trace query` |
|
|
12
|
+
| [Query spans](#query-spans) | List individual spans across traces, including nested agent runs and tool calls. | `querySpans()` | `POST /api/observability/spans/query` | `mastra api span query` |
|
|
13
|
+
| [Aggregate traces](#aggregate-traces) | Compute measures over matching traces, optionally grouped or bucketed by time. | `aggregateTraces()` | `POST /api/observability/traces/aggregate` | `mastra api trace aggregate` |
|
|
14
|
+
| [Aggregate spans](#aggregate-spans) | Compute measures over individual matching spans at any depth. | `aggregateSpans()` | `POST /api/observability/spans/aggregate` | `mastra api span aggregate` |
|
|
14
15
|
|
|
15
|
-
The trace
|
|
16
|
+
The trace operations share a [trace selection](#select-traces): root-span predicates and `spans.some` / `spans.none` clauses. The span operations share a direct span predicate, without nested `spans` clauses. Reuse a selection between the query and aggregate operations for the same record type.
|
|
17
|
+
|
|
18
|
+
| Option | Query traces | Query spans | Aggregate traces or spans |
|
|
19
|
+
| --------------------------- | ---------------------------------------- | -------------- | ------------------------------------- |
|
|
20
|
+
| Maximum time-range duration | No maximum | No maximum | 365 days |
|
|
21
|
+
| Pagination | Keyset, numbered pages, or delta polling | Keyset only | No pagination; `limit` selects groups |
|
|
22
|
+
| `orderBy` shape | One-item array | One-item array | One object |
|
|
23
|
+
|
|
24
|
+
CLI commands take filters and request options together as one JSON argument and wrap the response in `data`. To inspect a query result, use its `traceId` to [retrieve the full trace](https://mastra.ai/reference/client-js/observability), or call `getSpan(traceId, spanId)` to retrieve one span.
|
|
16
25
|
|
|
17
26
|
The endpoints use the same authentication as other observability routes and require the `observability:read` permission. Every request requires a time range.
|
|
18
27
|
|
|
19
|
-
For hosted traces,
|
|
28
|
+
These API time-range limits are separate from data retention. For hosted traces, your Platform plan's retention period applies. See [Query observability data on Mastra Platform](https://mastra.ai/docs/mastra-platform/observability) for retention, project credentials, and regional endpoints.
|
|
20
29
|
|
|
21
30
|
## Check capabilities
|
|
22
31
|
|
|
23
32
|
The configured observability store must support each API. Call `getObservabilityCapabilities()` or `GET /api/observability/capabilities` and check these flags before showing a feature:
|
|
24
33
|
|
|
25
|
-
| Capability | Store feature | Enables
|
|
26
|
-
| -------------------------------- | -------------------------------------- |
|
|
27
|
-
| `traceQuery` | `trace-query` | `queryTraces()`
|
|
28
|
-
| `
|
|
29
|
-
| `
|
|
30
|
-
| `
|
|
31
|
-
| `
|
|
32
|
-
| `
|
|
33
|
-
| `
|
|
34
|
-
| `
|
|
35
|
-
| `
|
|
36
|
-
| `
|
|
37
|
-
| `
|
|
38
|
-
| `
|
|
39
|
-
| `
|
|
40
|
-
|
|
41
|
-
|
|
34
|
+
| Capability | Store feature | Enables |
|
|
35
|
+
| -------------------------------- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
36
|
+
| `traceQuery` | `trace-query` | `queryTraces()` |
|
|
37
|
+
| `spanQuery` | `span-query` | `querySpans()` |
|
|
38
|
+
| `spanQueryContextFields` | `span-query-context-fields` | Additional identity, context, and parent/root entity predicates in span queries and aggregates |
|
|
39
|
+
| `spanQueryModelFields` | `span-query-model-fields` | `model` and `provider` predicates in span queries and aggregates |
|
|
40
|
+
| `traceAggregate` | `trace-aggregate` | `aggregateTraces()` |
|
|
41
|
+
| `spanAggregate` | `span-aggregate` | `aggregateSpans()` |
|
|
42
|
+
| `traceQueryRootDuration` | `trace-query-root-duration` | Trace-scope `durationMs` predicates |
|
|
43
|
+
| `traceQueryTimeToFirstChunk` | `trace-query-time-to-first-chunk` | `timeToFirstChunkMs` predicates and results |
|
|
44
|
+
| `traceQueryTimeToFirstChunkSort` | `trace-query-time-to-first-chunk-sort` | `orderBy` on `timeToFirstChunkMs` |
|
|
45
|
+
| `traceQueryTimeToFirstToken` | `trace-query-time-to-first-token` | `timeToFirstTokenMs` predicates and results |
|
|
46
|
+
| `traceQueryTimeToFirstTokenSort` | `trace-query-time-to-first-token-sort` | `orderBy` on `timeToFirstTokenMs` |
|
|
47
|
+
| `traceQueryContextIds` | `trace-query-context-ids` | `runId`, `sessionId`, `userId`, and `organizationId` predicates |
|
|
48
|
+
| `traceQueryRootSpanFields` | `trace-query-root-span-fields` | Trace-scope `name` and `spanType` predicates, dimensions, and `countDistinct` measures |
|
|
49
|
+
| `traceQueryRootEntityId` | `trace-query-root-entity-id` | Trace-scope `entityId` predicates, dimensions, and `countDistinct` measures |
|
|
50
|
+
| `traceQueryOutcomeFields` | `trace-query-outcome-fields` | `outcome` and `stopReason` in trace and `spans.some` / `spans.none` predicates, trace dimensions, and value discovery, plus the root span's outcome fields on trace results |
|
|
51
|
+
| `spanQueryOutcomeFields` | `span-query-outcome-fields` | `outcome` and `stopReason` in `querySpans()` and `aggregateSpans()` predicates, dimensions, and `countDistinct` measures, plus the outcome fields on span results |
|
|
52
|
+
| `traceQueryDiscovery` | `trace-query-discovery` | [Field and value discovery](#discover-fields-and-values) |
|
|
53
|
+
| `traceQueryTenantScope` | `trace-query-tenant-scope` | [Trusted tenant scope](#trusted-tenant-scope) |
|
|
54
|
+
|
|
55
|
+
A missing `traceAggregate` value means the server doesn't support aggregation. Servers without the aggregate route return `404`, and stores without the feature return `501 TRACE_AGGREGATE_UNSUPPORTED`. A missing `spanAggregate` value means the same for span aggregation, and stores without the feature return `501 SPAN_AGGREGATE_UNSUPPORTED`. Against a store without a sub-feature, a request that uses a gated field returns `501` before storage runs. Unsupported root span `name`, `spanType`, and `entityId`, and span predicates requiring `spanQueryContextFields`, return `422` with a `field_not_allowed` issue instead. A store without `spanQuery` returns `501 SPAN_QUERY_UNSUPPORTED`.
|
|
42
56
|
|
|
43
57
|
## Select traces
|
|
44
58
|
|
|
@@ -79,16 +93,16 @@ Span clauses examine the current root span and current child spans. The root `ti
|
|
|
79
93
|
|
|
80
94
|
### Fields and operators
|
|
81
95
|
|
|
82
|
-
| Predicate context | Fields
|
|
83
|
-
| ----------------- |
|
|
84
|
-
| Trace | `traceId`, `threadId`, `resourceId`, `runId`, `sessionId`, `userId`, `organizationId`, `entityId`, `entityName`, `entityType`, `name`, `spanType`, `environment`, `status` | `eq`, `ne`, `in`, `notIn`, `exists`, `notExists` |
|
|
85
|
-
| Trace metadata | `metadata.<key>`
|
|
86
|
-
| Trace | `startedAt`, `endedAt`, `durationMs`, `timeToFirstChunkMs`, `timeToFirstTokenMs`
|
|
87
|
-
| Trace | `tags`
|
|
88
|
-
| Span | `name`
|
|
89
|
-
| Span | `spanType`, `model`, `provider`, `status`, `entityType`, `entityId`, `entityName`, `entityVersionId`, `parentEntityVersionId`, `rootEntityVersionId`, `runId`, `sessionId`, `userId`, `organizationId` | `eq`, `ne`, `in`, `notIn`, `exists`, `notExists` |
|
|
90
|
-
| Span | `startedAt`, `endedAt`, `durationMs`, `timeToFirstChunkMs`, `timeToFirstTokenMs`
|
|
91
|
-
| Span | `error`
|
|
96
|
+
| Predicate context | Fields | Operators |
|
|
97
|
+
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- |
|
|
98
|
+
| Trace | `traceId`, `threadId`, `resourceId`, `runId`, `sessionId`, `userId`, `organizationId`, `entityId`, `entityName`, `entityType`, `name`, `spanType`, `environment`, `status`, `outcome`, `stopReason` | `eq`, `ne`, `in`, `notIn`, `exists`, `notExists` |
|
|
99
|
+
| Trace metadata | `metadata.<key>` | `eq`, `ne`, `in`, `notIn`, `exists`, `notExists` |
|
|
100
|
+
| Trace | `startedAt`, `endedAt`, `durationMs`, `timeToFirstChunkMs`, `timeToFirstTokenMs` | `eq`, `ne`, `in`, `notIn`, `lt`, `lte`, `gt`, `gte`, `exists`, `notExists` |
|
|
101
|
+
| Trace | `tags` | `includes`, `notIncludes`, `exists`, `notExists` |
|
|
102
|
+
| Span | `name` | `eq`, `ne`, `in`, `notIn`, `exists`, `notExists`, `matches`, `notMatches` |
|
|
103
|
+
| Span | `spanType`, `model`, `provider`, `status`, `outcome`, `stopReason`, `entityType`, `entityId`, `entityName`, `entityVersionId`, `parentEntityVersionId`, `rootEntityVersionId`, `runId`, `sessionId`, `userId`, `organizationId` | `eq`, `ne`, `in`, `notIn`, `exists`, `notExists` |
|
|
104
|
+
| Span | `startedAt`, `endedAt`, `durationMs`, `timeToFirstChunkMs`, `timeToFirstTokenMs` | `eq`, `ne`, `in`, `notIn`, `lt`, `lte`, `gt`, `gte`, `exists`, `notExists` |
|
|
105
|
+
| Span | `error` | `exists`, `notExists` |
|
|
92
106
|
|
|
93
107
|
Trace-scope `name`, `spanType`, and `entityId` read the current root span, so `{ op: 'eq', left: { path: 'spanType' }, right: { literal: 'workflow_run' } }` selects traces started by a workflow run. Inside `spans.some` or `spans.none`, the same fields match any span in the trace.
|
|
94
108
|
|
|
@@ -156,6 +170,26 @@ const slowFirstToken = { op: 'gt', left: { path: 'timeToFirstTokenMs' }, right:
|
|
|
156
170
|
|
|
157
171
|
Stores advertise support through the `trace-query-time-to-first-token` feature and the sort through the `trace-query-time-to-first-token-sort` feature. The server reports them as the `traceQueryTimeToFirstToken` and `traceQueryTimeToFirstTokenSort` capabilities. Against an older store, results omit the field, and a predicate or sort on it returns `501` before storage runs.
|
|
158
172
|
|
|
173
|
+
#### Filter by outcome
|
|
174
|
+
|
|
175
|
+
`outcome` records how a span ended, for example `success`, `error`, `abort`, `limit`, or `unknown`. `stopReason` records why, for example `max_steps`. Both are free strings, so custom spans can record their own values. They're separate from `status`, which still reads only whether the span recorded an error.
|
|
176
|
+
|
|
177
|
+
At trace scope, both fields read the current root span, the span with no parent that started the trace. Inside `spans.some` or `spans.none`, they read each related span. A trace whose root succeeded but whose tool call hit a step limit doesn't match a trace-scope `outcome` of `limit`. Find it through `spans.some`:
|
|
178
|
+
|
|
179
|
+
```typescript
|
|
180
|
+
const hitALimit = {
|
|
181
|
+
spans: {
|
|
182
|
+
some: { op: 'eq', left: { path: 'outcome' }, right: { literal: 'limit' } },
|
|
183
|
+
},
|
|
184
|
+
}
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Spans written by an SDK that predates these fields, or a root span that's still running, have no outcome. A missing outcome means "not recorded", never success. Select those spans with `{ op: 'notExists', path: 'outcome' }`. As with other optional string fields, `ne` and `notIn` match spans with no value.
|
|
188
|
+
|
|
189
|
+
Both fields support `eq`, `ne`, `in`, `notIn`, `exists`, and `notExists`, group trace and span aggregates, and return values from value discovery. They can't sort individual trace or span query results. In aggregates, either field can be used in `orderBy` when it's included in `groupBy`. Results also return `stopReasonContext`, an object with details about the stop reason such as `{ maxSteps: 5 }`. `stopReasonContext` is returned only and can't be filtered, grouped, or sorted.
|
|
190
|
+
|
|
191
|
+
Stores advertise support through the `trace-query-outcome-fields` feature for trace queries and aggregates, and the `span-query-outcome-fields` feature for span queries and aggregates. The server reports them as the `traceQueryOutcomeFields` and `spanQueryOutcomeFields` capabilities. Against an older store, results omit the fields, and a request that uses them returns `501` before storage runs.
|
|
192
|
+
|
|
159
193
|
#### Filter by context identifiers
|
|
160
194
|
|
|
161
195
|
`runId`, `sessionId`, `userId`, and `organizationId` are available at trace scope and inside `spans.some` / `spans.none`. Trace-scope predicates read the current root span. Span-scope predicates bind to one related current span, so a run recorded only on a child span is found through `spans.some`:
|
|
@@ -219,7 +253,7 @@ const slowModelCalls = {
|
|
|
219
253
|
}
|
|
220
254
|
```
|
|
221
255
|
|
|
222
|
-
`model` and `provider` read the
|
|
256
|
+
`model` and `provider` read the span's `model` and `provider` columns. When a column is empty, they fall back to the string `attributes.model` and `attributes.provider` values that older `@mastra/observability` versions exported, so existing filters keep matching older spans. Trace results omit `model` and `provider`, because a trace can match through one span while its other spans used a different model.
|
|
223
257
|
|
|
224
258
|
```typescript
|
|
225
259
|
const selectedModel = {
|
|
@@ -486,6 +520,14 @@ A query using keyset pagination returns lightweight completed traces and a curso
|
|
|
486
520
|
{
|
|
487
521
|
"traceId": "trace-123",
|
|
488
522
|
"rootSpanId": "span-123",
|
|
523
|
+
"name": "agent run: support-agent",
|
|
524
|
+
"entityId": "support-agent",
|
|
525
|
+
"parentSpanId": null,
|
|
526
|
+
"createdAt": "2026-08-03T10:00:00.000Z",
|
|
527
|
+
"metadata": null,
|
|
528
|
+
"inputPreview": "Where is my order?",
|
|
529
|
+
"outputPreview": "Your order arrives tomorrow.",
|
|
530
|
+
"errorPreview": null,
|
|
489
531
|
"threadId": "thread-123",
|
|
490
532
|
"resourceId": "resource-123",
|
|
491
533
|
"startedAt": "2026-08-03T10:00:00.000Z",
|
|
@@ -495,14 +537,19 @@ A query using keyset pagination returns lightweight completed traces and a curso
|
|
|
495
537
|
"environment": "production",
|
|
496
538
|
"status": "success",
|
|
497
539
|
"timeToFirstChunkMs": 412,
|
|
498
|
-
"timeToFirstTokenMs": 380
|
|
540
|
+
"timeToFirstTokenMs": 380,
|
|
541
|
+
"outcome": "success",
|
|
542
|
+
"stopReason": null,
|
|
543
|
+
"stopReasonContext": null
|
|
499
544
|
}
|
|
500
545
|
],
|
|
501
546
|
"page": { "next": null }
|
|
502
547
|
}
|
|
503
548
|
```
|
|
504
549
|
|
|
505
|
-
`
|
|
550
|
+
The response includes `name`, `createdAt`, identity fields, `metadata`, and input/output/error previews. Supported nullable fields remain present with `null` when no value was recorded. Fields that require an optional store feature are omitted when the store lacks that feature. Previews summarize payloads; retrieve the full trace for complete input, output, and error details.
|
|
551
|
+
|
|
552
|
+
`timeToFirstChunkMs` and `timeToFirstTokenMs` are present only when the configured store supports them. See [Filter by time to first chunk](#filter-by-time-to-first-chunk) and [Filter and sort by time to first token](#filter-and-sort-by-time-to-first-token). `outcome`, `stopReason`, and `stopReasonContext` read the root span and are present only when the store supports the `traceQueryOutcomeFields` capability. They're `null` when the root span didn't record them. See [Filter by outcome](#filter-by-outcome).
|
|
506
553
|
|
|
507
554
|
The response doesn't include the related spans that matched the predicate. Use [`getTrace()`](https://mastra.ai/reference/client-js/observability) to load the full trace after selecting a result.
|
|
508
555
|
|
|
@@ -599,7 +646,7 @@ Numbered-page handoff cursors and delta cursors also bind the authorization stat
|
|
|
599
646
|
|
|
600
647
|
Cursor pagination is deterministic, but it isn't a database snapshot. Traces or replacement signals written between page requests can change later pages.
|
|
601
648
|
|
|
602
|
-
PostgreSQL and ClickHouse stop trace queries after 15 seconds by default and return `504` when the database timeout is exceeded. Set `traceQueryTimeoutMs`
|
|
649
|
+
PostgreSQL and ClickHouse stop trace queries after 15 seconds by default and return `504` when the database timeout is exceeded. Set `observability.traceQueryTimeoutMs` on `PostgresStoreVNext`, or `observability.traceQuery.timeoutMs` on `ClickhouseStoreVNext`, to an integer from 1 through 300,000 milliseconds to change the timeout. DuckDB doesn't currently provide query-scoped timeout or cancellation through its driver wrapper, so this `504` guarantee doesn't apply to DuckDB.
|
|
603
650
|
|
|
604
651
|
| Status | Meaning |
|
|
605
652
|
| ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
@@ -611,6 +658,73 @@ PostgreSQL and ClickHouse stop trace queries after 15 seconds by default and ret
|
|
|
611
658
|
| `503` | Discovery exceeded a backend memory or resource budget and returned `TRACE_QUERY_RESOURCE_LIMIT`. |
|
|
612
659
|
| `504` | A PostgreSQL or ClickHouse query exceeded its configured database execution timeout. Discovery returns `TRACE_QUERY_EXECUTION_TIMEOUT`. |
|
|
613
660
|
|
|
661
|
+
## Query spans
|
|
662
|
+
|
|
663
|
+
Use `querySpans()`, `mastra api span query`, or `POST /api/observability/spans/query` to return completed span summaries across traces. It requires a positive ISO timestamp range over span start time and supports cursor pagination. This example finds tool calls that took at least one second:
|
|
664
|
+
|
|
665
|
+
```typescript
|
|
666
|
+
const result = await mastraClient.querySpans({
|
|
667
|
+
timeRange: {
|
|
668
|
+
from: '2026-10-01T00:00:00.000Z',
|
|
669
|
+
to: '2026-10-02T00:00:00.000Z',
|
|
670
|
+
},
|
|
671
|
+
where: {
|
|
672
|
+
op: 'and',
|
|
673
|
+
args: [
|
|
674
|
+
{ op: 'eq', left: { path: 'spanType' }, right: { literal: 'tool_call' } },
|
|
675
|
+
{ op: 'gte', left: { path: 'durationMs' }, right: { literal: 1000 } },
|
|
676
|
+
],
|
|
677
|
+
},
|
|
678
|
+
orderBy: [{ field: 'startedAt', direction: 'desc' }],
|
|
679
|
+
page: { limit: 100 },
|
|
680
|
+
})
|
|
681
|
+
|
|
682
|
+
// Pass result.page.next as page.after to continue with the same filters and ordering.
|
|
683
|
+
console.log(result.spans)
|
|
684
|
+
```
|
|
685
|
+
|
|
686
|
+
Pages default to 100 rows and allow at most 1,000. The response is `{ spans, page: { next } }`, without a total count. Ordering accepts exactly one of `startedAt`, `endedAt`, `timeToFirstChunkMs`, or `timeToFirstTokenMs`, ascending or descending, and defaults to descending start time. Spans without a value for the `timeToFirstChunkMs` or `timeToFirstTokenMs` sort field sort last in both directions.
|
|
687
|
+
|
|
688
|
+
| Fields | Operators |
|
|
689
|
+
| -------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
|
|
690
|
+
| `traceId`, `spanId`, `parentSpanId`, `threadId` | `eq`, `ne`, `in`, `notIn`, `exists`, `notExists` |
|
|
691
|
+
| `organizationId`, `resourceId`, `runId`, `sessionId`, `userId`, `requestId`, `serviceName`, `experimentId`, `environment`, `executionSource` | Same string operators |
|
|
692
|
+
| Entity, parent entity and root entity `Type`, `Id`, `Name`, `VersionId` fields, such as `parentEntityId` | Same string operators |
|
|
693
|
+
| `name`, `spanType`, `status` | Same string operators; `name` also accepts `matches` / `notMatches` for normalized whole words and phrases |
|
|
694
|
+
| `model`, `provider`, `outcome`, `stopReason` | Same string operators |
|
|
695
|
+
| `startedAt`, `endedAt`, `durationMs`, `timeToFirstChunkMs`, `timeToFirstTokenMs` | Comparisons, membership and presence |
|
|
696
|
+
| `error` | `exists`, `notExists` |
|
|
697
|
+
|
|
698
|
+
Combine predicates with `and`, `or` and `not`. Membership uses `{ op: 'in', value: { path: 'spanType' }, set: ['agent_run', 'tool_call'] }`. A span's status is `error` when it has an error and `success` otherwise.
|
|
699
|
+
|
|
700
|
+
Each summary includes identity, context, entity lineage, status, timestamps, `durationMs`, model/provider and input/output/error previews.
|
|
701
|
+
|
|
702
|
+
Check `capabilities.spanQueryContextFields` from `getObservabilityCapabilities()` before filtering on `traceId`, `spanId`, `parentSpanId`, `threadId`, `resourceId`, `requestId`, `serviceName`, `experimentId`, `environment`, `executionSource`, or the parent/root entity `Type`, `Id`, and `Name` fields. If this capability is missing or false, requests using these filters return a structured `422` error. It doesn't gate `runId`, `sessionId`, `userId`, `organizationId`, or entity `VersionId` filters.
|
|
703
|
+
|
|
704
|
+
Check `capabilities.spanQueryModelFields` before filtering on `model` or `provider`. These filters read only the span's `model` and `provider` columns. `@mastra/observability` fills the columns only when a span has both a `model` and a `provider`, and versions before it filled them left them empty. Filters treat empty columns as missing, so `eq`, `in` and `exists` don't match those spans while `ne`, `notIn` and `notExists` do. Results still return `model` and `provider` for them from the span's attributes. Trace query `spans.some` filters fall back to the attributes and do match those spans. With an older adapter, requests using these filters return `501`.
|
|
705
|
+
|
|
706
|
+
Check `capabilities.spanQueryOutcomeFields` before filtering or grouping spans by `outcome` and `stopReason`. Rows then also include `outcome`, `stopReason` and `stopReasonContext`. These record how the span ended, separately from `status`, and are `null` when the span didn't record them. A missing outcome means "not recorded", not success. `stopReasonContext` can't be used in filters and is omitted when the store lacks `span-query-outcome-fields`. For traces, `capabilities.traceQueryOutcomeFields` enables the same fields on `queryTraces()` and `aggregateTraces()`, where they read the root span. With an older adapter, requests using these fields return `501`:
|
|
707
|
+
|
|
708
|
+
```typescript
|
|
709
|
+
const { capabilities } = await mastraClient.getObservabilityCapabilities()
|
|
710
|
+
|
|
711
|
+
if (capabilities.spanQueryOutcomeFields) {
|
|
712
|
+
const limited = await mastraClient.querySpans({
|
|
713
|
+
timeRange: { from: '2026-10-01T00:00:00.000Z', to: '2026-10-02T00:00:00.000Z' },
|
|
714
|
+
where: { op: 'eq', left: { path: 'outcome' }, right: { literal: 'limit' } },
|
|
715
|
+
})
|
|
716
|
+
console.log(limited.spans.map(span => [span.stopReason, span.stopReasonContext]))
|
|
717
|
+
}
|
|
718
|
+
```
|
|
719
|
+
|
|
720
|
+
See [Filter by outcome](https://mastra.ai/reference/observability/tracing/advanced-queries) for trace-scope and `spans.some` semantics.
|
|
721
|
+
|
|
722
|
+
DuckDB returns `null` for parent/root entity type, ID and name on spans written before `@mastra/duckdb` stored them. Equality filters on these fields don't match those spans.
|
|
723
|
+
|
|
724
|
+
The CLI returns the same response under `data`: read `data.spans` and pass `data.page.next` as `page.after`. Span queries support only keyset pagination, so reuse the same time range, predicate, and ordering when following the cursor. A malformed cursor returns `400`, while a cursor that doesn't match the query or scope returns `409`.
|
|
725
|
+
|
|
726
|
+
Use `mastra api trace span <traceId> <spanId>` or `getSpan(traceId, spanId)` to retrieve a selected span's full details.
|
|
727
|
+
|
|
614
728
|
## Aggregate traces
|
|
615
729
|
|
|
616
730
|
Use `aggregateTraces()` or `POST /api/observability/traces/aggregate` to count the selected traces and compute measures over them. Add `groupBy` to split the result by up to two dimensions, and `interval` to split each group into time buckets. The response contains one row per group, or one row per group and bucket.
|
|
@@ -667,16 +781,16 @@ curl --request POST \
|
|
|
667
781
|
|
|
668
782
|
### Aggregate request fields
|
|
669
783
|
|
|
670
|
-
| Field | Required | Description
|
|
671
|
-
| ----------- | -------- |
|
|
672
|
-
| `timeRange` | Yes | Trace start-time boundary, as in [Select traces](#select-traces). The range can span up to 365 days.
|
|
673
|
-
| `where` | No | Trace predicate, as in [Select traces](#select-traces).
|
|
674
|
-
| `measures` | Yes | One or more distinct [measure names](#measures).
|
|
675
|
-
| `groupBy` | No | Up to two distinct [dimensions](#dimensions). Defaults to `[]`, which
|
|
676
|
-
| `interval` | No | Bucket width: `'1m'`, `'5m'`, `'15m'`, `'1h'`, or `'1d'`. See [Time buckets](#time-buckets).
|
|
677
|
-
| `having` | No | Scalar predicate over measures that removes groups after grouping. See [Filter and order groups](#filter-and-order-groups).
|
|
678
|
-
| `orderBy` | No | `{ field, direction }`. `field` is a requested measure, `count`, or a `groupBy` dimension, and `direction` is `asc` or `desc`. Defaults to `count desc`.
|
|
679
|
-
| `limit` | No | Maximum number of groups. Defaults to 100 and has a maximum of 1,000.
|
|
784
|
+
| Field | Required | Description |
|
|
785
|
+
| ----------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
786
|
+
| `timeRange` | Yes | Trace start-time boundary, as in [Select traces](#select-traces). The range can span up to 365 days. |
|
|
787
|
+
| `where` | No | Trace predicate, as in [Select traces](#select-traces). |
|
|
788
|
+
| `measures` | Yes | One or more distinct [measure names](#measures). |
|
|
789
|
+
| `groupBy` | No | Up to two distinct [dimensions](#dimensions). Defaults to `[]`, which treats all selected traces as one group. With `interval`, that group has one row per non-empty bucket. No matching traces returns no rows. |
|
|
790
|
+
| `interval` | No | Bucket width: `'1m'`, `'5m'`, `'15m'`, `'1h'`, or `'1d'`. See [Time buckets](#time-buckets). |
|
|
791
|
+
| `having` | No | Scalar predicate over measures that removes groups after grouping. See [Filter and order groups](#filter-and-order-groups). |
|
|
792
|
+
| `orderBy` | No | `{ field, direction }`. `field` is a requested measure, `count`, or a `groupBy` dimension, and `direction` is `asc` or `desc`. Defaults to `count desc`. |
|
|
793
|
+
| `limit` | No | Maximum number of groups. Defaults to 100 and has a maximum of 1,000. |
|
|
680
794
|
|
|
681
795
|
Unknown fields are rejected. Aggregation isn't paginated. Narrow the selection, or rank by a different measure, when `truncated` is `true`.
|
|
682
796
|
|
|
@@ -684,15 +798,16 @@ Unknown fields are rejected. Aggregation isn't paginated. Narrow the selection,
|
|
|
684
798
|
|
|
685
799
|
Dimensions read the current root span of each selected trace:
|
|
686
800
|
|
|
687
|
-
| Dimension | Notes
|
|
688
|
-
| ------------------------------------------------------------------------------------- |
|
|
689
|
-
| `entityType`, `entityName`, `environment`, `status`, `serviceName`, `executionSource` | Root span attributes.
|
|
690
|
-
| `threadId`, `resourceId`, `userId`, `sessionId`, `organizationId`, `experimentId` | Context identifiers recorded on the root span.
|
|
691
|
-
| `name`, `spanType` | Root span name and type. Requires the `traceQueryRootSpanFields` capability.
|
|
692
|
-
| `entityId` | Root span entity ID. Groups traces that have no `entityName`, such as scorer runs. Requires the `traceQueryRootEntityId` capability.
|
|
693
|
-
| `
|
|
801
|
+
| Dimension | Notes |
|
|
802
|
+
| ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
803
|
+
| `entityType`, `entityName`, `environment`, `status`, `serviceName`, `executionSource` | Root span attributes. |
|
|
804
|
+
| `threadId`, `resourceId`, `userId`, `sessionId`, `organizationId`, `experimentId` | Context identifiers recorded on the root span. |
|
|
805
|
+
| `name`, `spanType` | Root span name and type. Requires the `traceQueryRootSpanFields` capability. |
|
|
806
|
+
| `entityId` | Root span entity ID. Groups traces that have no `entityName`, such as scorer runs. Requires the `traceQueryRootEntityId` capability. |
|
|
807
|
+
| `outcome`, `stopReason` | How the root span ended and why. Traces whose root span didn't record a value group as `null`. Requires the `traceQueryOutcomeFields` capability. |
|
|
808
|
+
| `metadata.<key>` | One non-empty top-level metadata key. Values are trimmed. Missing, empty, and non-string values group as `null`. |
|
|
694
809
|
|
|
695
|
-
Every dimension except `serviceName`, `executionSource`, and `experimentId` is also a trace-scope `where` field, so you can drill into a group by adding its values to the predicate. `traceId`, `spanId`, `runId`, and `requestId` aren't groupable. Traces with no value for a dimension form their own group with a `null` value.
|
|
810
|
+
Every dimension except `serviceName`, `executionSource`, and `experimentId` is also a trace-scope `where` field, so you can drill into a group by adding its values to the predicate. `traceId`, `spanId`, `runId`, and `requestId` aren't groupable. Traces with no value for a dimension form their own group with a `null` value. To select that group in a query, use `notExists` for the dimension instead of `eq` with `literal: null`.
|
|
696
811
|
|
|
697
812
|
### Measures
|
|
698
813
|
|
|
@@ -812,21 +927,21 @@ const dailyCost = await mastraClient.aggregateTraces({
|
|
|
812
927
|
|
|
813
928
|
Predicate limits from [Query limits](#query-limits) apply to both `where` and `having`, and the request body is limited to 256 KiB. A time range that doesn't start on a bucket boundary touches one more bucket than its length divided by the interval. If the range has too many buckets, choose a wider `interval`. If `limit` × buckets exceeds 10,000, reduce `limit` or widen the interval.
|
|
814
929
|
|
|
815
|
-
PostgreSQL and ClickHouse
|
|
930
|
+
PostgreSQL and ClickHouse use the same [execution timeout](#keyset-ordering-and-errors) for aggregations as for trace queries, defaulting to 15 seconds. The timeout doesn't apply to DuckDB.
|
|
816
931
|
|
|
817
|
-
| Status | Meaning
|
|
818
|
-
| ------ |
|
|
819
|
-
| `400` | Malformed JSON.
|
|
820
|
-
| `413` | The request body exceeds 256 KiB.
|
|
821
|
-
| `422` | The request is invalid and returns `TRACE_QUERY_INVALID` with issues.
|
|
822
|
-
| `501` | The installed Core or configured observability store doesn't support aggregation or a requested field. Returns `TRACE_AGGREGATE_UNSUPPORTED`.
|
|
823
|
-
| `504` | A PostgreSQL or ClickHouse aggregation exceeded its execution timeout. Returns `TRACE_QUERY_EXECUTION_TIMEOUT` without partial results.
|
|
932
|
+
| Status | Meaning |
|
|
933
|
+
| ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
934
|
+
| `400` | Malformed JSON. |
|
|
935
|
+
| `413` | The request body exceeds 256 KiB. |
|
|
936
|
+
| `422` | The request is invalid and returns `TRACE_QUERY_INVALID` with issues. A time range exceeding 365 days produces an `invalid_request` issue. Other issue codes include `too_many_buckets`, `too_many_rows`, `field_not_allowed`, `invalid_metadata_key`, and `operator_not_allowed`. |
|
|
937
|
+
| `501` | The installed Core or configured observability store doesn't support aggregation or a requested field. Returns `TRACE_AGGREGATE_UNSUPPORTED`. |
|
|
938
|
+
| `504` | A PostgreSQL or ClickHouse aggregation exceeded its execution timeout. Returns `TRACE_QUERY_EXECUTION_TIMEOUT` without partial results. |
|
|
824
939
|
|
|
825
940
|
## Aggregate spans
|
|
826
941
|
|
|
827
942
|
Use `aggregateSpans()` or `POST /api/observability/spans/aggregate` to compute measures over individual spans instead of traces. Trace aggregation only sees each trace's root span, so an agent run nested inside a workflow isn't counted. Span aggregation counts every span that matches, at any depth.
|
|
828
943
|
|
|
829
|
-
The request has the same shape as [`aggregateTraces()`](#aggregate-request-fields), with the same measures, `having`, `orderBy`, `interval`, limits, and
|
|
944
|
+
The request has the same shape as [`aggregateTraces()`](#aggregate-request-fields), with the same fixed measures, `having`, `orderBy`, `interval`, and shared limits. Its `where`, dimensions, and `countDistinct` fields follow the span contract below. Unsupported span aggregation returns `501 SPAN_AGGREGATE_UNSUPPORTED`.
|
|
830
945
|
|
|
831
946
|
### Aggregate spans with the client SDK
|
|
832
947
|
|
|
@@ -853,30 +968,36 @@ Send the same body to `POST /api/observability/spans/aggregate` to use HTTP dire
|
|
|
853
968
|
|
|
854
969
|
### Span selection
|
|
855
970
|
|
|
856
|
-
Aggregation
|
|
971
|
+
Aggregation uses the same span predicates as [`querySpans()`](#query-spans), with the ClickHouse selection differences described below:
|
|
857
972
|
|
|
858
973
|
- `timeRange` bounds each span's `startedAt`, and can span up to 365 days.
|
|
859
|
-
- `where` accepts the `querySpans()` fields and operators, such as `spanType`, `name`, `status`, `durationMs`, entity and parent/root entity fields, and context identifiers.
|
|
974
|
+
- `where` accepts the `querySpans()` fields and operators, such as `spanType`, `name`, `status`, `durationMs`, `model`, `provider`, entity and parent/root entity fields, and context identifiers. Like `querySpans()`, `model` and `provider` filters read only the span's columns, not its attributes.
|
|
860
975
|
- Only completed spans count. When a span was exported more than once, only its latest copy counts.
|
|
861
976
|
|
|
862
|
-
`count`
|
|
977
|
+
`count` counts selected spans in each group or time bucket. Buckets use each span's `startedAt`. `duration.*` measures read the span's own duration, and `errorCount` and `errorRate` read whether the span itself recorded an error.
|
|
863
978
|
|
|
864
|
-
On ClickHouse, a
|
|
979
|
+
On ClickHouse, a direct `spanType` `eq` or `in` predicate, or one immediately inside an `and`, can select the smaller branch table. All selected types must be branch types: `agent_run`, `workflow_run`, `processor_run`, `scorer_run`, `rag_ingestion`, `tool_call`, or `mcp_tool_call`. Every requested field must be available in that table. An equivalent `or` predicate doesn't select this optimization. The branch path can differ from `querySpans()`:
|
|
865
980
|
|
|
866
981
|
- Spans written before an upgrade from `@mastra/clickhouse` 1.4, 1.5, or 1.6 aren't in the branch table, so older windows can count fewer spans.
|
|
867
|
-
- After ClickHouse merges the copies of a span exported more than once, the branch table keeps the copy written last, not the one that ended last.
|
|
982
|
+
- After ClickHouse merges the copies of a span exported more than once, the branch table keeps the copy written last, not the one that ended last. The retained version's status and duration can differ. This can change filtered counts and group membership even though the span identity is unchanged.
|
|
868
983
|
|
|
869
984
|
### Span dimensions
|
|
870
985
|
|
|
871
|
-
| Dimension | Notes
|
|
872
|
-
| --------------------------------------------------------------------------------- |
|
|
873
|
-
| `name`, `spanType`, `status` | The span's own name, type, and status. `status` is `error` when the span recorded an error and `success` otherwise.
|
|
874
|
-
| `
|
|
875
|
-
| `
|
|
876
|
-
| `
|
|
877
|
-
| `
|
|
986
|
+
| Dimension | Notes |
|
|
987
|
+
| --------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
988
|
+
| `name`, `spanType`, `status` | The span's own name, type, and status. `status` is `error` when the span recorded an error and `success` otherwise. |
|
|
989
|
+
| `outcome`, `stopReason` | How the span ended and why. Spans without a recorded value group as `null`. Requires the `spanQueryOutcomeFields` capability. |
|
|
990
|
+
| `entityType`, `entityId`, `entityName` | The entity that produced the span. |
|
|
991
|
+
| `parentEntityType`, `parentEntityName`, `rootEntityType`, `rootEntityName` | The calling entity and the trace's root entity, such as the agent that made a model call. |
|
|
992
|
+
| `environment`, `serviceName`, `executionSource` | Deployment attributes recorded on the span. |
|
|
993
|
+
| `threadId`, `resourceId`, `userId`, `sessionId`, `organizationId`, `experimentId` | Context identifiers recorded on the span. |
|
|
994
|
+
| `model`, `provider` | The model and provider recorded on the span's `model` and `provider` columns, such as `claude-sonnet-5-5` and `anthropic`. Requires the `spanAggregateModelDimensions` capability. |
|
|
878
995
|
|
|
879
|
-
`countDistinct.<field>` accepts these dimensions and `traceId`, so `countDistinct.traceId` counts the traces the selected spans belong to. Span aggregation can't group by `metadata.<key
|
|
996
|
+
`countDistinct.<field>` accepts these dimensions and `traceId`, so `countDistinct.traceId` counts the traces the selected spans belong to. Span aggregation can't group by `metadata.<key>` or identity fields such as `traceId`, `spanId`, or `runId`.
|
|
997
|
+
|
|
998
|
+
Like the `model` and `provider` filters, the `model` and `provider` dimensions read only the span's columns, not its attributes. Existing spans aren't backfilled, so spans exported before `@mastra/observability` filled the columns group under `null`, and `countDistinct.model` doesn't count them.
|
|
999
|
+
|
|
1000
|
+
Stores advertise support through the `span-aggregate-model-dimensions` feature, and the server reports it as the `spanAggregateModelDimensions` capability. Against an older store, a request that groups by `model` or `provider` or counts their distinct values returns `501` before storage runs.
|
|
880
1001
|
|
|
881
1002
|
### Span tokens and cost
|
|
882
1003
|
|
|
@@ -897,9 +1018,28 @@ const tokensByAgent = await mastraClient.aggregateSpans({
|
|
|
897
1018
|
})
|
|
898
1019
|
```
|
|
899
1020
|
|
|
1021
|
+
To compare models instead, group the same spans by `model`:
|
|
1022
|
+
|
|
1023
|
+
```typescript
|
|
1024
|
+
const { capabilities } = await mastraClient.getObservabilityCapabilities()
|
|
1025
|
+
|
|
1026
|
+
if (capabilities.spanAggregateModelDimensions) {
|
|
1027
|
+
const { rows } = await mastraClient.aggregateSpans({
|
|
1028
|
+
timeRange: {
|
|
1029
|
+
from: '2026-08-01T00:00:00.000Z',
|
|
1030
|
+
to: '2026-08-08T00:00:00.000Z',
|
|
1031
|
+
},
|
|
1032
|
+
where: { op: 'eq', left: { path: 'spanType' }, right: { literal: 'model_generation' } },
|
|
1033
|
+
groupBy: ['provider', 'model'],
|
|
1034
|
+
measures: ['count', 'tokens.total.sum', 'cost.sum'],
|
|
1035
|
+
})
|
|
1036
|
+
// rows[0]: { dimensions: { provider: 'anthropic', model: 'claude-sonnet-5-5' }, measures: { ... } }
|
|
1037
|
+
}
|
|
1038
|
+
```
|
|
1039
|
+
|
|
900
1040
|
### List the spans behind a group
|
|
901
1041
|
|
|
902
|
-
Aggregation never returns span records. To inspect a group, send the same `timeRange` and `where` to `querySpans()`, and add
|
|
1042
|
+
Aggregation never returns span records. To inspect a group, send the same `timeRange` and `where` to `querySpans()`, and add non-null dimension values as `eq` conditions. For a null dimension, use `notExists`, such as `{ op: 'notExists', path: 'entityName' }`. `eq` with `literal: null` is rejected. For example, add `{ op: 'eq', left: { path: 'status' }, right: { literal: 'error' } }` to list the failed agent runs from the first example.
|
|
903
1043
|
|
|
904
1044
|
## Trusted tenant scope
|
|
905
1045
|
|
|
@@ -1017,12 +1157,12 @@ Discovery returns string values only. Missing values, empty strings, and strings
|
|
|
1017
1157
|
|
|
1018
1158
|
Value suggestions are available for these canonical fields:
|
|
1019
1159
|
|
|
1020
|
-
| Scope | Fields
|
|
1021
|
-
| ----- |
|
|
1022
|
-
| Trace | `entityName`, `entityType`, `name`, `spanType`, `environment`, `status`, `tags` |
|
|
1023
|
-
| Spans | `name`, `spanType`, `model`, `provider`, `status`, `entityType`, `entityName` |
|
|
1160
|
+
| Scope | Fields |
|
|
1161
|
+
| ----- | -------------------------------------------------------------------------------------------------------- |
|
|
1162
|
+
| Trace | `entityName`, `entityType`, `name`, `spanType`, `environment`, `status`, `outcome`, `stopReason`, `tags` |
|
|
1163
|
+
| Spans | `name`, `spanType`, `model`, `provider`, `status`, `outcome`, `stopReason`, `entityType`, `entityName` |
|
|
1024
1164
|
|
|
1025
|
-
Each executable top-level string `metadata.<key>` field also supports value suggestions in the `trace` scope. Identifiers, timestamps, durations, time to first chunk, time to first token, errors, and version IDs don't. The fields response still includes canonical fields with `valueSuggestions: false`. The values endpoint rejects those paths with `422 TRACE_QUERY_INVALID`.
|
|
1165
|
+
`outcome` and `stopReason` suggestions require the `traceQueryOutcomeFields` capability. Each executable top-level string `metadata.<key>` field also supports value suggestions in the `trace` scope. Identifiers, timestamps, durations, time to first chunk, time to first token, errors, and version IDs don't. The fields response still includes canonical fields with `valueSuggestions: false`. The values endpoint rejects those paths with `422 TRACE_QUERY_INVALID`.
|
|
1026
1166
|
|
|
1027
1167
|
Both discovery requests require a half-open time range. They use the same current-record semantics as `queryTraces()`: qualifying completed roots satisfy `from <= startedAt < to`, and related values come only from current records joined to those roots by `traceId`. Discovery doesn't accept or apply a draft `where` predicate.
|
|
1028
1168
|
|
|
@@ -704,20 +704,24 @@ How a supplied `id` is treated depends on the storage adapter. Some adapters ups
|
|
|
704
704
|
|
|
705
705
|
## MCP
|
|
706
706
|
|
|
707
|
-
| Method | Path | Description
|
|
708
|
-
| ------ | ------------------------------------------ |
|
|
709
|
-
| `GET` | `/api/mcp/v0/servers` | List MCP servers
|
|
710
|
-
| `GET` | `/api/mcp/v0/servers/:id` | Get server details
|
|
711
|
-
| `GET` | `/api/mcp/:serverId/tools` | List server tools
|
|
712
|
-
| `GET` | `/api/mcp/:serverId/tools/:toolId` | Get tool details
|
|
713
|
-
| `POST` | `/api/mcp/:serverId/tools/:toolId/execute` | Execute a tool
|
|
714
|
-
| `GET` | `/api/mcp/:serverId/resources` | List server resources
|
|
715
|
-
| `POST` | `/api/mcp/:serverId/resources/read` | Read a resource
|
|
716
|
-
| `
|
|
717
|
-
| `
|
|
718
|
-
| `POST` | `/api/mcp/:serverId/
|
|
719
|
-
|
|
720
|
-
|
|
707
|
+
| Method | Path | Description |
|
|
708
|
+
| ------ | ------------------------------------------ | -------------------------------------------------- |
|
|
709
|
+
| `GET` | `/api/mcp/v0/servers` | List MCP servers |
|
|
710
|
+
| `GET` | `/api/mcp/v0/servers/:id` | Get server details |
|
|
711
|
+
| `GET` | `/api/mcp/:serverId/tools` | List server tools |
|
|
712
|
+
| `GET` | `/api/mcp/:serverId/tools/:toolId` | Get tool details |
|
|
713
|
+
| `POST` | `/api/mcp/:serverId/tools/:toolId/execute` | Execute a tool |
|
|
714
|
+
| `GET` | `/api/mcp/:serverId/resources` | List server resources |
|
|
715
|
+
| `POST` | `/api/mcp/:serverId/resources/read` | Read a resource |
|
|
716
|
+
| `GET` | `/api/mcp/:serverId/tasks` | List a server's recent MCP tasks (experimental) |
|
|
717
|
+
| `POST` | `/api/mcp/:serverId/tasks/find` | Find an MCP task by its task id (experimental) |
|
|
718
|
+
| `POST` | `/api/mcp/:serverId/tasks/:taskKey/cancel` | Cancel an MCP task (experimental) |
|
|
719
|
+
| `POST` | `/api/mcp/:serverId/tasks/:taskKey/input` | Answer an MCP task's input requests (experimental) |
|
|
720
|
+
| `ALL` | `/api/mcp/:serverId/mcp` | MCP HTTP transport |
|
|
721
|
+
| `ALL` | `/api/mcp/:serverId/sse` | Legacy MCP SSE transport |
|
|
722
|
+
| `POST` | `/api/mcp/:serverId/messages` | Legacy MCP SSE messages |
|
|
723
|
+
|
|
724
|
+
All MCP routes declare `requiresAuth: true`. Authentication is enforced only when `server.auth` is configured. With an RBAC provider, listing and finding tasks need `mcp:read`. Cancelling and answering a task need `mcp:write`, since they act on a task another client started. Paths use the default `/api` prefix. The legacy SSE routes return `404` for MCP v2 servers.
|
|
721
725
|
|
|
722
726
|
## Responses API
|
|
723
727
|
|