@mastra/libsql 0.0.0-structured-output-issue-20260227214155 → 0.0.0-structured-output-errors-20260409185629
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +344 -3
- package/LICENSE.md +15 -0
- package/dist/docs/SKILL.md +15 -19
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/docs-agents-agent-approval.md +136 -185
- package/dist/docs/references/docs-agents-networks.md +90 -207
- package/dist/docs/references/docs-memory-memory-processors.md +15 -15
- package/dist/docs/references/docs-memory-message-history.md +10 -8
- package/dist/docs/references/docs-memory-overview.md +219 -24
- package/dist/docs/references/docs-memory-semantic-recall.md +54 -29
- package/dist/docs/references/docs-memory-storage.md +14 -16
- package/dist/docs/references/docs-memory-working-memory.md +22 -22
- package/dist/docs/references/docs-rag-retrieval.md +16 -16
- package/dist/docs/references/docs-workflows-snapshots.md +1 -1
- package/dist/docs/references/guides-agent-frameworks-ai-sdk.md +3 -3
- package/dist/docs/references/reference-core-getMemory.md +4 -5
- package/dist/docs/references/reference-core-listMemory.md +3 -4
- package/dist/docs/references/reference-core-mastra-class.md +18 -18
- package/dist/docs/references/reference-memory-memory-class.md +16 -18
- package/dist/docs/references/reference-storage-composite.md +19 -11
- package/dist/docs/references/reference-storage-dynamodb.md +16 -16
- package/dist/docs/references/reference-storage-libsql.md +3 -3
- package/dist/docs/references/reference-vectors-libsql.md +47 -47
- package/dist/index.cjs +512 -82
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +513 -83
- package/dist/index.js.map +1 -1
- package/dist/storage/db/index.d.ts +12 -0
- package/dist/storage/db/index.d.ts.map +1 -1
- package/dist/storage/domains/datasets/index.d.ts.map +1 -1
- package/dist/storage/domains/experiments/index.d.ts +3 -1
- package/dist/storage/domains/experiments/index.d.ts.map +1 -1
- package/dist/storage/domains/memory/index.d.ts +5 -2
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/package.json +8 -8
- package/dist/docs/references/docs-agents-agent-memory.md +0 -209
- package/dist/docs/references/docs-agents-network-approval.md +0 -275
- package/dist/docs/references/docs-observability-overview.md +0 -70
- package/dist/docs/references/docs-observability-tracing-exporters-default.md +0 -209
|
@@ -1,275 +0,0 @@
|
|
|
1
|
-
# Network Approval
|
|
2
|
-
|
|
3
|
-
Agent networks can require the same [human-in-the-loop](https://mastra.ai/docs/workflows/human-in-the-loop) oversight used in individual agents and workflows. When a tool, subagent, or workflow within a network requires approval or suspends execution, the network pauses and emits events that allow your application to collect user input before resuming.
|
|
4
|
-
|
|
5
|
-
## Storage
|
|
6
|
-
|
|
7
|
-
Network approval uses snapshots to capture execution state. Ensure you've enabled a storage provider in your Mastra instance. If storage isn't enabled you'll see an error relating to snapshot not found.
|
|
8
|
-
|
|
9
|
-
```typescript
|
|
10
|
-
import { Mastra } from '@mastra/core/mastra'
|
|
11
|
-
import { LibSQLStore } from '@mastra/libsql'
|
|
12
|
-
|
|
13
|
-
export const mastra = new Mastra({
|
|
14
|
-
storage: new LibSQLStore({
|
|
15
|
-
id: 'mastra-storage',
|
|
16
|
-
url: ':memory:',
|
|
17
|
-
}),
|
|
18
|
-
})
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
## Approving network tool calls
|
|
22
|
-
|
|
23
|
-
When a tool within a network has `requireApproval: true`, the network stream emits an `agent-execution-approval` chunk and pauses. To allow the tool to execute, call `approveNetworkToolCall` with the `runId`.
|
|
24
|
-
|
|
25
|
-
```typescript
|
|
26
|
-
const stream = await routingAgent.network('Process this query', {
|
|
27
|
-
memory: {
|
|
28
|
-
thread: 'user-123',
|
|
29
|
-
resource: 'my-app',
|
|
30
|
-
},
|
|
31
|
-
})
|
|
32
|
-
|
|
33
|
-
let runId: string
|
|
34
|
-
|
|
35
|
-
for await (const chunk of stream) {
|
|
36
|
-
runId = stream.runId
|
|
37
|
-
// if the requirApproval is in a tool inside a subAgent or the subAgent has requireToolApproval set to true
|
|
38
|
-
if (chunk.type === 'agent-execution-approval') {
|
|
39
|
-
console.log('Tool requires approval:', chunk.payload)
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
// if the requirApproval is in a tool directly in the network agent
|
|
43
|
-
if (chunk.type === 'tool-execution-approval') {
|
|
44
|
-
console.log('Tool requires approval:', chunk.payload)
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
// Approve and resume execution
|
|
49
|
-
const approvedStream = await routingAgent.approveNetworkToolCall({
|
|
50
|
-
runId,
|
|
51
|
-
memory: {
|
|
52
|
-
thread: 'user-123',
|
|
53
|
-
resource: 'my-app',
|
|
54
|
-
},
|
|
55
|
-
})
|
|
56
|
-
|
|
57
|
-
for await (const chunk of approvedStream) {
|
|
58
|
-
if (chunk.type === 'network-execution-event-step-finish') {
|
|
59
|
-
console.log(chunk.payload.result)
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
```
|
|
63
|
-
|
|
64
|
-
## Declining network tool calls
|
|
65
|
-
|
|
66
|
-
To decline a pending tool call and prevent execution, call `declineNetworkToolCall`. The network continues without executing the tool.
|
|
67
|
-
|
|
68
|
-
```typescript
|
|
69
|
-
const declinedStream = await routingAgent.declineNetworkToolCall({
|
|
70
|
-
runId,
|
|
71
|
-
memory: {
|
|
72
|
-
thread: 'user-123',
|
|
73
|
-
resource: 'my-app',
|
|
74
|
-
},
|
|
75
|
-
})
|
|
76
|
-
|
|
77
|
-
for await (const chunk of declinedStream) {
|
|
78
|
-
if (chunk.type === 'network-execution-event-step-finish') {
|
|
79
|
-
console.log(chunk.payload.result)
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
```
|
|
83
|
-
|
|
84
|
-
## Resuming suspended networks
|
|
85
|
-
|
|
86
|
-
When a primitive in the network calls `suspend()`, the stream emits an `agent-execution-suspended`/`tool-execution-suspended`/`workflow-execution-suspended` chunk with a `suspendPayload` containing context from the primitive. Use `resumeNetwork` to provide the data requested by the primitive and continue execution.
|
|
87
|
-
|
|
88
|
-
```typescript
|
|
89
|
-
import { createTool } from '@mastra/core/tools'
|
|
90
|
-
import { z } from 'zod'
|
|
91
|
-
|
|
92
|
-
const confirmationTool = createTool({
|
|
93
|
-
id: 'confirmation-tool',
|
|
94
|
-
description: 'Requests user confirmation before proceeding',
|
|
95
|
-
inputSchema: z.object({
|
|
96
|
-
action: z.string(),
|
|
97
|
-
}),
|
|
98
|
-
outputSchema: z.object({
|
|
99
|
-
confirmed: z.boolean(),
|
|
100
|
-
action: z.string(),
|
|
101
|
-
}),
|
|
102
|
-
suspendSchema: z.object({
|
|
103
|
-
message: z.string(),
|
|
104
|
-
action: z.string(),
|
|
105
|
-
}),
|
|
106
|
-
resumeSchema: z.object({
|
|
107
|
-
confirmed: z.boolean(),
|
|
108
|
-
}),
|
|
109
|
-
execute: async (inputData, context) => {
|
|
110
|
-
const { resumeData, suspend } = context?.agent ?? {}
|
|
111
|
-
|
|
112
|
-
if (!resumeData?.confirmed) {
|
|
113
|
-
return suspend?.({
|
|
114
|
-
message: `Please confirm: ${inputData.action}`,
|
|
115
|
-
action: inputData.action,
|
|
116
|
-
})
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
return { confirmed: true, action: inputData.action }
|
|
120
|
-
},
|
|
121
|
-
})
|
|
122
|
-
```
|
|
123
|
-
|
|
124
|
-
Handle the suspension and resume with user-provided data:
|
|
125
|
-
|
|
126
|
-
```typescript
|
|
127
|
-
const stream = await routingAgent.network('Delete the old records', {
|
|
128
|
-
memory: {
|
|
129
|
-
thread: 'user-123',
|
|
130
|
-
resource: 'my-app',
|
|
131
|
-
},
|
|
132
|
-
})
|
|
133
|
-
|
|
134
|
-
for await (const chunk of stream) {
|
|
135
|
-
if (chunk.type === 'workflow-execution-suspended') {
|
|
136
|
-
console.log(chunk.payload.suspendPayload)
|
|
137
|
-
// { message: "Please confirm: delete old records", action: "delete old records" }
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
// Resume with user confirmation
|
|
142
|
-
const resumedStream = await routingAgent.resumeNetwork(
|
|
143
|
-
{ confirmed: true },
|
|
144
|
-
{
|
|
145
|
-
runId: stream.runId,
|
|
146
|
-
memory: {
|
|
147
|
-
thread: 'user-123',
|
|
148
|
-
resource: 'my-app',
|
|
149
|
-
},
|
|
150
|
-
},
|
|
151
|
-
)
|
|
152
|
-
|
|
153
|
-
for await (const chunk of resumedStream) {
|
|
154
|
-
if (chunk.type === 'network-execution-event-step-finish') {
|
|
155
|
-
console.log(chunk.payload.result)
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
## Automatic primitive resumption
|
|
161
|
-
|
|
162
|
-
When using primitives that call `suspend()`, you can enable automatic resumption so the network resumes suspended primitives based on the user's next message. This creates a conversational flow where users provide the required information naturally.
|
|
163
|
-
|
|
164
|
-
### Enabling auto-resume
|
|
165
|
-
|
|
166
|
-
Set `autoResumeSuspendedTools` to `true` in the agent's `defaultNetworkOptions` or when calling `network()`:
|
|
167
|
-
|
|
168
|
-
```typescript
|
|
169
|
-
import { Agent } from '@mastra/core/agent'
|
|
170
|
-
import { Memory } from '@mastra/memory'
|
|
171
|
-
|
|
172
|
-
// Option 1: In agent configuration
|
|
173
|
-
const routingAgent = new Agent({
|
|
174
|
-
id: 'routing-agent',
|
|
175
|
-
name: 'Routing Agent',
|
|
176
|
-
instructions: 'You coordinate tasks across multiple agents',
|
|
177
|
-
model: 'openai/gpt-4o-mini',
|
|
178
|
-
tools: { confirmationTool },
|
|
179
|
-
memory: new Memory(),
|
|
180
|
-
defaultNetworkOptions: {
|
|
181
|
-
autoResumeSuspendedTools: true,
|
|
182
|
-
},
|
|
183
|
-
})
|
|
184
|
-
|
|
185
|
-
// Option 2: Per-request
|
|
186
|
-
const stream = await routingAgent.network('Process this request', {
|
|
187
|
-
autoResumeSuspendedTools: true,
|
|
188
|
-
memory: {
|
|
189
|
-
thread: 'user-123',
|
|
190
|
-
resource: 'my-app',
|
|
191
|
-
},
|
|
192
|
-
})
|
|
193
|
-
```
|
|
194
|
-
|
|
195
|
-
### How it works
|
|
196
|
-
|
|
197
|
-
When `autoResumeSuspendedTools` is enabled:
|
|
198
|
-
|
|
199
|
-
1. A primitive suspends execution by calling `suspend()` with a payload
|
|
200
|
-
|
|
201
|
-
2. The suspension is persisted to memory along with the conversation
|
|
202
|
-
|
|
203
|
-
3. When the user sends their next message on the same thread, the network:
|
|
204
|
-
|
|
205
|
-
- Detects the suspended primitive from message history
|
|
206
|
-
- Extracts `resumeData` from the user's message based on the tool's `resumeSchema`
|
|
207
|
-
- Automatically resumes the primitive with the extracted data
|
|
208
|
-
|
|
209
|
-
### Example
|
|
210
|
-
|
|
211
|
-
```typescript
|
|
212
|
-
const stream = await routingAgent.network('Delete the old records', {
|
|
213
|
-
autoResumeSuspendedTools: true,
|
|
214
|
-
memory: {
|
|
215
|
-
thread: 'user-123',
|
|
216
|
-
resource: 'my-app',
|
|
217
|
-
},
|
|
218
|
-
})
|
|
219
|
-
|
|
220
|
-
for await (const chunk of stream) {
|
|
221
|
-
if (chunk.type === 'workflow-execution-suspended') {
|
|
222
|
-
console.log(chunk.payload.suspendPayload)
|
|
223
|
-
// { message: "Please confirm: delete old records", action: "delete old records" }
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// User provides confirmation in their next message
|
|
228
|
-
const resumedStream = await routingAgent.network('Yes, confirmed', {
|
|
229
|
-
autoResumeSuspendedTools: true,
|
|
230
|
-
memory: {
|
|
231
|
-
thread: 'user-123',
|
|
232
|
-
resource: 'my-app',
|
|
233
|
-
},
|
|
234
|
-
})
|
|
235
|
-
|
|
236
|
-
for await (const chunk of resumedStream) {
|
|
237
|
-
if (chunk.type === 'network-execution-event-step-finish') {
|
|
238
|
-
console.log(chunk.payload.result)
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
```
|
|
242
|
-
|
|
243
|
-
**Conversation flow:**
|
|
244
|
-
|
|
245
|
-
```text
|
|
246
|
-
User: "Delete the old records"
|
|
247
|
-
Agent: "Please confirm: delete old records"
|
|
248
|
-
|
|
249
|
-
User: "Yes, confirmed"
|
|
250
|
-
Agent: "Records deleted successfully"
|
|
251
|
-
```
|
|
252
|
-
|
|
253
|
-
### Requirements
|
|
254
|
-
|
|
255
|
-
For automatic tool resumption to work:
|
|
256
|
-
|
|
257
|
-
- **Memory configured**: The agent needs memory to track suspended tools across messages
|
|
258
|
-
- **Same thread**: The follow-up message must use the same memory thread and resource identifiers
|
|
259
|
-
- **`resumeSchema` defined**: The tool (either directly in the network agent or in a subAgent) / workflow (step that gets suspended) must define a `resumeSchema` so the agent knows what data to extract from the user's message
|
|
260
|
-
|
|
261
|
-
### Manual vs automatic resumption
|
|
262
|
-
|
|
263
|
-
| Approach | Use case |
|
|
264
|
-
| -------------------------------------- | ------------------------------------------------------------------------ |
|
|
265
|
-
| Manual (`resumeNetwork()`) | Programmatic control, webhooks, button clicks, external triggers |
|
|
266
|
-
| Automatic (`autoResumeSuspendedTools`) | Conversational flows where users provide resume data in natural language |
|
|
267
|
-
|
|
268
|
-
Both approaches work with the same tool definitions. Automatic resumption triggers only when suspended tools exist in the message history and the user sends a new message on the same thread.
|
|
269
|
-
|
|
270
|
-
## Related
|
|
271
|
-
|
|
272
|
-
- [Agent Networks](https://mastra.ai/docs/agents/networks)
|
|
273
|
-
- [Agent Approval](https://mastra.ai/docs/agents/agent-approval)
|
|
274
|
-
- [Human-in-the-Loop](https://mastra.ai/docs/workflows/human-in-the-loop)
|
|
275
|
-
- [Agent Memory](https://mastra.ai/docs/agents/agent-memory)
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
# Observability Overview
|
|
2
|
-
|
|
3
|
-
Mastra provides observability features for AI applications. Monitor LLM operations, trace agent decisions, and debug complex workflows with tools that understand AI-specific patterns.
|
|
4
|
-
|
|
5
|
-
## Key Features
|
|
6
|
-
|
|
7
|
-
### Tracing
|
|
8
|
-
|
|
9
|
-
Specialized tracing for AI operations that captures:
|
|
10
|
-
|
|
11
|
-
- **Model interactions**: Token usage, latency, prompts, and completions
|
|
12
|
-
- **Agent execution**: Decision paths, tool calls, and memory operations
|
|
13
|
-
- **Workflow steps**: Branching logic, parallel execution, and step outputs
|
|
14
|
-
- **Automatic instrumentation**: Tracing with decorators
|
|
15
|
-
|
|
16
|
-
## Storage Requirements
|
|
17
|
-
|
|
18
|
-
The `DefaultExporter` persists traces to your configured storage backend. Not all storage providers support observability—for the full list, see [Storage Provider Support](https://mastra.ai/docs/observability/tracing/exporters/default).
|
|
19
|
-
|
|
20
|
-
For production environments with high traffic, we recommend using **ClickHouse** for the observability domain via [composite storage](https://mastra.ai/reference/storage/composite). See [Production Recommendations](https://mastra.ai/docs/observability/tracing/exporters/default) for details.
|
|
21
|
-
|
|
22
|
-
## Quick Start
|
|
23
|
-
|
|
24
|
-
Configure Observability in your Mastra instance:
|
|
25
|
-
|
|
26
|
-
```typescript
|
|
27
|
-
import { Mastra } from '@mastra/core'
|
|
28
|
-
import { PinoLogger } from '@mastra/loggers'
|
|
29
|
-
import { LibSQLStore } from '@mastra/libsql'
|
|
30
|
-
import {
|
|
31
|
-
Observability,
|
|
32
|
-
DefaultExporter,
|
|
33
|
-
CloudExporter,
|
|
34
|
-
SensitiveDataFilter,
|
|
35
|
-
} from '@mastra/observability'
|
|
36
|
-
|
|
37
|
-
export const mastra = new Mastra({
|
|
38
|
-
logger: new PinoLogger(),
|
|
39
|
-
storage: new LibSQLStore({
|
|
40
|
-
id: 'mastra-storage',
|
|
41
|
-
url: 'file:./mastra.db', // Storage is required for tracing
|
|
42
|
-
}),
|
|
43
|
-
observability: new Observability({
|
|
44
|
-
configs: {
|
|
45
|
-
default: {
|
|
46
|
-
serviceName: 'mastra',
|
|
47
|
-
exporters: [
|
|
48
|
-
new DefaultExporter(), // Persists traces to storage for Mastra Studio
|
|
49
|
-
new CloudExporter(), // Sends traces to Mastra Cloud (if MASTRA_CLOUD_ACCESS_TOKEN is set)
|
|
50
|
-
],
|
|
51
|
-
spanOutputProcessors: [
|
|
52
|
-
new SensitiveDataFilter(), // Redacts sensitive data like passwords, tokens, keys
|
|
53
|
-
],
|
|
54
|
-
},
|
|
55
|
-
},
|
|
56
|
-
}),
|
|
57
|
-
})
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
> **Serverless environments:** The `file:./mastra.db` storage URL uses the local filesystem, which doesn't work in serverless environments like Vercel, AWS Lambda, or Cloudflare Workers. For serverless deployments, use external storage. See the [Vercel deployment guide](https://mastra.ai/guides/deployment/vercel) for a complete example.
|
|
61
|
-
|
|
62
|
-
With this basic setup, you will see Traces and Logs in both Studio and in Mastra Cloud.
|
|
63
|
-
|
|
64
|
-
We also support various external tracing providers like MLflow, Langfuse, Braintrust, and any OpenTelemetry-compatible platform (Datadog, New Relic, SigNoz, etc.). See more about this in the [Tracing](https://mastra.ai/docs/observability/tracing/overview) documentation.
|
|
65
|
-
|
|
66
|
-
## What's Next?
|
|
67
|
-
|
|
68
|
-
- **[Set up Tracing](https://mastra.ai/docs/observability/tracing/overview)**: Configure tracing for your application
|
|
69
|
-
- **[Configure Logging](https://mastra.ai/docs/observability/logging)**: Add structured logging
|
|
70
|
-
- **[API Reference](https://mastra.ai/reference/observability/tracing/instances)**: Detailed configuration options
|
|
@@ -1,209 +0,0 @@
|
|
|
1
|
-
# Default Exporter
|
|
2
|
-
|
|
3
|
-
The `DefaultExporter` persists traces to your configured storage backend, making them accessible through Studio. It's automatically enabled when using the default observability configuration and requires no external services.
|
|
4
|
-
|
|
5
|
-
> **Production Observability:** Observability data can quickly overwhelm general-purpose databases in production. For high-traffic applications, we recommend using **ClickHouse** for the observability storage domain via [composite storage](https://mastra.ai/reference/storage/composite). See [Production Recommendations](#production-recommendations) for details.
|
|
6
|
-
|
|
7
|
-
## Configuration
|
|
8
|
-
|
|
9
|
-
### Prerequisites
|
|
10
|
-
|
|
11
|
-
1. **Storage Backend**: Configure a storage provider (libSQL, PostgreSQL, etc.)
|
|
12
|
-
2. **Studio**: Install for viewing traces locally
|
|
13
|
-
|
|
14
|
-
### Basic Setup
|
|
15
|
-
|
|
16
|
-
```typescript
|
|
17
|
-
import { Mastra } from '@mastra/core'
|
|
18
|
-
import { Observability, DefaultExporter } from '@mastra/observability'
|
|
19
|
-
import { LibSQLStore } from '@mastra/libsql'
|
|
20
|
-
|
|
21
|
-
export const mastra = new Mastra({
|
|
22
|
-
storage: new LibSQLStore({
|
|
23
|
-
id: 'mastra-storage',
|
|
24
|
-
url: 'file:./mastra.db', // Required for trace persistence
|
|
25
|
-
}),
|
|
26
|
-
observability: new Observability({
|
|
27
|
-
configs: {
|
|
28
|
-
local: {
|
|
29
|
-
serviceName: 'my-service',
|
|
30
|
-
exporters: [new DefaultExporter()],
|
|
31
|
-
},
|
|
32
|
-
},
|
|
33
|
-
}),
|
|
34
|
-
})
|
|
35
|
-
```
|
|
36
|
-
|
|
37
|
-
### Recommended Configuration
|
|
38
|
-
|
|
39
|
-
Include DefaultExporter in your observability configuration:
|
|
40
|
-
|
|
41
|
-
```typescript
|
|
42
|
-
import { Mastra } from '@mastra/core'
|
|
43
|
-
import {
|
|
44
|
-
Observability,
|
|
45
|
-
DefaultExporter,
|
|
46
|
-
CloudExporter,
|
|
47
|
-
SensitiveDataFilter,
|
|
48
|
-
} from '@mastra/observability'
|
|
49
|
-
import { LibSQLStore } from '@mastra/libsql'
|
|
50
|
-
|
|
51
|
-
export const mastra = new Mastra({
|
|
52
|
-
storage: new LibSQLStore({
|
|
53
|
-
id: 'mastra-storage',
|
|
54
|
-
url: 'file:./mastra.db',
|
|
55
|
-
}),
|
|
56
|
-
observability: new Observability({
|
|
57
|
-
configs: {
|
|
58
|
-
default: {
|
|
59
|
-
serviceName: 'mastra',
|
|
60
|
-
exporters: [
|
|
61
|
-
new DefaultExporter(), // Persists traces to storage for Mastra Studio
|
|
62
|
-
new CloudExporter(), // Sends traces to Mastra Cloud (requires MASTRA_CLOUD_ACCESS_TOKEN)
|
|
63
|
-
],
|
|
64
|
-
spanOutputProcessors: [new SensitiveDataFilter()],
|
|
65
|
-
},
|
|
66
|
-
},
|
|
67
|
-
}),
|
|
68
|
-
})
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
## Viewing Traces
|
|
72
|
-
|
|
73
|
-
### Studio
|
|
74
|
-
|
|
75
|
-
Access your traces through Studio:
|
|
76
|
-
|
|
77
|
-
1. Start Studio
|
|
78
|
-
2. Navigate to Observability
|
|
79
|
-
3. Filter and search your local traces
|
|
80
|
-
4. Inspect detailed span information
|
|
81
|
-
|
|
82
|
-
## Tracing Strategies
|
|
83
|
-
|
|
84
|
-
DefaultExporter automatically selects the optimal tracing strategy based on your storage provider. You can also override this selection if needed.
|
|
85
|
-
|
|
86
|
-
### Available Strategies
|
|
87
|
-
|
|
88
|
-
| Strategy | Description | Use Case |
|
|
89
|
-
| ---------------------- | --------------------------------------------------------- | ----------------------------------- |
|
|
90
|
-
| **realtime** | Process each event immediately | Development, debugging, low traffic |
|
|
91
|
-
| **batch-with-updates** | Buffer events and batch write with full lifecycle support | Low volume Production |
|
|
92
|
-
| **insert-only** | Only process completed spans, ignore updates | High volume Production |
|
|
93
|
-
|
|
94
|
-
### Strategy Configuration
|
|
95
|
-
|
|
96
|
-
```typescript
|
|
97
|
-
new DefaultExporter({
|
|
98
|
-
strategy: 'auto', // Default - let storage provider decide
|
|
99
|
-
// or explicitly set:
|
|
100
|
-
// strategy: 'realtime' | 'batch-with-updates' | 'insert-only'
|
|
101
|
-
|
|
102
|
-
// Batching configuration (applies to both batch-with-updates and insert-only)
|
|
103
|
-
maxBatchSize: 1000, // Max spans per batch
|
|
104
|
-
maxBatchWaitMs: 5000, // Max wait before flushing
|
|
105
|
-
maxBufferSize: 10000, // Max spans to buffer
|
|
106
|
-
})
|
|
107
|
-
```
|
|
108
|
-
|
|
109
|
-
## Storage Provider Support
|
|
110
|
-
|
|
111
|
-
Different storage providers support different tracing strategies. Some providers support observability for production workloads, while others are intended primarily for local development.
|
|
112
|
-
|
|
113
|
-
If you set the strategy to `'auto'`, the `DefaultExporter` automatically selects the optimal strategy for the storage provider. If you set the strategy to a mode that the storage provider doesn't support, you will get an error message.
|
|
114
|
-
|
|
115
|
-
### Providers with Observability Support
|
|
116
|
-
|
|
117
|
-
| Storage Provider | Preferred Strategy | Supported Strategies | Recommended Use |
|
|
118
|
-
| ---------------------------------------------------------------- | ------------------ | ------------------------------- | ------------------------------------- |
|
|
119
|
-
| **ClickHouse** (`@mastra/clickhouse`) | insert-only | insert-only | Production (high-volume) |
|
|
120
|
-
| **[PostgreSQL](https://mastra.ai/reference/storage/postgresql)** | batch-with-updates | batch-with-updates, insert-only | Production (low volume) |
|
|
121
|
-
| **[MSSQL](https://mastra.ai/reference/storage/mssql)** | batch-with-updates | batch-with-updates, insert-only | Production (low volume) |
|
|
122
|
-
| **[MongoDB](https://mastra.ai/reference/storage/mongodb)** | batch-with-updates | batch-with-updates, insert-only | Production (low volume) |
|
|
123
|
-
| **[libSQL](https://mastra.ai/reference/storage/libsql)** | batch-with-updates | batch-with-updates, insert-only | Default storage, good for development |
|
|
124
|
-
|
|
125
|
-
### Providers without Observability Support
|
|
126
|
-
|
|
127
|
-
The following storage providers **do not support** the observability domain. If you're using one of these providers and need observability, use [composite storage](https://mastra.ai/reference/storage/composite) to route observability data to a supported provider:
|
|
128
|
-
|
|
129
|
-
- [Convex](https://mastra.ai/reference/storage/convex)
|
|
130
|
-
- [DynamoDB](https://mastra.ai/reference/storage/dynamodb)
|
|
131
|
-
- [Cloudflare D1](https://mastra.ai/reference/storage/cloudflare-d1)
|
|
132
|
-
- [Cloudflare Durable Objects](https://mastra.ai/reference/storage/cloudflare)
|
|
133
|
-
- [Upstash](https://mastra.ai/reference/storage/upstash)
|
|
134
|
-
- [LanceDB](https://mastra.ai/reference/storage/lance)
|
|
135
|
-
|
|
136
|
-
### Strategy Benefits
|
|
137
|
-
|
|
138
|
-
- **realtime**: Immediate visibility, best for debugging
|
|
139
|
-
- **batch-with-updates**: 10-100x throughput improvement, full span lifecycle
|
|
140
|
-
- **insert-only**: Additional 70% reduction in database operations, perfect for analytics
|
|
141
|
-
|
|
142
|
-
## Production Recommendations
|
|
143
|
-
|
|
144
|
-
Observability data grows quickly in production environments. A single agent interaction can generate hundreds of spans, and high-traffic applications can produce thousands of traces per day. Most general-purpose databases aren't optimized for this write-heavy, append-only workload.
|
|
145
|
-
|
|
146
|
-
### Recommended: ClickHouse for High-Volume Production
|
|
147
|
-
|
|
148
|
-
[ClickHouse](https://mastra.ai/reference/storage/composite) is a columnar database designed for high-volume analytics workloads. It's the recommended choice for production observability because:
|
|
149
|
-
|
|
150
|
-
- **Optimized for writes**: Handles millions of inserts per second
|
|
151
|
-
- **Efficient compression**: Reduces storage costs for trace data
|
|
152
|
-
- **Fast queries**: Columnar storage enables quick trace lookups and aggregations
|
|
153
|
-
- **Time-series native**: Built-in support for time-based data retention and partitioning
|
|
154
|
-
|
|
155
|
-
### Using Composite Storage
|
|
156
|
-
|
|
157
|
-
If you're using a provider without observability support (like Convex or DynamoDB) or want to optimize performance, use [composite storage](https://mastra.ai/reference/storage/composite) to route observability data to ClickHouse while keeping other data in your primary database.
|
|
158
|
-
|
|
159
|
-
## Batching Behavior
|
|
160
|
-
|
|
161
|
-
### Flush Triggers
|
|
162
|
-
|
|
163
|
-
For both batch strategies (`batch-with-updates` and `insert-only`), traces are flushed to storage when any of these conditions are met:
|
|
164
|
-
|
|
165
|
-
1. **Size trigger**: Buffer reaches `maxBatchSize` spans
|
|
166
|
-
2. **Time trigger**: `maxBatchWaitMs` elapsed since first event
|
|
167
|
-
3. **Emergency flush**: Buffer approaches `maxBufferSize` limit
|
|
168
|
-
4. **Shutdown**: Force flush all pending events
|
|
169
|
-
|
|
170
|
-
### Error Handling
|
|
171
|
-
|
|
172
|
-
The DefaultExporter includes robust error handling for production use:
|
|
173
|
-
|
|
174
|
-
- **Retry Logic**: Exponential backoff (500ms, 1s, 2s, 4s)
|
|
175
|
-
- **Transient Failures**: Automatic retry with backoff
|
|
176
|
-
- **Persistent Failures**: Drop batch after 4 failed attempts
|
|
177
|
-
- **Buffer Overflow**: Prevent memory issues during storage outages
|
|
178
|
-
|
|
179
|
-
### Configuration Examples
|
|
180
|
-
|
|
181
|
-
```typescript
|
|
182
|
-
// Zero config - recommended for most users
|
|
183
|
-
new DefaultExporter()
|
|
184
|
-
|
|
185
|
-
// Development override
|
|
186
|
-
new DefaultExporter({
|
|
187
|
-
strategy: 'realtime', // Immediate visibility for debugging
|
|
188
|
-
})
|
|
189
|
-
|
|
190
|
-
// High-throughput production
|
|
191
|
-
new DefaultExporter({
|
|
192
|
-
maxBatchSize: 2000, // Larger batches
|
|
193
|
-
maxBatchWaitMs: 10000, // Wait longer to fill batches
|
|
194
|
-
maxBufferSize: 50000, // Handle longer outages
|
|
195
|
-
})
|
|
196
|
-
|
|
197
|
-
// Low-latency production
|
|
198
|
-
new DefaultExporter({
|
|
199
|
-
maxBatchSize: 100, // Smaller batches
|
|
200
|
-
maxBatchWaitMs: 1000, // Flush quickly
|
|
201
|
-
})
|
|
202
|
-
```
|
|
203
|
-
|
|
204
|
-
## Related
|
|
205
|
-
|
|
206
|
-
- [Tracing Overview](https://mastra.ai/docs/observability/tracing/overview)
|
|
207
|
-
- [CloudExporter](https://mastra.ai/docs/observability/tracing/exporters/cloud)
|
|
208
|
-
- [Composite Storage](https://mastra.ai/reference/storage/composite) - Combine multiple storage providers
|
|
209
|
-
- [Storage Configuration](https://mastra.ai/docs/memory/storage)
|