@mastra/pg 0.0.0-error-handler-fix-20251020202607 → 0.0.0-esbuild-bundle-worker-20260807182016
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5579 -3
- package/LICENSE.md +15 -0
- package/README.md +72 -20
- package/dist/docs/SKILL.md +45 -0
- package/dist/docs/assets/SOURCE_MAP.json +6 -0
- package/dist/docs/references/docs-deployment-workers.md +137 -0
- package/dist/docs/references/docs-memory-semantic-recall.md +401 -0
- package/dist/docs/references/docs-memory-working-memory.md +431 -0
- package/dist/docs/references/docs-storage-overview.md +214 -0
- package/dist/docs/references/guides-rag-overview.md +74 -0
- package/dist/docs/references/guides-rag-retrieval.md +520 -0
- package/dist/docs/references/guides-rag-vector-databases.md +669 -0
- package/dist/docs/references/reference-memory-memory-class.md +148 -0
- package/dist/docs/references/reference-processors-message-history-processor.md +86 -0
- package/dist/docs/references/reference-processors-semantic-recall-processor.md +118 -0
- package/dist/docs/references/reference-processors-working-memory-processor.md +151 -0
- package/dist/docs/references/reference-rag-metadata-filters.md +218 -0
- package/dist/docs/references/reference-storage-composite.md +397 -0
- package/dist/docs/references/reference-storage-dynamodb.md +284 -0
- package/dist/docs/references/reference-storage-postgresql.md +531 -0
- package/dist/docs/references/reference-storage-retention.md +248 -0
- package/dist/docs/references/reference-tools-vector-query-tool.md +495 -0
- package/dist/docs/references/reference-vectors-pg.md +430 -0
- package/dist/index.cjs +20692 -4338
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +20640 -4310
- package/dist/index.js.map +1 -1
- package/dist/shared/config.d.ts +129 -31
- package/dist/shared/config.d.ts.map +1 -1
- package/dist/shared/pool-config.d.ts +32 -0
- package/dist/shared/pool-config.d.ts.map +1 -0
- package/dist/storage/client.d.ts +165 -0
- package/dist/storage/client.d.ts.map +1 -0
- package/dist/storage/db/constraint-utils.d.ts +16 -0
- package/dist/storage/db/constraint-utils.d.ts.map +1 -0
- package/dist/storage/db/index.d.ts +364 -0
- package/dist/storage/db/index.d.ts.map +1 -0
- package/dist/storage/db/pg-errors.d.ts +24 -0
- package/dist/storage/db/pg-errors.d.ts.map +1 -0
- package/dist/storage/db/schema-snapshot.d.ts +71 -0
- package/dist/storage/db/schema-snapshot.d.ts.map +1 -0
- package/dist/storage/domains/agents/index.d.ts +51 -0
- package/dist/storage/domains/agents/index.d.ts.map +1 -0
- package/dist/storage/domains/background-tasks/index.d.ts +43 -0
- package/dist/storage/domains/background-tasks/index.d.ts.map +1 -0
- package/dist/storage/domains/blobs/index.d.ts +18 -0
- package/dist/storage/domains/blobs/index.d.ts.map +1 -0
- package/dist/storage/domains/channels/index.d.ts +25 -0
- package/dist/storage/domains/channels/index.d.ts.map +1 -0
- package/dist/storage/domains/datasets/index.d.ts +47 -0
- package/dist/storage/domains/datasets/index.d.ts.map +1 -0
- package/dist/storage/domains/experiments/index.d.ts +70 -0
- package/dist/storage/domains/experiments/index.d.ts.map +1 -0
- package/dist/storage/domains/favorites/index.d.ts +19 -0
- package/dist/storage/domains/favorites/index.d.ts.map +1 -0
- package/dist/storage/domains/mcp-clients/index.d.ts +34 -0
- package/dist/storage/domains/mcp-clients/index.d.ts.map +1 -0
- package/dist/storage/domains/mcp-servers/index.d.ts +34 -0
- package/dist/storage/domains/mcp-servers/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +130 -56
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/notifications/index.d.ts +43 -0
- package/dist/storage/domains/notifications/index.d.ts.map +1 -0
- package/dist/storage/domains/observability/index.d.ts +87 -39
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +55 -0
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/discovery.d.ts +52 -0
- package/dist/storage/domains/observability/v-next/discovery.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/feedback.d.ts +17 -0
- package/dist/storage/domains/observability/v-next/feedback.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/filters.d.ts +29 -0
- package/dist/storage/domains/observability/v-next/filters.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/helpers.d.ts +33 -0
- package/dist/storage/domains/observability/v-next/helpers.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/index.d.ts +120 -0
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/listing.d.ts +53 -0
- package/dist/storage/domains/observability/v-next/listing.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/logs.d.ts +8 -0
- package/dist/storage/domains/observability/v-next/logs.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/metrics.d.ts +19 -0
- package/dist/storage/domains/observability/v-next/metrics.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/olap.d.ts +93 -0
- package/dist/storage/domains/observability/v-next/olap.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/partitioning.d.ts +76 -0
- package/dist/storage/domains/observability/v-next/partitioning.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/pg-errors.d.ts +5 -0
- package/dist/storage/domains/observability/v-next/pg-errors.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/polling.d.ts +28 -0
- package/dist/storage/domains/observability/v-next/polling.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/retention.d.ts +64 -0
- package/dist/storage/domains/observability/v-next/retention.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/scores.d.ts +17 -0
- package/dist/storage/domains/observability/v-next/scores.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/signal-schema.d.ts +776 -0
- package/dist/storage/domains/observability/v-next/signal-schema.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/sql.d.ts +31 -0
- package/dist/storage/domains/observability/v-next/sql.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/traces.d.ts +16 -0
- package/dist/storage/domains/observability/v-next/traces.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/tracing.d.ts +19 -0
- package/dist/storage/domains/observability/v-next/tracing.d.ts.map +1 -0
- package/dist/storage/domains/prompt-blocks/index.d.ts +42 -0
- package/dist/storage/domains/prompt-blocks/index.d.ts.map +1 -0
- package/dist/storage/domains/schedules/index.d.ts +52 -0
- package/dist/storage/domains/schedules/index.d.ts.map +1 -0
- package/dist/storage/domains/scorer-definitions/index.d.ts +42 -0
- package/dist/storage/domains/scorer-definitions/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +61 -33
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/skills/index.d.ts +34 -0
- package/dist/storage/domains/skills/index.d.ts.map +1 -0
- package/dist/storage/domains/tool-provider-connections/index.d.ts +20 -0
- package/dist/storage/domains/tool-provider-connections/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +38 -5
- package/dist/storage/domains/utils.d.ts.map +1 -1
- package/dist/storage/domains/workflow-definitions/index.d.ts +20 -0
- package/dist/storage/domains/workflow-definitions/index.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +70 -30
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/domains/workspaces/index.d.ts +34 -0
- package/dist/storage/domains/workspaces/index.d.ts.map +1 -0
- package/dist/storage/factory-storage.d.ts +35 -0
- package/dist/storage/factory-storage.d.ts.map +1 -0
- package/dist/storage/index.d.ts +159 -253
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/performance-indexes/performance-test.d.ts +3 -1
- package/dist/storage/performance-indexes/performance-test.d.ts.map +1 -1
- package/dist/storage/retention.d.ts +76 -0
- package/dist/storage/retention.d.ts.map +1 -0
- package/dist/storage/test-utils.d.ts.map +1 -1
- package/dist/vector/index.d.ts +91 -11
- package/dist/vector/index.d.ts.map +1 -1
- package/dist/vector/sql-builder.d.ts +4 -0
- package/dist/vector/sql-builder.d.ts.map +1 -1
- package/dist/vector/types.d.ts +40 -0
- package/dist/vector/types.d.ts.map +1 -1
- package/package.json +31 -18
- package/dist/storage/domains/legacy-evals/index.d.ts +0 -20
- package/dist/storage/domains/legacy-evals/index.d.ts.map +0 -1
- package/dist/storage/domains/operations/index.d.ts +0 -118
- package/dist/storage/domains/operations/index.d.ts.map +0 -1
- package/dist/storage/domains/traces/index.d.ts +0 -23
- package/dist/storage/domains/traces/index.d.ts.map +0 -1
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
|
|
2
|
+
|
|
3
|
+
# Metadata filters
|
|
4
|
+
|
|
5
|
+
Mastra provides a unified metadata filtering syntax across all vector stores, based on MongoDB/Sift query syntax. Each vector store translates these filters into their native format.
|
|
6
|
+
|
|
7
|
+
## Basic example
|
|
8
|
+
|
|
9
|
+
```typescript
|
|
10
|
+
import { PgVector } from '@mastra/pg'
|
|
11
|
+
|
|
12
|
+
const store = new PgVector({
|
|
13
|
+
id: 'pg-vector',
|
|
14
|
+
connectionString,
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
const results = await store.query({
|
|
18
|
+
indexName: 'my_index',
|
|
19
|
+
queryVector: queryVector,
|
|
20
|
+
topK: 10,
|
|
21
|
+
filter: {
|
|
22
|
+
category: 'electronics', // Simple equality
|
|
23
|
+
price: { $gt: 100 }, // Numeric comparison
|
|
24
|
+
tags: { $in: ['sale', 'new'] }, // Array membership
|
|
25
|
+
},
|
|
26
|
+
})
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Supported operators
|
|
30
|
+
|
|
31
|
+
### Basic Comparison
|
|
32
|
+
|
|
33
|
+
`$eq`Matches values equal to specified value{ age: { $eq: 25 } }Supported by: All except Couchbase`$ne`Matches values not equal{ status: { $ne: 'inactive' } }Supported by: All except Couchbase`$gt`Greater than{ price: { $gt: 100 } }Supported by: All except Couchbase`$gte`Greater than or equal{ rating: { $gte: 4.5 } }Supported by: All except Couchbase`$lt`Less than{ stock: { $lt: 20 } }Supported by: All except Couchbase`$lte`Less than or equal{ priority: { $lte: 3 } }Supported by: All except Couchbase
|
|
34
|
+
|
|
35
|
+
### Array Operators
|
|
36
|
+
|
|
37
|
+
`$in`Matches any value in array{ category: { $in: \["A", "B"] } }Supported by: All except Couchbase`$nin`Matches none of the values{ status: { $nin: \["deleted", "archived"] } }Supported by: All except Couchbase`$all`Matches arrays containing all elements{ tags: { $all: \["urgent", "high"] } }Supported by: Astra, Pinecone, Upstash, MongoDB`$elemMatch`Matches array elements meeting criteria{ scores: { $elemMatch: { $gt: 80 } } }Supported by: libSQL, PgVector, MongoDB
|
|
38
|
+
|
|
39
|
+
### Logical Operators
|
|
40
|
+
|
|
41
|
+
`$and`Logical AND{ $and: \[{ price: { $gt: 100 } }, { stock: { $gt: 0 } }] }Supported by: All except Vectorize, Couchbase`$or`Logical OR{ $or: \[{ status: "active" }, { priority: "high" }] }Supported by: All except Vectorize, Couchbase`$not`Logical NOT{ price: { $not: { $lt: 100 } } }Supported by: Astra, Qdrant, Upstash, PgVector, libSQL, MongoDB`$nor`Logical NOR{ $nor: \[{ status: "deleted" }, { archived: true }] }Supported by: Qdrant, Upstash, PgVector, libSQL, MongoDB
|
|
42
|
+
|
|
43
|
+
### Element Operators
|
|
44
|
+
|
|
45
|
+
`$exists`Matches documents with field{ rating: { $exists: true } }Supported by: All except Vectorize, Chroma, Couchbase
|
|
46
|
+
|
|
47
|
+
### Custom Operators
|
|
48
|
+
|
|
49
|
+
`$contains`Text contains substring{ description: { $contains: "sale" } }Supported by: Upstash, libSQL, PgVector`$regex`Regular expression match{ name: { $regex: "^test" } }Supported by: Qdrant, PgVector, Upstash, MongoDB`$size`Array length check{ tags: { $size: { $gt: 2 } } }Supported by: Astra, libSQL, PgVector, MongoDB`$geo`Geospatial query{ location: { $geo: { type: "radius", ... } } }Supported by: Qdrant`$datetime`Datetime range query{ created: { $datetime: { range: { gt: "2024-01-01" } } } }Supported by: Qdrant`$hasId`Vector ID existence check{ $hasId: \["id1", "id2"] }Supported by: Qdrant`$hasVector`Vector existence check{ $hasVector: true }Supported by: Qdrant
|
|
50
|
+
|
|
51
|
+
## Common rules and restrictions
|
|
52
|
+
|
|
53
|
+
1. Field names can't:
|
|
54
|
+
|
|
55
|
+
- Contain dots (.) unless referring to nested fields
|
|
56
|
+
- Start with $ or contain null characters
|
|
57
|
+
- Be empty strings
|
|
58
|
+
|
|
59
|
+
2. Values must be:
|
|
60
|
+
|
|
61
|
+
- Valid JSON types (string, number, boolean, object, array)
|
|
62
|
+
- Not undefined
|
|
63
|
+
- Properly typed for the operator (e.g., numbers for numeric comparisons)
|
|
64
|
+
|
|
65
|
+
3. Logical operators:
|
|
66
|
+
|
|
67
|
+
- Must contain valid conditions
|
|
68
|
+
- Can't be empty
|
|
69
|
+
- Must be properly nested
|
|
70
|
+
- Can only be used at top level or nested within other logical operators
|
|
71
|
+
- Can't be used at field level or nested inside a field
|
|
72
|
+
- Can't be used inside an operator
|
|
73
|
+
- Valid: `{ "$and": [{ "field": { "$gt": 100 } }] }`
|
|
74
|
+
- Valid: `{ "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }`
|
|
75
|
+
- Invalid: `{ "field": { "$and": [{ "$gt": 100 }] } }`
|
|
76
|
+
- Invalid: `{ "field": { "$gt": { "$and": [{...}] } } }`
|
|
77
|
+
|
|
78
|
+
4. $not operator:
|
|
79
|
+
|
|
80
|
+
- Must be an object
|
|
81
|
+
- Can't be empty
|
|
82
|
+
- Can be used at field level or top level
|
|
83
|
+
- Valid: `{ "$not": { "field": "value" } }`
|
|
84
|
+
- Valid: `{ "field": { "$not": { "$eq": "value" } } }`
|
|
85
|
+
|
|
86
|
+
5. Operator nesting:
|
|
87
|
+
|
|
88
|
+
- Logical operators must contain field conditions, not direct operators
|
|
89
|
+
- Valid: `{ "$and": [{ "field": { "$gt": 100 } }] }`
|
|
90
|
+
- Invalid: `{ "$and": [{ "$gt": 100 }] }`
|
|
91
|
+
|
|
92
|
+
## Store-specific notes
|
|
93
|
+
|
|
94
|
+
### Astra
|
|
95
|
+
|
|
96
|
+
- Nested field queries are supported using dot notation
|
|
97
|
+
- Array fields must be explicitly defined as arrays in the metadata
|
|
98
|
+
- Metadata values are case-sensitive
|
|
99
|
+
|
|
100
|
+
### ChromaDB
|
|
101
|
+
|
|
102
|
+
- Where filters only return results where the filtered field exists in metadata
|
|
103
|
+
- Empty metadata fields aren't included in filter results
|
|
104
|
+
- Metadata fields must be present for negative matches (e.g., $ne won't match documents missing the field)
|
|
105
|
+
|
|
106
|
+
### Cloudflare Vectorize
|
|
107
|
+
|
|
108
|
+
- Requires explicit metadata indexing before filtering can be used
|
|
109
|
+
- Use `createMetadataIndex()` to index fields you want to filter on
|
|
110
|
+
- Up to 10 metadata indexes per Vectorize index
|
|
111
|
+
- String values are indexed up to first 64 bytes (truncated on UTF-8 boundaries)
|
|
112
|
+
- Number values use float64 precision
|
|
113
|
+
- Filter JSON must be under 2048 bytes
|
|
114
|
+
- Field names can't contain dots (.) or start with $
|
|
115
|
+
- Field names limited to 512 characters
|
|
116
|
+
- Vectors must be re-upserted after creating new metadata indexes to be included in filtered results
|
|
117
|
+
- Range queries may have reduced accuracy with very large datasets (\~10M+ vectors)
|
|
118
|
+
|
|
119
|
+
### libSQL
|
|
120
|
+
|
|
121
|
+
- Supports nested object queries with dot notation
|
|
122
|
+
- Array fields are validated to ensure they contain valid JSON arrays
|
|
123
|
+
- Numeric comparisons maintain proper type handling
|
|
124
|
+
- Empty arrays in conditions are handled gracefully
|
|
125
|
+
- Metadata is stored in a JSONB column for efficient querying
|
|
126
|
+
|
|
127
|
+
### PgVector
|
|
128
|
+
|
|
129
|
+
- Full support for PostgreSQL's native JSON querying capabilities
|
|
130
|
+
- Efficient handling of array operations using native array functions
|
|
131
|
+
- Proper type handling for numbers, strings, and booleans
|
|
132
|
+
- Nested field queries use PostgreSQL's JSON path syntax internally
|
|
133
|
+
- Metadata is stored in a JSONB column for efficient indexing
|
|
134
|
+
|
|
135
|
+
### Pinecone
|
|
136
|
+
|
|
137
|
+
- Metadata field names are limited to 512 characters
|
|
138
|
+
- Numeric values must be within the range of ±1e38
|
|
139
|
+
- Arrays in metadata are limited to 64KB total size
|
|
140
|
+
- Nested objects are flattened with dot notation
|
|
141
|
+
- Metadata updates replace the entire metadata object
|
|
142
|
+
|
|
143
|
+
### Qdrant
|
|
144
|
+
|
|
145
|
+
- Supports advanced filtering with nested conditions
|
|
146
|
+
- Payload (metadata) fields must be explicitly indexed for filtering
|
|
147
|
+
- Use `createPayloadIndex()` to index fields you want to filter on:
|
|
148
|
+
|
|
149
|
+
```typescript
|
|
150
|
+
// Index a field before filtering on it
|
|
151
|
+
await store.createPayloadIndex({
|
|
152
|
+
indexName: 'my_index',
|
|
153
|
+
fieldName: 'source',
|
|
154
|
+
fieldSchema: 'keyword', // 'keyword' | 'integer' | 'float' | 'geo' | 'text' | 'bool' | 'datetime' | 'uuid'
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
// Now filtering works
|
|
158
|
+
const results = await store.query({
|
|
159
|
+
indexName: 'my_index',
|
|
160
|
+
queryVector: queryVector,
|
|
161
|
+
filter: { source: 'document-a' },
|
|
162
|
+
})
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
- Efficient handling of geo-spatial queries
|
|
166
|
+
- Special handling for null and empty values
|
|
167
|
+
- Vector-specific filtering capabilities
|
|
168
|
+
- Datetime values must be in RFC 3339 format
|
|
169
|
+
|
|
170
|
+
### Upstash
|
|
171
|
+
|
|
172
|
+
- 512-character limit for metadata field keys
|
|
173
|
+
- Query size is limited (avoid large IN clauses)
|
|
174
|
+
- No support for null/undefined values in filters
|
|
175
|
+
- Translates to SQL-like syntax internally
|
|
176
|
+
- Case-sensitive string comparisons
|
|
177
|
+
- Metadata updates are atomic
|
|
178
|
+
|
|
179
|
+
### MongoDB
|
|
180
|
+
|
|
181
|
+
- Full support for MongoDB/Sift query syntax for metadata filters
|
|
182
|
+
- Supports all standard comparison, array, logical, and element operators
|
|
183
|
+
- Supports nested fields and arrays in metadata
|
|
184
|
+
- Filtering can be applied to both `metadata` and the original document content using the `filter` and `documentFilter` options, respectively
|
|
185
|
+
- `filter` applies to the metadata object; `documentFilter` applies to the original document fields
|
|
186
|
+
- No artificial limits on filter size or complexity (subject to MongoDB query limits)
|
|
187
|
+
- Indexing metadata fields is recommended for optimal performance
|
|
188
|
+
|
|
189
|
+
### Couchbase
|
|
190
|
+
|
|
191
|
+
- Currently doesn't have support for metadata filters. Filtering must be done client-side after retrieving results or by using the Couchbase SDK's Search capabilities directly for more complex queries.
|
|
192
|
+
|
|
193
|
+
### Amazon S3 Vectors
|
|
194
|
+
|
|
195
|
+
- Equality values must be primitives (string/number/boolean). `null`/`undefined`, arrays, objects, and Date aren't allowed for equality. Range operators accept numbers or Date (Dates are normalized to epoch ms).
|
|
196
|
+
- `$in`/`$nin` require **non-empty arrays of primitives**; Date elements are allowed and normalized to epoch ms. **Array equality** isn't supported.
|
|
197
|
+
- Implicit AND is canonicalized (`{a:1,b:2}` → `{$and:[{a:1},{b:2}]`). Logical operators must contain field conditions and use non-empty arrays. They may appear only at the root or within other logical operators (not inside field values).
|
|
198
|
+
- Keys listed in `nonFilterableMetadataKeys` at index creation are stored but not filterable. This setting is immutable.
|
|
199
|
+
- $exists requires a boolean value.
|
|
200
|
+
- undefined/null/empty filters are treated as no filter.
|
|
201
|
+
- Each metadata key name limited to 63 characters.
|
|
202
|
+
- Total metadata per vector: Up to 40 KB (filterable + non-filterable)
|
|
203
|
+
- Total metadata keys per vector: Up to 10
|
|
204
|
+
- Filterable metadata per vector: Up to 2 KB
|
|
205
|
+
- Non-filterable metadata keys per vector index: Up to 10
|
|
206
|
+
|
|
207
|
+
## Related
|
|
208
|
+
|
|
209
|
+
- [Astra](https://mastra.ai/reference/vectors/astra)
|
|
210
|
+
- [Chroma](https://mastra.ai/reference/vectors/chroma)
|
|
211
|
+
- [Cloudflare Vectorize](https://mastra.ai/reference/vectors/vectorize)
|
|
212
|
+
- [libSQL](https://mastra.ai/reference/vectors/libsql)
|
|
213
|
+
- [MongoDB](https://mastra.ai/reference/vectors/mongodb)
|
|
214
|
+
- [PgStore](https://mastra.ai/reference/vectors/pg)
|
|
215
|
+
- [Pinecone](https://mastra.ai/reference/vectors/pinecone)
|
|
216
|
+
- [Qdrant](https://mastra.ai/reference/vectors/qdrant)
|
|
217
|
+
- [Upstash](https://mastra.ai/reference/vectors/upstash)
|
|
218
|
+
- [Amazon S3 Vectors](https://mastra.ai/reference/vectors/s3vectors)
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
|
|
2
|
+
|
|
3
|
+
# Composite storage
|
|
4
|
+
|
|
5
|
+
`MastraCompositeStore` can compose storage domains from different providers. Use it when you need different databases for different purposes. For example, use LibSQL for memory and PostgreSQL for workflows.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
`MastraCompositeStore` is included in `@mastra/core`:
|
|
10
|
+
|
|
11
|
+
**npm**:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @mastra/core@latest
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
**pnpm**:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pnpm add @mastra/core@latest
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
**Yarn**:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
yarn add @mastra/core@latest
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
**Bun**:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
bun add @mastra/core@latest
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
You'll also need to install the storage providers you want to compose:
|
|
36
|
+
|
|
37
|
+
**npm**:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
npm install @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
**pnpm**:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pnpm add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
**Yarn**:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
yarn add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**Bun**:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
bun add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Storage domains
|
|
62
|
+
|
|
63
|
+
Mastra organizes storage into domains, each handling a specific type of data. Each domain can be backed by a different storage adapter, and domain classes are exported from each storage package.
|
|
64
|
+
|
|
65
|
+
| Domain | Description |
|
|
66
|
+
| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
67
|
+
| `memory` | Conversation persistence for agents. Stores threads (conversation sessions), messages, resources (user identities), and working memory (persistent context across conversations). |
|
|
68
|
+
| `workflows` | Workflow execution state. When workflows suspend for human input, external events, or scheduled resumption, their state is persisted here to enable resumption after server restarts. |
|
|
69
|
+
| `scores` | Evaluation results from Mastra's evals system. Scores and metrics are persisted here for analysis and comparison over time. |
|
|
70
|
+
| `observability` | Telemetry data including traces and spans. Agent interactions, tool calls, and LLM requests generate spans collected into traces for debugging and performance analysis. |
|
|
71
|
+
| `agents` | Agent configurations for stored agents. Enables agents to be defined and updated at runtime without code deployments. |
|
|
72
|
+
| `datasets` | Evaluation datasets used for experiment runs. Stores dataset definitions, schemas, and versioned items. |
|
|
73
|
+
| `experiments` | Experiment runs and per-item experiment results linked to datasets and targets. |
|
|
74
|
+
|
|
75
|
+
> **Note:** `MastraCompositeStore` accepts all of the domain keys above, but storage adapter support varies by package. You can mix adapters per domain, but only for domains implemented and exported by those adapters. For example, `memory: new MemoryLibSQL(...)` and `workflows: new WorkflowsPG(...)` is valid because both packages export those domain classes.
|
|
76
|
+
|
|
77
|
+
## Usage
|
|
78
|
+
|
|
79
|
+
### Basic composition
|
|
80
|
+
|
|
81
|
+
Import domain classes directly from each store package and compose them:
|
|
82
|
+
|
|
83
|
+
```typescript
|
|
84
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
85
|
+
import { WorkflowsPG, ScoresPG } from '@mastra/pg'
|
|
86
|
+
import { MemoryLibSQL } from '@mastra/libsql'
|
|
87
|
+
import { Mastra } from '@mastra/core'
|
|
88
|
+
|
|
89
|
+
export const mastra = new Mastra({
|
|
90
|
+
storage: new MastraCompositeStore({
|
|
91
|
+
id: 'composite',
|
|
92
|
+
domains: {
|
|
93
|
+
memory: new MemoryLibSQL({ url: 'file:./local.db' }),
|
|
94
|
+
workflows: new WorkflowsPG({ connectionString: process.env.DATABASE_URL }),
|
|
95
|
+
scores: new ScoresPG({ connectionString: process.env.DATABASE_URL }),
|
|
96
|
+
},
|
|
97
|
+
}),
|
|
98
|
+
})
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### With a default storage
|
|
102
|
+
|
|
103
|
+
Use `default` to specify a fallback storage, then override specific domains:
|
|
104
|
+
|
|
105
|
+
```typescript
|
|
106
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
107
|
+
import { PostgresStore } from '@mastra/pg'
|
|
108
|
+
import { MemoryLibSQL } from '@mastra/libsql'
|
|
109
|
+
import { Mastra } from '@mastra/core'
|
|
110
|
+
|
|
111
|
+
const pgStore = new PostgresStore({
|
|
112
|
+
id: 'pg',
|
|
113
|
+
connectionString: process.env.DATABASE_URL,
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
export const mastra = new Mastra({
|
|
117
|
+
storage: new MastraCompositeStore({
|
|
118
|
+
id: 'composite',
|
|
119
|
+
default: pgStore,
|
|
120
|
+
domains: {
|
|
121
|
+
memory: new MemoryLibSQL({ url: 'file:./local.db' }),
|
|
122
|
+
},
|
|
123
|
+
}),
|
|
124
|
+
})
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Mixed backends
|
|
128
|
+
|
|
129
|
+
Use domain classes from each storage package to route different domains to different backends. The following example stores memory and workflow state in MongoDB, then routes observability to ClickHouse:
|
|
130
|
+
|
|
131
|
+
```typescript
|
|
132
|
+
import { Mastra } from '@mastra/core'
|
|
133
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
134
|
+
import { ObservabilityStorageClickhouse } from '@mastra/clickhouse'
|
|
135
|
+
import { MemoryStorageMongoDB, WorkflowsStorageMongoDB } from '@mastra/mongodb'
|
|
136
|
+
|
|
137
|
+
export const mastra = new Mastra({
|
|
138
|
+
storage: new MastraCompositeStore({
|
|
139
|
+
id: 'composite',
|
|
140
|
+
domains: {
|
|
141
|
+
memory: new MemoryStorageMongoDB({
|
|
142
|
+
uri: process.env.MONGODB_URI,
|
|
143
|
+
dbName: 'mastra_memory',
|
|
144
|
+
}),
|
|
145
|
+
workflows: new WorkflowsStorageMongoDB({
|
|
146
|
+
uri: process.env.MONGODB_URI,
|
|
147
|
+
dbName: 'mastra_workflows',
|
|
148
|
+
}),
|
|
149
|
+
observability: new ObservabilityStorageClickhouse({
|
|
150
|
+
url: process.env.CLICKHOUSE_URL,
|
|
151
|
+
username: process.env.CLICKHOUSE_USERNAME,
|
|
152
|
+
password: process.env.CLICKHOUSE_PASSWORD,
|
|
153
|
+
}),
|
|
154
|
+
},
|
|
155
|
+
}),
|
|
156
|
+
})
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### Disabling a domain
|
|
160
|
+
|
|
161
|
+
Set a domain to `false` to disable it. A disabled domain doesn't fall back to `default`, so data for that domain isn't persisted:
|
|
162
|
+
|
|
163
|
+
```typescript
|
|
164
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
165
|
+
import { PostgresStore } from '@mastra/pg'
|
|
166
|
+
import { Mastra } from '@mastra/core'
|
|
167
|
+
|
|
168
|
+
const pgStore = new PostgresStore({
|
|
169
|
+
id: 'pg',
|
|
170
|
+
connectionString: process.env.DATABASE_URL,
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
export const mastra = new Mastra({
|
|
174
|
+
storage: new MastraCompositeStore({
|
|
175
|
+
id: 'composite',
|
|
176
|
+
default: pgStore,
|
|
177
|
+
domains: {
|
|
178
|
+
// don't persist traces and spans
|
|
179
|
+
observability: false,
|
|
180
|
+
},
|
|
181
|
+
}),
|
|
182
|
+
})
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Options
|
|
186
|
+
|
|
187
|
+
**id** (`string`): Unique identifier for this storage instance.
|
|
188
|
+
|
|
189
|
+
**default** (`MastraCompositeStore`): Default storage adapter. Domains not explicitly specified in domains will use this storage's domains as fallbacks.
|
|
190
|
+
|
|
191
|
+
**editor** (`MastraCompositeStore`): Storage adapter for Editor-owned domains, including agents, prompt blocks, scorers, MCP clients and servers, workspaces, and skills. Takes precedence over default storage but not explicit domain overrides.
|
|
192
|
+
|
|
193
|
+
**disableInit** (`boolean`): When true, automatic initialization is disabled. You must call init() explicitly.
|
|
194
|
+
|
|
195
|
+
**domains** (`object`): Individual domain overrides. Each domain can come from a different storage adapter. These take precedence over both editor and default storage. Set a domain to false to disable it entirely; a disabled domain does not fall back to editor or default.
|
|
196
|
+
|
|
197
|
+
**domains.memory** (`MemoryStorage`): Storage for threads, messages, and resources.
|
|
198
|
+
|
|
199
|
+
**domains.workflows** (`WorkflowsStorage`): Storage for workflow snapshots.
|
|
200
|
+
|
|
201
|
+
**domains.scores** (`ScoresStorage`): Storage for evaluation scores.
|
|
202
|
+
|
|
203
|
+
**domains.observability** (`ObservabilityStorage`): Storage for traces and spans.
|
|
204
|
+
|
|
205
|
+
**domains.agents** (`AgentsStorage`): Storage for stored agent configurations.
|
|
206
|
+
|
|
207
|
+
**domains.datasets** (`DatasetsStorage`): Storage for dataset metadata, dataset items, and dataset versions.
|
|
208
|
+
|
|
209
|
+
**domains.experiments** (`ExperimentsStorage`): Storage for experiment runs and per-item experiment results.
|
|
210
|
+
|
|
211
|
+
## Initialization
|
|
212
|
+
|
|
213
|
+
`MastraCompositeStore` initializes each configured domain independently. When passed to the Mastra class, `init()` is called automatically:
|
|
214
|
+
|
|
215
|
+
```typescript
|
|
216
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
217
|
+
import { MemoryPG, WorkflowsPG, ScoresPG } from '@mastra/pg'
|
|
218
|
+
import { Mastra } from '@mastra/core'
|
|
219
|
+
|
|
220
|
+
const storage = new MastraCompositeStore({
|
|
221
|
+
id: 'composite',
|
|
222
|
+
domains: {
|
|
223
|
+
memory: new MemoryPG({ connectionString: process.env.DATABASE_URL }),
|
|
224
|
+
workflows: new WorkflowsPG({ connectionString: process.env.DATABASE_URL }),
|
|
225
|
+
scores: new ScoresPG({ connectionString: process.env.DATABASE_URL }),
|
|
226
|
+
},
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
export const mastra = new Mastra({
|
|
230
|
+
storage, // init() called automatically
|
|
231
|
+
})
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
If using storage directly, call `init()` explicitly:
|
|
235
|
+
|
|
236
|
+
```typescript
|
|
237
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
238
|
+
import { MemoryPG } from '@mastra/pg'
|
|
239
|
+
|
|
240
|
+
const storage = new MastraCompositeStore({
|
|
241
|
+
id: 'composite',
|
|
242
|
+
domains: {
|
|
243
|
+
memory: new MemoryPG({ connectionString: process.env.DATABASE_URL }),
|
|
244
|
+
},
|
|
245
|
+
})
|
|
246
|
+
|
|
247
|
+
await storage.init()
|
|
248
|
+
|
|
249
|
+
// Access domain-specific stores via getStore()
|
|
250
|
+
const memoryStore = await storage.getStore('memory')
|
|
251
|
+
const thread = await memoryStore?.getThreadById({ threadId: '...' })
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
## Closing connections
|
|
255
|
+
|
|
256
|
+
`close()` releases the connections of the stores a composite was built from: the `default` and `editor` stores, plus any domain that owns its own client. Each store is closed once, even when it backs several domains. When passed to the Mastra class, `close()` is called by `shutdown()`:
|
|
257
|
+
|
|
258
|
+
```typescript
|
|
259
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
260
|
+
import { PostgresStore } from '@mastra/pg'
|
|
261
|
+
import { Mastra } from '@mastra/core'
|
|
262
|
+
|
|
263
|
+
const pgStore = new PostgresStore({
|
|
264
|
+
id: 'pg-storage',
|
|
265
|
+
connectionString: process.env.DATABASE_URL,
|
|
266
|
+
})
|
|
267
|
+
|
|
268
|
+
export const mastra = new Mastra({
|
|
269
|
+
storage: new MastraCompositeStore({ id: 'composite', default: pgStore }),
|
|
270
|
+
})
|
|
271
|
+
|
|
272
|
+
process.on('SIGTERM', async () => {
|
|
273
|
+
// Releases the Postgres pool, so the process can exit
|
|
274
|
+
await mastra.shutdown()
|
|
275
|
+
})
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
A store you construct only to supply a domain isn't reachable through the composite. Keep a reference to it and close it yourself:
|
|
279
|
+
|
|
280
|
+
```typescript
|
|
281
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
282
|
+
import { ClickhouseStore } from '@mastra/clickhouse'
|
|
283
|
+
import { PostgresStore } from '@mastra/pg'
|
|
284
|
+
import { Mastra } from '@mastra/core'
|
|
285
|
+
|
|
286
|
+
const pgStore = new PostgresStore({
|
|
287
|
+
id: 'pg-storage',
|
|
288
|
+
connectionString: process.env.DATABASE_URL,
|
|
289
|
+
})
|
|
290
|
+
|
|
291
|
+
const clickhouseStore = new ClickhouseStore({
|
|
292
|
+
id: 'clickhouse-storage',
|
|
293
|
+
url: process.env.CLICKHOUSE_URL,
|
|
294
|
+
username: process.env.CLICKHOUSE_USERNAME,
|
|
295
|
+
password: process.env.CLICKHOUSE_PASSWORD,
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
export const mastra = new Mastra({
|
|
299
|
+
storage: new MastraCompositeStore({
|
|
300
|
+
id: 'composite',
|
|
301
|
+
default: pgStore,
|
|
302
|
+
domains: { observability: clickhouseStore.stores?.observability },
|
|
303
|
+
}),
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
process.on('SIGTERM', async () => {
|
|
307
|
+
await mastra.shutdown()
|
|
308
|
+
await clickhouseStore.close()
|
|
309
|
+
})
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
## Use cases
|
|
313
|
+
|
|
314
|
+
### Separate databases for different workloads
|
|
315
|
+
|
|
316
|
+
Use a local database for development while keeping production data in a managed service:
|
|
317
|
+
|
|
318
|
+
```typescript
|
|
319
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
320
|
+
import { MemoryPG, WorkflowsPG, ScoresPG } from '@mastra/pg'
|
|
321
|
+
import { MemoryLibSQL } from '@mastra/libsql'
|
|
322
|
+
|
|
323
|
+
const storage = new MastraCompositeStore({
|
|
324
|
+
id: 'composite',
|
|
325
|
+
domains: {
|
|
326
|
+
// Use local SQLite for development, PostgreSQL for production
|
|
327
|
+
memory:
|
|
328
|
+
process.env.NODE_ENV === 'development'
|
|
329
|
+
? new MemoryLibSQL({ url: 'file:./dev.db' })
|
|
330
|
+
: new MemoryPG({ connectionString: process.env.DATABASE_URL }),
|
|
331
|
+
workflows: new WorkflowsPG({ connectionString: process.env.DATABASE_URL }),
|
|
332
|
+
scores: new ScoresPG({ connectionString: process.env.DATABASE_URL }),
|
|
333
|
+
},
|
|
334
|
+
})
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
### Specialized storage for observability
|
|
338
|
+
|
|
339
|
+
Observability data can quickly overwhelm general-purpose databases in production. A single agent interaction can generate hundreds of spans, and high-traffic applications can produce thousands of traces per day.
|
|
340
|
+
|
|
341
|
+
**[ClickHouse](https://mastra.ai/reference/storage/clickhouse)** is recommended for production observability because it's optimized for high-volume, write-heavy analytics workloads. Use composite storage to route observability to ClickHouse while keeping other data in your primary database:
|
|
342
|
+
|
|
343
|
+
```typescript
|
|
344
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
345
|
+
import { MemoryPG, WorkflowsPG, ScoresPG } from '@mastra/pg'
|
|
346
|
+
import { ObservabilityStorageClickhouseVNext } from '@mastra/clickhouse'
|
|
347
|
+
|
|
348
|
+
const storage = new MastraCompositeStore({
|
|
349
|
+
id: 'composite',
|
|
350
|
+
domains: {
|
|
351
|
+
memory: new MemoryPG({ connectionString: process.env.DATABASE_URL }),
|
|
352
|
+
workflows: new WorkflowsPG({ connectionString: process.env.DATABASE_URL }),
|
|
353
|
+
scores: new ScoresPG({ connectionString: process.env.DATABASE_URL }),
|
|
354
|
+
observability: new ObservabilityStorageClickhouseVNext({
|
|
355
|
+
url: process.env.CLICKHOUSE_URL,
|
|
356
|
+
username: process.env.CLICKHOUSE_USERNAME,
|
|
357
|
+
password: process.env.CLICKHOUSE_PASSWORD,
|
|
358
|
+
}),
|
|
359
|
+
},
|
|
360
|
+
})
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
> **Note:** `ObservabilityStorageClickhouseVNext` is the current observability domain implementation. The legacy `ObservabilityStorageClickhouse` class is also exported and remains supported for projects that haven't migrated. See the [ClickHouse storage reference](https://mastra.ai/reference/storage/clickhouse) for details.
|
|
364
|
+
|
|
365
|
+
### Replicated ClickHouse for multi-replica clusters
|
|
366
|
+
|
|
367
|
+
For self-managed ClickHouse clusters with multiple replicas, set `replication` so Mastra emits `ReplicatedMergeTree` engines and applies `ON CLUSTER` to its DDL:
|
|
368
|
+
|
|
369
|
+
```typescript
|
|
370
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
371
|
+
import { MemoryPG, WorkflowsPG, ScoresPG } from '@mastra/pg'
|
|
372
|
+
import { ObservabilityStorageClickhouseVNext } from '@mastra/clickhouse'
|
|
373
|
+
|
|
374
|
+
const storage = new MastraCompositeStore({
|
|
375
|
+
id: 'composite',
|
|
376
|
+
domains: {
|
|
377
|
+
memory: new MemoryPG({ connectionString: process.env.DATABASE_URL }),
|
|
378
|
+
workflows: new WorkflowsPG({ connectionString: process.env.DATABASE_URL }),
|
|
379
|
+
scores: new ScoresPG({ connectionString: process.env.DATABASE_URL }),
|
|
380
|
+
observability: new ObservabilityStorageClickhouseVNext({
|
|
381
|
+
url: process.env.CLICKHOUSE_URL,
|
|
382
|
+
username: process.env.CLICKHOUSE_USERNAME,
|
|
383
|
+
password: process.env.CLICKHOUSE_PASSWORD,
|
|
384
|
+
replication: {
|
|
385
|
+
cluster: 'production_cluster',
|
|
386
|
+
// Optional (defaults shown):
|
|
387
|
+
// zookeeperPath: '/clickhouse/tables/{shard}/{database}/{table}',
|
|
388
|
+
// replicaName: '{replica}',
|
|
389
|
+
},
|
|
390
|
+
}),
|
|
391
|
+
},
|
|
392
|
+
})
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
Don't set `replication` on ClickHouse Cloud. Cloud rewrites `MergeTree` to `SharedMergeTree` server-side. See the [ClickHouse storage reference](https://mastra.ai/reference/storage/clickhouse) for the full config shape and operator notes.
|
|
396
|
+
|
|
397
|
+
> **Info:** This approach is also required when using storage providers that don't support observability (like Convex, DynamoDB, or Cloudflare). See the [MastraStorageExporter documentation](https://mastra.ai/docs/observability/integrations/exporters/mastra-storage) for the full list of supported providers.
|