@mastra/libsql 0.0.0-error-handler-fix-20251020202607 → 0.0.0-esbuild-bundle-worker-20260807182016
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4789 -3
- package/LICENSE.md +15 -0
- package/README.md +30 -20
- package/dist/docs/SKILL.md +52 -0
- package/dist/docs/assets/SOURCE_MAP.json +6 -0
- package/dist/docs/references/docs-agents-agent-approval.md +664 -0
- package/dist/docs/references/docs-agents-networks.md +184 -0
- package/dist/docs/references/docs-deployment-workers.md +137 -0
- package/dist/docs/references/docs-editor-overview.md +349 -0
- package/dist/docs/references/docs-memory-memory-processors.md +385 -0
- package/dist/docs/references/docs-memory-message-history.md +348 -0
- package/dist/docs/references/docs-memory-multi-user-threads.md +208 -0
- package/dist/docs/references/docs-memory-overview.md +266 -0
- package/dist/docs/references/docs-memory-semantic-recall.md +401 -0
- package/dist/docs/references/docs-memory-working-memory.md +431 -0
- package/dist/docs/references/docs-storage-overview.md +214 -0
- package/dist/docs/references/docs-workflows-snapshots.md +240 -0
- package/dist/docs/references/guides-agent-frameworks-ai-sdk.md +142 -0
- package/dist/docs/references/guides-rag-retrieval.md +520 -0
- package/dist/docs/references/reference-core-getMemory.md +51 -0
- package/dist/docs/references/reference-core-listMemory.md +57 -0
- package/dist/docs/references/reference-core-mastra-class.md +153 -0
- package/dist/docs/references/reference-file-based-agents-memory.md +58 -0
- package/dist/docs/references/reference-file-based-agents-storage.md +30 -0
- package/dist/docs/references/reference-memory-memory-class.md +148 -0
- package/dist/docs/references/reference-storage-composite.md +397 -0
- package/dist/docs/references/reference-storage-dynamodb.md +284 -0
- package/dist/docs/references/reference-storage-libsql.md +143 -0
- package/dist/docs/references/reference-storage-retention.md +248 -0
- package/dist/docs/references/reference-vectors-libsql.md +307 -0
- package/dist/index.cjs +13164 -3390
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +13134 -3383
- package/dist/index.js.map +1 -1
- package/dist/storage/db/index.d.ts +389 -0
- package/dist/storage/db/index.d.ts.map +1 -0
- package/dist/storage/db/utils.d.ts +84 -0
- package/dist/storage/db/utils.d.ts.map +1 -0
- package/dist/storage/db/write-lock.d.ts +8 -0
- package/dist/storage/db/write-lock.d.ts.map +1 -0
- package/dist/storage/domains/agents/index.d.ts +30 -0
- package/dist/storage/domains/agents/index.d.ts.map +1 -0
- package/dist/storage/domains/background-tasks/index.d.ts +27 -0
- package/dist/storage/domains/background-tasks/index.d.ts.map +1 -0
- package/dist/storage/domains/blobs/index.d.ts +17 -0
- package/dist/storage/domains/blobs/index.d.ts.map +1 -0
- package/dist/storage/domains/channels/index.d.ts +20 -0
- package/dist/storage/domains/channels/index.d.ts.map +1 -0
- package/dist/storage/domains/datasets/index.d.ts +43 -0
- package/dist/storage/domains/datasets/index.d.ts.map +1 -0
- package/dist/storage/domains/experiments/index.d.ts +54 -0
- package/dist/storage/domains/experiments/index.d.ts.map +1 -0
- package/dist/storage/domains/favorites/index.d.ts +17 -0
- package/dist/storage/domains/favorites/index.d.ts.map +1 -0
- package/dist/storage/domains/harness/index.d.ts +17 -0
- package/dist/storage/domains/harness/index.d.ts.map +1 -0
- package/dist/storage/domains/mcp-clients/index.d.ts +26 -0
- package/dist/storage/domains/mcp-clients/index.d.ts.map +1 -0
- package/dist/storage/domains/mcp-servers/index.d.ts +26 -0
- package/dist/storage/domains/mcp-servers/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +58 -54
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/notifications/index.d.ts +23 -0
- package/dist/storage/domains/notifications/index.d.ts.map +1 -0
- package/dist/storage/domains/observability/index.d.ts +49 -29
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/dist/storage/domains/prompt-blocks/index.d.ts +25 -0
- package/dist/storage/domains/prompt-blocks/index.d.ts.map +1 -0
- package/dist/storage/domains/schedules/index.d.ts +27 -0
- package/dist/storage/domains/schedules/index.d.ts.map +1 -0
- package/dist/storage/domains/scorer-definitions/index.d.ts +26 -0
- package/dist/storage/domains/scorer-definitions/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +29 -31
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/skills/index.d.ts +26 -0
- package/dist/storage/domains/skills/index.d.ts.map +1 -0
- package/dist/storage/domains/thread-state/index.d.ts +38 -0
- package/dist/storage/domains/thread-state/index.d.ts.map +1 -0
- package/dist/storage/domains/tool-provider-connections/index.d.ts +14 -0
- package/dist/storage/domains/tool-provider-connections/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -54
- package/dist/storage/domains/utils.d.ts.map +1 -1
- package/dist/storage/domains/workflow-definitions/index.d.ts +14 -0
- package/dist/storage/domains/workflow-definitions/index.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +27 -32
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/domains/workspaces/index.d.ts +26 -0
- package/dist/storage/domains/workspaces/index.d.ts.map +1 -0
- package/dist/storage/factory-storage.d.ts +28 -0
- package/dist/storage/factory-storage.d.ts.map +1 -0
- package/dist/storage/index.d.ts +128 -255
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/retention.d.ts +77 -0
- package/dist/storage/retention.d.ts.map +1 -0
- package/dist/vector/index.d.ts +31 -4
- package/dist/vector/index.d.ts.map +1 -1
- package/dist/vector/sql-builder.d.ts.map +1 -1
- package/package.json +22 -16
- package/dist/storage/domains/legacy-evals/index.d.ts +0 -18
- package/dist/storage/domains/legacy-evals/index.d.ts.map +0 -1
- package/dist/storage/domains/operations/index.d.ts +0 -110
- package/dist/storage/domains/operations/index.d.ts.map +0 -1
- package/dist/storage/domains/traces/index.d.ts +0 -21
- package/dist/storage/domains/traces/index.d.ts.map +0 -1
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
|
|
2
|
+
|
|
3
|
+
# libSQL storage
|
|
4
|
+
|
|
5
|
+
[libSQL](https://docs.turso.tech/libsql) is an open-source, SQLite-compatible database that supports both local and remote deployments. It can be used to store message history, workflow snapshots, traces, and eval scores.
|
|
6
|
+
|
|
7
|
+
For vectors like semantic recall or traditional RAG, use [libSQL Vector](https://mastra.ai/reference/vectors/libsql) which covers embeddings and vector search.
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
Storage providers must be installed as separate packages:
|
|
12
|
+
|
|
13
|
+
**npm**:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
npm install @mastra/libsql@latest
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
**pnpm**:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pnpm add @mastra/libsql@latest
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
**Yarn**:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
yarn add @mastra/libsql@latest
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
**Bun**:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
bun add @mastra/libsql@latest
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Usage
|
|
38
|
+
|
|
39
|
+
```typescript
|
|
40
|
+
import { LibSQLStore } from '@mastra/libsql'
|
|
41
|
+
import { Mastra } from '@mastra/core'
|
|
42
|
+
|
|
43
|
+
const mastra = new Mastra({
|
|
44
|
+
storage: new LibSQLStore({
|
|
45
|
+
id: 'libsql-storage',
|
|
46
|
+
url: 'file:./storage.db',
|
|
47
|
+
}),
|
|
48
|
+
})
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Agent-level file storage:
|
|
52
|
+
|
|
53
|
+
```typescript
|
|
54
|
+
import { Memory } from '@mastra/memory'
|
|
55
|
+
import { Agent } from '@mastra/core/agent'
|
|
56
|
+
import { LibSQLStore } from '@mastra/libsql'
|
|
57
|
+
|
|
58
|
+
export const agent = new Agent({
|
|
59
|
+
id: 'example-agent',
|
|
60
|
+
memory: new Memory({
|
|
61
|
+
storage: new LibSQLStore({
|
|
62
|
+
id: 'libsql-storage',
|
|
63
|
+
url: 'file:./agent.db',
|
|
64
|
+
}),
|
|
65
|
+
}),
|
|
66
|
+
})
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
> **Warning:** File storage doesn't work with serverless platforms that have ephemeral file systems. For serverless deployments, use [Turso](https://turso.tech) or a different database engine.
|
|
70
|
+
|
|
71
|
+
Production with remote database:
|
|
72
|
+
|
|
73
|
+
```typescript
|
|
74
|
+
storage: new LibSQLStore({
|
|
75
|
+
id: 'libsql-storage',
|
|
76
|
+
url: 'libsql://your-db-name.aws-ap-northeast-1.turso.io',
|
|
77
|
+
authToken: process.env.TURSO_AUTH_TOKEN,
|
|
78
|
+
})
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
For local development and testing, you can store data in memory:
|
|
82
|
+
|
|
83
|
+
```typescript
|
|
84
|
+
storage: new LibSQLStore({
|
|
85
|
+
id: 'libsql-storage',
|
|
86
|
+
url: ':memory:',
|
|
87
|
+
})
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
> **Warning:** In-memory storage resets when the process changes. Only suitable for development.
|
|
91
|
+
|
|
92
|
+
## Options
|
|
93
|
+
|
|
94
|
+
**url** (`string`): Database URL. Use :memory: for in-memory database, file:filename.db for a file database, or a libSQL connection string (e.g., libsql://your-database.turso.io) for remote storage.
|
|
95
|
+
|
|
96
|
+
**authToken** (`string`): Authentication token for remote libSQL databases.
|
|
97
|
+
|
|
98
|
+
## Managed tables
|
|
99
|
+
|
|
100
|
+
The storage implementation creates the core storage tables automatically, including `mastra_notifications` for notification inbox records and delivery metadata.
|
|
101
|
+
|
|
102
|
+
`LibSQLStore` exposes notification storage through `getStore('notifications')`.
|
|
103
|
+
|
|
104
|
+
## Initialization
|
|
105
|
+
|
|
106
|
+
When you pass storage to the Mastra class, `init()` is called automatically to create the [core schema](https://mastra.ai/reference/storage/overview):
|
|
107
|
+
|
|
108
|
+
```typescript
|
|
109
|
+
import { Mastra } from '@mastra/core'
|
|
110
|
+
import { LibSQLStore } from '@mastra/libsql'
|
|
111
|
+
|
|
112
|
+
const storage = new LibSQLStore({
|
|
113
|
+
id: 'libsql-storage',
|
|
114
|
+
url: 'file:./storage.db',
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
const mastra = new Mastra({
|
|
118
|
+
storage, // init() called automatically
|
|
119
|
+
})
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
If using storage directly without Mastra, call `init()` explicitly:
|
|
123
|
+
|
|
124
|
+
```typescript
|
|
125
|
+
import { LibSQLStore } from '@mastra/libsql'
|
|
126
|
+
|
|
127
|
+
const storage = new LibSQLStore({
|
|
128
|
+
id: 'libsql-storage',
|
|
129
|
+
url: 'file:./storage.db',
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
await storage.init()
|
|
133
|
+
|
|
134
|
+
// Access domain-specific stores via getStore()
|
|
135
|
+
const memoryStore = await storage.getStore('memory')
|
|
136
|
+
const thread = await memoryStore?.getThreadById({ threadId: '...' })
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## Observability
|
|
140
|
+
|
|
141
|
+
libSQL supports observability and is ideal for local development. Use the `realtime` [tracing strategy](https://mastra.ai/docs/observability/integrations/exporters/mastra-storage) for immediate visibility while debugging.
|
|
142
|
+
|
|
143
|
+
For production environments with higher trace volumes, consider using [PostgreSQL](https://mastra.ai/reference/storage/postgresql) or [ClickHouse via composite storage](https://mastra.ai/reference/storage/composite).
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
|
|
2
|
+
|
|
3
|
+
# Storage retention
|
|
4
|
+
|
|
5
|
+
Storage grows without bound by default. Retention is an opt-in, age-based cleanup system: you declare per-table `maxAge` policies in the `retention` config, then call `storage.prune()` to delete rows older than their configured age. Anything you don't configure is kept forever, so there is no behavior change until you opt in.
|
|
6
|
+
|
|
7
|
+
`prune()` deletes rows. It caps growth and is safe to run against large tables (batched, bounded, resumable, cancellable). It never reclaims disk: on SQLite/libSQL the freed pages are reused by future writes so the file stops growing, but handing disk back to the OS (for example a `VACUUM`) is left to the underlying database and the operator to manage.
|
|
8
|
+
|
|
9
|
+
Retention covers **growth tables** only: tables that accumulate rows unbounded as a side effect of normal operation (conversation history, telemetry, job and run records, schedule fire history, event feeds). User-authored artifacts and config (agents, skills, workspaces, prompt blocks, datasets, schedule definitions, channel installations, and so on) grow with user intent and are edited or deleted explicitly, so they're not valid retention keys.
|
|
10
|
+
|
|
11
|
+
The reference implementations are [libSQL](https://mastra.ai/reference/storage/libsql), [PostgreSQL](https://mastra.ai/reference/storage/postgresql), and [MongoDB](https://mastra.ai/reference/storage/mongodb). Other adapters keep rows forever until they implement retention.
|
|
12
|
+
|
|
13
|
+
## Usage example
|
|
14
|
+
|
|
15
|
+
Declare `retention` on any `MastraCompositeStore` (or an adapter that extends it, such as `LibSQLStore`), then call `prune()` from your own scheduler.
|
|
16
|
+
|
|
17
|
+
```typescript
|
|
18
|
+
import { LibSQLStore } from '@mastra/libsql'
|
|
19
|
+
|
|
20
|
+
const storage = new LibSQLStore({
|
|
21
|
+
id: 'mastra-storage',
|
|
22
|
+
url: 'file:./mastra.db',
|
|
23
|
+
retention: {
|
|
24
|
+
memory: {
|
|
25
|
+
messages: { maxAge: '30d' },
|
|
26
|
+
threads: { maxAge: '90d', batchSize: 500 },
|
|
27
|
+
},
|
|
28
|
+
observability: {
|
|
29
|
+
spans: { maxAge: '7d' },
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
// Wire this to your own cron/scheduler: Mastra never runs it for you.
|
|
35
|
+
const results = await storage.prune()
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`retention` is fully typed. Keys must be real domain keys, and each table key must be one the domain declares as retention-eligible. Passing the object straight into a store config type-checks it; if you build it standalone, use `satisfies RetentionConfig` so unknown domains or tables are compile errors:
|
|
39
|
+
|
|
40
|
+
```typescript
|
|
41
|
+
import type { RetentionConfig } from '@mastra/core/storage'
|
|
42
|
+
|
|
43
|
+
const retention = {
|
|
44
|
+
memory: {
|
|
45
|
+
messages: { maxAge: '30d' }, // ok
|
|
46
|
+
bogus: { maxAge: '30d' }, // Error: not a memory retention table
|
|
47
|
+
},
|
|
48
|
+
bogusDomain: {}, // Error: not a storage domain
|
|
49
|
+
} satisfies RetentionConfig
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Retention config
|
|
53
|
+
|
|
54
|
+
Set the `retention` field on the store config.
|
|
55
|
+
|
|
56
|
+
**retention** (`RetentionConfig`): Per-domain, per-table age policies. Unset domains and tables are kept forever.
|
|
57
|
+
|
|
58
|
+
**retention.\[domain]** (`Record<TableKey, TableRetentionPolicy>`): A real storage domain key (e.g. memory, observability). Maps that domain's retention-eligible table keys to their policies.
|
|
59
|
+
|
|
60
|
+
### TableRetentionPolicy
|
|
61
|
+
|
|
62
|
+
**maxAge** (`Duration`): Maximum age to keep rows. Rows whose anchor timestamp is strictly older than Date.now() - maxAge are eligible for deletion. A number is milliseconds, or a string with a unit suffix: ms, s, m, h, d, w (e.g. '30d', '12h').
|
|
63
|
+
|
|
64
|
+
**batchSize** (`number`): Rows deleted per batch. Each batch is its own transaction, which bounds lock duration and WAL growth on large tables. (Default: `1000`)
|
|
65
|
+
|
|
66
|
+
### Retention-eligible tables
|
|
67
|
+
|
|
68
|
+
Each domain declares which of its tables can be age-pruned and which timestamp column anchors the comparison. The anchor is chosen so `maxAge` means what you'd expect for that data. Append-only logs use creation time, and live state uses last activity. Jobs and runs use completion time, so in-flight work is never pruned.
|
|
69
|
+
|
|
70
|
+
| Domain | Table key | Anchor column | `maxAge` measures |
|
|
71
|
+
| ----------------- | ------------------ | ---------------- | ---------------------------------------------------------------- |
|
|
72
|
+
| `memory` | `threads` | `createdAt` | Thread age |
|
|
73
|
+
| `memory` | `messages` | `createdAt` | Message age |
|
|
74
|
+
| `memory` | `resources` | `createdAt` | Resource age |
|
|
75
|
+
| `threadState` | `threadState` | `updatedAt` | Inactivity: state for still-active threads survives |
|
|
76
|
+
| `observability` | `spans` | `startedAt` | Span age |
|
|
77
|
+
| `observability` | `metrics` | `timestamp` | Metric event age (v-next only) |
|
|
78
|
+
| `observability` | `logs` | `timestamp` | Log event age (v-next only) |
|
|
79
|
+
| `observability` | `scores` | `timestamp` | Score event age (v-next only) |
|
|
80
|
+
| `observability` | `feedback` | `timestamp` | Feedback event age (v-next only) |
|
|
81
|
+
| `scores` | `scorers` | `createdAt` | Score record age |
|
|
82
|
+
| `workflows` | `workflowSnapshot` | `updatedAt` | Inactivity, suspended or long-running workflows survive |
|
|
83
|
+
| `backgroundTasks` | `backgroundTasks` | `completedAt` | Time since completion, in-flight tasks (`NULL`) are never pruned |
|
|
84
|
+
| `experiments` | `experiments` | `completedAt` | Time since completion, running experiments are never pruned |
|
|
85
|
+
| `notifications` | `notifications` | `createdAt` | Notification age |
|
|
86
|
+
| `harness` | `sessions` | `createdAt` | Session record age |
|
|
87
|
+
| `schedules` | `triggers` | `actual_fire_at` | Fire-history age (epoch-ms column) |
|
|
88
|
+
|
|
89
|
+
> **Note:**
|
|
90
|
+
>
|
|
91
|
+
> - The memory `observational_memory` table has no timestamp anchor, so it can't be age-pruned and isn't a valid retention key.
|
|
92
|
+
> - Experiments prune as whole units: an aged experiment's result rows are deleted together with it (results cascade with their parent), so a run is never left partially deleted. Retention doesn't have a separate `results` key.
|
|
93
|
+
> - For `schedules`, the growth table is the fire history (`schedule_triggers`, one row per fire): schedule definitions are config and aren't pruned.
|
|
94
|
+
> - On PostgreSQL, timestamp anchors use the timezone-aware mirror columns (for example `createdAtZ`, `completedAtZ`).
|
|
95
|
+
> - LibSQL supports all domains above; PostgreSQL and MongoDB support all except `threadState` and `harness`, which they don't implement.
|
|
96
|
+
> - The v-next PostgreSQL observability domain stores signal events in day-partitioned tables (`spans`, `metrics`, `logs`, `scores`, `feedback`). For it, `prune()` drops whole day partitions (or TimescaleDB chunks) that are entirely older than the cutoff instead of deleting rows: effective level of detail is one day, and a partition is only dropped once its entire day is past `maxAge`. `PruneResult.deleted` reports the number of rows in the dropped partitions.
|
|
97
|
+
|
|
98
|
+
## Methods
|
|
99
|
+
|
|
100
|
+
### Retention
|
|
101
|
+
|
|
102
|
+
#### `prune(options?)`
|
|
103
|
+
|
|
104
|
+
Deletes rows older than their configured `maxAge` across every domain that has a policy in `retention`. Returns one `PruneResult` per table touched. With no `retention` configured it's a no-op returning `[]`.
|
|
105
|
+
|
|
106
|
+
`prune()` is designed to be safe on tables with millions of rows. It deletes in bounded, batched chunks (each batch is its own transaction) so it never takes a long lock or bloats the transaction log. It never runs a `VACUUM`.
|
|
107
|
+
|
|
108
|
+
Pass `options.retention` to replace the configured policies for that call only: for example to skip a domain (keep chat history) or prune more aggressively than the standing config. The store's configured `retention` is unchanged.
|
|
109
|
+
|
|
110
|
+
Anchor-column indexes are created lazily on the first `prune()` call for each table with a policy (never at `init()`) so deployments that don't configure retention pay no extra index write or disk overhead. The first prune of an existing large table pays a one-time index build. Subsequent prunes reuse the index.
|
|
111
|
+
|
|
112
|
+
```typescript
|
|
113
|
+
const results = await storage.prune({
|
|
114
|
+
maxRows: 50_000, // cap work this call
|
|
115
|
+
pauseMs: 50, // breathe between batches
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
for (const r of results) {
|
|
119
|
+
console.log(`${r.domain}.${r.table}: deleted ${r.deleted}, done=${r.done}`)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// One-off pass with different policies (configured retention untouched):
|
|
123
|
+
await storage.prune({
|
|
124
|
+
retention: {
|
|
125
|
+
observability: { spans: { maxAge: '1d' } },
|
|
126
|
+
},
|
|
127
|
+
})
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Returns: `Promise<PruneResult[]>`
|
|
131
|
+
|
|
132
|
+
##### PruneOptions
|
|
133
|
+
|
|
134
|
+
**maxBatches** (`number`): Maximum delete batches per table per call. When reached, that table's result is returned with done: false.
|
|
135
|
+
|
|
136
|
+
**maxRows** (`number`): Maximum rows deleted per table per call. When reached, that table's result is returned with done: false.
|
|
137
|
+
|
|
138
|
+
**pauseMs** (`number`): Delay in milliseconds between batches, to avoid starving live traffic.
|
|
139
|
+
|
|
140
|
+
**signal** (`AbortSignal`): Cooperative cancellation. The batch loop checks it between batches and stops cleanly, returning partial results with done: false.
|
|
141
|
+
|
|
142
|
+
**retention** (`RetentionConfig`): Replaces the store's configured retention policies for this call only — e.g. to skip a domain or prune more aggressively. The configured retention is unchanged.
|
|
143
|
+
|
|
144
|
+
##### PruneResult
|
|
145
|
+
|
|
146
|
+
Each result describes one table's progress:
|
|
147
|
+
|
|
148
|
+
```typescript
|
|
149
|
+
interface PruneResult {
|
|
150
|
+
domain: string // e.g. 'memory'
|
|
151
|
+
table: string // physical table name, e.g. 'mastra_messages'
|
|
152
|
+
deleted: number // rows deleted during this call
|
|
153
|
+
done: boolean // false => eligible rows remain; call prune() again
|
|
154
|
+
}
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Running prune on a schedule
|
|
158
|
+
|
|
159
|
+
`prune()` has no built-in scheduler: you decide when it runs. Because it's bounded, a single call may not delete everything. When any result has `done: false`, eligible rows remain and you call again on the next tick. This keeps each invocation short and lets a large backlog drain over several runs.
|
|
160
|
+
|
|
161
|
+
```typescript
|
|
162
|
+
// Runs on your own cron (node-cron, a workflow schedule, an external job, etc.).
|
|
163
|
+
async function retentionTick() {
|
|
164
|
+
const results = await storage.prune({ maxRows: 100_000, pauseMs: 25 })
|
|
165
|
+
const incomplete = results.filter(r => !r.done)
|
|
166
|
+
if (incomplete.length) {
|
|
167
|
+
// Rows remain; the next scheduled tick will continue where this one stopped.
|
|
168
|
+
console.log(
|
|
169
|
+
'retention still draining:',
|
|
170
|
+
incomplete.map(r => `${r.domain}.${r.table}`),
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
You can also cancel a long-running prune with an `AbortSignal`: the loop stops between batches and returns partial results with `done: false`, so the next run resumes cleanly.
|
|
177
|
+
|
|
178
|
+
## MongoDB TTL indexes (alternative to prune)
|
|
179
|
+
|
|
180
|
+
MongoDB offers native [TTL (Time-To-Live) indexes](https://www.mongodb.com/docs/manual/core/index-ttl/) that automatically delete expired documents without requiring manual `prune()` calls. This is a database-level feature that runs as a background thread.
|
|
181
|
+
|
|
182
|
+
> **When to use TTL vs prune():** **Use MongoDB TTL indexes when:**
|
|
183
|
+
>
|
|
184
|
+
> - You want automated, zero-maintenance deletion
|
|
185
|
+
> - Your retention periods are fixed (e.g., "always 30 days")
|
|
186
|
+
> - You prefer database-native solutions
|
|
187
|
+
>
|
|
188
|
+
> **Use `prune()` when:**
|
|
189
|
+
>
|
|
190
|
+
> - You need fine-grained control over deletion timing
|
|
191
|
+
> - You want to cap deletion rate during business hours
|
|
192
|
+
> - You need resumable, cancellable cleanup operations
|
|
193
|
+
> - You're using composite storage with multiple databases
|
|
194
|
+
>
|
|
195
|
+
> Both approaches are valid. TTL is simpler. `prune()` gives more control.
|
|
196
|
+
|
|
197
|
+
### Setting up TTL indexes on MongoDB
|
|
198
|
+
|
|
199
|
+
TTL indexes work on date fields. MongoDB checks the index every 60 seconds and deletes documents where the date field + TTL duration < current time.
|
|
200
|
+
|
|
201
|
+
```typescript
|
|
202
|
+
import { MongoDBStore } from '@mastra/mongodb'
|
|
203
|
+
|
|
204
|
+
const storage = new MongoDBStore({
|
|
205
|
+
id: 'mongodb-storage',
|
|
206
|
+
uri: process.env.MONGODB_URI!,
|
|
207
|
+
dbName: process.env.MONGODB_DB_NAME!,
|
|
208
|
+
indexes: [
|
|
209
|
+
// Messages expire after 30 days
|
|
210
|
+
{
|
|
211
|
+
collection: 'mastra_messages',
|
|
212
|
+
keys: { createdAt: 1 },
|
|
213
|
+
options: { expireAfterSeconds: 30 * 24 * 60 * 60 }, // 30 days
|
|
214
|
+
},
|
|
215
|
+
// Threads expire after 90 days
|
|
216
|
+
{
|
|
217
|
+
collection: 'mastra_threads',
|
|
218
|
+
keys: { createdAt: 1 },
|
|
219
|
+
options: { expireAfterSeconds: 90 * 24 * 60 * 60 }, // 90 days
|
|
220
|
+
},
|
|
221
|
+
// Spans expire after 7 days
|
|
222
|
+
{
|
|
223
|
+
collection: 'mastra_ai_spans',
|
|
224
|
+
keys: { startedAt: 1 },
|
|
225
|
+
options: { expireAfterSeconds: 7 * 24 * 60 * 60 }, // 7 days
|
|
226
|
+
},
|
|
227
|
+
],
|
|
228
|
+
})
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
> **Tip:** TTL indexes delete documents shortly after they expire (background thread runs every \~60 seconds), but the exact timing isn't guaranteed. For precise, immediate cleanup, use `prune()` instead.
|
|
232
|
+
|
|
233
|
+
## Reclaiming disk
|
|
234
|
+
|
|
235
|
+
`prune()` deletes rows but doesn't shrink the database file. On SQLite/libSQL the freed pages go on a freelist and are reused by future writes, so the file stops growing: for most users this alone solves the unbounded-growth problem.
|
|
236
|
+
|
|
237
|
+
Handing that free space back to the OS is a separate concern that Mastra doesn't manage. If you specifically need to shrink the file, run the underlying database's compaction (for example `VACUUM` on self-hosted libSQL) yourself in a maintenance window. A full `VACUUM` locks the file and needs roughly twice the file size in free disk. On PostgreSQL, autovacuum reclaims dead tuples for reuse automatically; a manual `VACUUM FULL` is only needed if you must return disk to the OS.
|
|
238
|
+
|
|
239
|
+
For MongoDB, deleted documents are reused by future insertions. To reclaim disk space, run [`db.runCommand({ compact: "collection_name" })`](https://www.mongodb.com/docs/manual/reference/command/compact/) during a maintenance window.
|
|
240
|
+
|
|
241
|
+
> **LibSQL and Turso:** [Turso Cloud](https://mastra.ai/reference/storage/libsql) manages storage compaction for you, so there's nothing to reclaim manually. This applies only to self-hosted libSQL files.
|
|
242
|
+
|
|
243
|
+
## Related
|
|
244
|
+
|
|
245
|
+
- [libSQL storage](https://mastra.ai/reference/storage/libsql)
|
|
246
|
+
- [PostgreSQL storage](https://mastra.ai/reference/storage/postgresql)
|
|
247
|
+
- [Composite storage](https://mastra.ai/reference/storage/composite)
|
|
248
|
+
- [Storage overview](https://mastra.ai/reference/storage/overview)
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
|
|
2
|
+
|
|
3
|
+
# libSQL vector store
|
|
4
|
+
|
|
5
|
+
The libSQL storage implementation provides a SQLite-compatible vector search [libSQL](https://github.com/tursodatabase/libsql), a fork of SQLite with vector extensions, and [Turso](https://turso.tech/) with vector extensions, offering a lightweight and efficient vector database solution. It's part of the `@mastra/libsql` package and offers efficient vector similarity search with metadata filtering.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
**npm**:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npm install @mastra/libsql@latest
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
**pnpm**:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pnpm add @mastra/libsql@latest
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
**Yarn**:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
yarn add @mastra/libsql@latest
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
**Bun**:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
bun add @mastra/libsql@latest
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
```typescript
|
|
36
|
+
import { LibSQLVector } from "@mastra/libsql";
|
|
37
|
+
|
|
38
|
+
// Create a new vector store instance
|
|
39
|
+
const store = new LibSQLVector({
|
|
40
|
+
id: 'libsql-vector',
|
|
41
|
+
url: process.env.DATABASE_URL,
|
|
42
|
+
// Optional: for Turso cloud databases
|
|
43
|
+
authToken: process.env.DATABASE_AUTH_TOKEN,
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// Create an index
|
|
47
|
+
await store.createIndex({
|
|
48
|
+
indexName: "myCollection",
|
|
49
|
+
dimension: 1536,
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// Add vectors with metadata
|
|
53
|
+
const vectors = [[0.1, 0.2, ...], [0.3, 0.4, ...]];
|
|
54
|
+
const metadata = [
|
|
55
|
+
{ text: "first document", category: "A" },
|
|
56
|
+
{ text: "second document", category: "B" }
|
|
57
|
+
];
|
|
58
|
+
await store.upsert({
|
|
59
|
+
indexName: "myCollection",
|
|
60
|
+
vectors,
|
|
61
|
+
metadata,
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
// Query similar vectors
|
|
65
|
+
const queryVector = [0.1, 0.2, ...];
|
|
66
|
+
const results = await store.query({
|
|
67
|
+
indexName: "myCollection",
|
|
68
|
+
queryVector,
|
|
69
|
+
topK: 10, // top K results
|
|
70
|
+
filter: { category: "A" } // optional metadata filter
|
|
71
|
+
});
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Constructor options
|
|
75
|
+
|
|
76
|
+
**url** (`string`): libSQL database URL. Use ':memory:' for in-memory database, 'file:dbname.db' for local file, or a libSQL-compatible connection string like 'libsql://your-database.turso.io'.
|
|
77
|
+
|
|
78
|
+
**authToken** (`string`): Authentication token for Turso cloud databases
|
|
79
|
+
|
|
80
|
+
**syncUrl** (`string`): URL for database replication (Turso specific)
|
|
81
|
+
|
|
82
|
+
**syncInterval** (`number`): Interval in milliseconds for database sync (Turso specific)
|
|
83
|
+
|
|
84
|
+
## Methods
|
|
85
|
+
|
|
86
|
+
### `createIndex()`
|
|
87
|
+
|
|
88
|
+
Creates a new vector collection. The index name must start with a letter or underscore and can only contain letters, numbers, and underscore characters. The dimension must be a positive integer.
|
|
89
|
+
|
|
90
|
+
**indexName** (`string`): Name of the index to create
|
|
91
|
+
|
|
92
|
+
**dimension** (`number`): Vector dimension size (must match your embedding model)
|
|
93
|
+
|
|
94
|
+
**metric** (`'cosine' | 'euclidean' | 'dotproduct'`): Distance metric for similarity search. Note: Currently only cosine similarity is supported by libSQL. (Default: `cosine`)
|
|
95
|
+
|
|
96
|
+
### `upsert()`
|
|
97
|
+
|
|
98
|
+
Adds or updates vectors and their metadata in the index. Uses a transaction to ensure all vectors are inserted atomically - if any insert fails, the entire operation is rolled back.
|
|
99
|
+
|
|
100
|
+
**indexName** (`string`): Name of the index to insert into
|
|
101
|
+
|
|
102
|
+
**vectors** (`number[][]`): Array of embedding vectors
|
|
103
|
+
|
|
104
|
+
**metadata** (`Record<string, any>[]`): Metadata for each vector
|
|
105
|
+
|
|
106
|
+
**ids** (`string[]`): Optional vector IDs (auto-generated if not provided)
|
|
107
|
+
|
|
108
|
+
### `query()`
|
|
109
|
+
|
|
110
|
+
Searches for similar vectors with optional metadata filtering.
|
|
111
|
+
|
|
112
|
+
**indexName** (`string`): Name of the index to search in
|
|
113
|
+
|
|
114
|
+
**queryVector** (`number[]`): Query vector to find similar vectors for
|
|
115
|
+
|
|
116
|
+
**topK** (`number`): Number of results to return (Default: `10`)
|
|
117
|
+
|
|
118
|
+
**filter** (`Filter`): Metadata filters
|
|
119
|
+
|
|
120
|
+
**includeVector** (`boolean`): Whether to include vector data in results (Default: `false`)
|
|
121
|
+
|
|
122
|
+
**minScore** (`number`): Minimum similarity score threshold (Default: `0`)
|
|
123
|
+
|
|
124
|
+
### `describeIndex()`
|
|
125
|
+
|
|
126
|
+
Gets information about an index.
|
|
127
|
+
|
|
128
|
+
**indexName** (`string`): Name of the index to describe
|
|
129
|
+
|
|
130
|
+
Returns:
|
|
131
|
+
|
|
132
|
+
```typescript
|
|
133
|
+
interface IndexStats {
|
|
134
|
+
dimension: number
|
|
135
|
+
count: number
|
|
136
|
+
metric: 'cosine' | 'euclidean' | 'dotproduct'
|
|
137
|
+
}
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### `deleteIndex()`
|
|
141
|
+
|
|
142
|
+
Deletes an index and all its data.
|
|
143
|
+
|
|
144
|
+
**indexName** (`string`): Name of the index to delete
|
|
145
|
+
|
|
146
|
+
### `listIndexes()`
|
|
147
|
+
|
|
148
|
+
Lists all vector indexes in the database.
|
|
149
|
+
|
|
150
|
+
Returns: `Promise<string[]>`
|
|
151
|
+
|
|
152
|
+
### `truncateIndex()`
|
|
153
|
+
|
|
154
|
+
Removes all vectors from an index while keeping the index structure.
|
|
155
|
+
|
|
156
|
+
**indexName** (`string`): Name of the index to truncate
|
|
157
|
+
|
|
158
|
+
### `updateVector()`
|
|
159
|
+
|
|
160
|
+
Update a single vector by ID or by metadata filter. Either `id` or `filter` must be provided, but not both.
|
|
161
|
+
|
|
162
|
+
**indexName** (`string`): Name of the index containing the vector
|
|
163
|
+
|
|
164
|
+
**id** (`string`): ID of the vector entry to update (mutually exclusive with filter)
|
|
165
|
+
|
|
166
|
+
**filter** (`Record<string, any>`): Metadata filter to identify vector(s) to update (mutually exclusive with id)
|
|
167
|
+
|
|
168
|
+
**update** (`object`): Update data containing vector and/or metadata
|
|
169
|
+
|
|
170
|
+
**update.vector** (`number[]`): New vector data to update
|
|
171
|
+
|
|
172
|
+
**update.metadata** (`Record<string, any>`): New metadata to update
|
|
173
|
+
|
|
174
|
+
### `deleteVector()`
|
|
175
|
+
|
|
176
|
+
Deletes a specific vector entry from an index by its ID.
|
|
177
|
+
|
|
178
|
+
**indexName** (`string`): Name of the index containing the vector
|
|
179
|
+
|
|
180
|
+
**id** (`string`): ID of the vector entry to delete
|
|
181
|
+
|
|
182
|
+
### `deleteVectors()`
|
|
183
|
+
|
|
184
|
+
Delete multiple vectors by IDs or by metadata filter. Either `ids` or `filter` must be provided, but not both.
|
|
185
|
+
|
|
186
|
+
**indexName** (`string`): Name of the index containing the vectors to delete
|
|
187
|
+
|
|
188
|
+
**ids** (`string[]`): Array of vector IDs to delete (mutually exclusive with filter)
|
|
189
|
+
|
|
190
|
+
**filter** (`Record<string, any>`): Metadata filter to identify vectors to delete (mutually exclusive with ids)
|
|
191
|
+
|
|
192
|
+
## Response types
|
|
193
|
+
|
|
194
|
+
Query results are returned in this format:
|
|
195
|
+
|
|
196
|
+
```typescript
|
|
197
|
+
interface QueryResult {
|
|
198
|
+
id: string
|
|
199
|
+
score: number
|
|
200
|
+
metadata: Record<string, any>
|
|
201
|
+
vector?: number[] // Only included if includeVector is true
|
|
202
|
+
}
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
## Error handling
|
|
206
|
+
|
|
207
|
+
The store throws specific errors for different failure cases:
|
|
208
|
+
|
|
209
|
+
```typescript
|
|
210
|
+
try {
|
|
211
|
+
await store.query({
|
|
212
|
+
indexName: 'my-collection',
|
|
213
|
+
queryVector: queryVector,
|
|
214
|
+
})
|
|
215
|
+
} catch (error) {
|
|
216
|
+
// Handle specific error cases
|
|
217
|
+
if (error.message.includes('Invalid index name format')) {
|
|
218
|
+
console.error(
|
|
219
|
+
'Index name must start with a letter/underscore and contain only alphanumeric characters',
|
|
220
|
+
)
|
|
221
|
+
} else if (error.message.includes('Table not found')) {
|
|
222
|
+
console.error('The specified index does not exist')
|
|
223
|
+
} else {
|
|
224
|
+
console.error('Vector store error:', error.message)
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Common error cases include:
|
|
230
|
+
|
|
231
|
+
- Invalid index name format
|
|
232
|
+
- Invalid vector dimensions
|
|
233
|
+
- Table/index not found
|
|
234
|
+
- Database connection issues
|
|
235
|
+
- Transaction failures during upsert
|
|
236
|
+
|
|
237
|
+
## Usage example
|
|
238
|
+
|
|
239
|
+
### Local embeddings with fastembed
|
|
240
|
+
|
|
241
|
+
Embeddings are numeric vectors used by memory's `semanticRecall` to retrieve related messages by meaning (not keywords). This setup uses `@mastra/fastembed` to generate vector embeddings.
|
|
242
|
+
|
|
243
|
+
Install `fastembed` to get started:
|
|
244
|
+
|
|
245
|
+
**npm**:
|
|
246
|
+
|
|
247
|
+
```bash
|
|
248
|
+
npm install @mastra/fastembed@latest
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
**pnpm**:
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
pnpm add @mastra/fastembed@latest
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
**Yarn**:
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
yarn add @mastra/fastembed@latest
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
**Bun**:
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
bun add @mastra/fastembed@latest
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Add the following to your agent:
|
|
270
|
+
|
|
271
|
+
```typescript
|
|
272
|
+
import { Memory } from '@mastra/memory'
|
|
273
|
+
import { Agent } from '@mastra/core/agent'
|
|
274
|
+
import { LibSQLStore, LibSQLVector } from '@mastra/libsql'
|
|
275
|
+
import { fastembed } from '@mastra/fastembed'
|
|
276
|
+
|
|
277
|
+
export const libsqlAgent = new Agent({
|
|
278
|
+
id: 'libsql-agent',
|
|
279
|
+
name: 'libSQL Agent',
|
|
280
|
+
instructions:
|
|
281
|
+
'You are an AI agent with the ability to automatically recall memories from previous interactions.',
|
|
282
|
+
model: 'openai/gpt-5.6-sol',
|
|
283
|
+
memory: new Memory({
|
|
284
|
+
storage: new LibSQLStore({
|
|
285
|
+
id: 'libsql-agent-storage',
|
|
286
|
+
url: 'file:libsql-agent.db',
|
|
287
|
+
}),
|
|
288
|
+
vector: new LibSQLVector({
|
|
289
|
+
id: 'libsql-agent-vector',
|
|
290
|
+
url: 'file:libsql-agent.db',
|
|
291
|
+
}),
|
|
292
|
+
embedder: fastembed,
|
|
293
|
+
options: {
|
|
294
|
+
lastMessages: 10,
|
|
295
|
+
semanticRecall: {
|
|
296
|
+
topK: 3,
|
|
297
|
+
messageRange: 2,
|
|
298
|
+
},
|
|
299
|
+
generateTitle: true, // Explicitly enable automatic title generation
|
|
300
|
+
},
|
|
301
|
+
}),
|
|
302
|
+
})
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
## Related
|
|
306
|
+
|
|
307
|
+
- [Metadata Filters](https://mastra.ai/reference/rag/metadata-filters)
|