@mastra/libsql 0.0.4 → 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +36 -0
- package/dist/_tsup-dts-rollup.d.cts +17 -59
- package/dist/_tsup-dts-rollup.d.ts +17 -59
- package/dist/index.cjs +277 -223
- package/dist/index.js +277 -223
- package/package.json +7 -4
- package/src/storage/index.ts +18 -10
- package/src/vector/index.test.ts +153 -216
- package/src/vector/index.ts +56 -84
- package/src/vector/sql-builder.ts +243 -175
package/src/vector/index.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { createClient } from '@libsql/client';
|
|
2
2
|
import type { Client as TursoClient, InValue } from '@libsql/client';
|
|
3
|
+
|
|
4
|
+
import { parseSqlIdentifier } from '@mastra/core/utils';
|
|
3
5
|
import { MastraVector } from '@mastra/core/vector';
|
|
4
6
|
import type {
|
|
5
7
|
IndexStats,
|
|
@@ -7,8 +9,10 @@ import type {
|
|
|
7
9
|
QueryVectorParams,
|
|
8
10
|
CreateIndexParams,
|
|
9
11
|
UpsertVectorParams,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
DescribeIndexParams,
|
|
13
|
+
DeleteIndexParams,
|
|
14
|
+
DeleteVectorParams,
|
|
15
|
+
UpdateVectorParams,
|
|
12
16
|
} from '@mastra/core/vector';
|
|
13
17
|
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
14
18
|
import { LibSQLFilterTranslator } from './filter';
|
|
@@ -18,8 +22,6 @@ interface LibSQLQueryParams extends QueryVectorParams {
|
|
|
18
22
|
minScore?: number;
|
|
19
23
|
}
|
|
20
24
|
|
|
21
|
-
type LibSQLQueryArgs = [...QueryVectorArgs, number?];
|
|
22
|
-
|
|
23
25
|
export class LibSQLVector extends MastraVector {
|
|
24
26
|
private turso: TursoClient;
|
|
25
27
|
|
|
@@ -56,17 +58,30 @@ export class LibSQLVector extends MastraVector {
|
|
|
56
58
|
return translator.translate(filter);
|
|
57
59
|
}
|
|
58
60
|
|
|
59
|
-
async query(
|
|
60
|
-
|
|
61
|
-
|
|
61
|
+
async query({
|
|
62
|
+
indexName,
|
|
63
|
+
queryVector,
|
|
64
|
+
topK = 10,
|
|
65
|
+
filter,
|
|
66
|
+
includeVector = false,
|
|
67
|
+
minScore = 0,
|
|
68
|
+
}: LibSQLQueryParams): Promise<QueryResult[]> {
|
|
62
69
|
try {
|
|
63
|
-
|
|
70
|
+
if (!Number.isInteger(topK) || topK <= 0) {
|
|
71
|
+
throw new Error('topK must be a positive integer');
|
|
72
|
+
}
|
|
73
|
+
if (!Array.isArray(queryVector) || !queryVector.every(x => typeof x === 'number' && Number.isFinite(x))) {
|
|
74
|
+
throw new Error('queryVector must be an array of finite numbers');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
64
78
|
|
|
65
79
|
const vectorStr = `[${queryVector.join(',')}]`;
|
|
66
80
|
|
|
67
81
|
const translatedFilter = this.transformFilter(filter);
|
|
68
82
|
const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter);
|
|
69
83
|
filterValues.push(minScore);
|
|
84
|
+
filterValues.push(topK);
|
|
70
85
|
|
|
71
86
|
const query = `
|
|
72
87
|
WITH vector_scores AS (
|
|
@@ -75,14 +90,14 @@ export class LibSQLVector extends MastraVector {
|
|
|
75
90
|
(1-vector_distance_cos(embedding, '${vectorStr}')) as score,
|
|
76
91
|
metadata
|
|
77
92
|
${includeVector ? ', vector_extract(embedding) as embedding' : ''}
|
|
78
|
-
FROM ${
|
|
93
|
+
FROM ${parsedIndexName}
|
|
79
94
|
${filterQuery}
|
|
80
95
|
)
|
|
81
96
|
SELECT *
|
|
82
97
|
FROM vector_scores
|
|
83
98
|
WHERE score > ?
|
|
84
99
|
ORDER BY score DESC
|
|
85
|
-
LIMIT
|
|
100
|
+
LIMIT ?`;
|
|
86
101
|
|
|
87
102
|
const result = await this.turso.execute({
|
|
88
103
|
sql: query,
|
|
@@ -100,18 +115,16 @@ export class LibSQLVector extends MastraVector {
|
|
|
100
115
|
}
|
|
101
116
|
}
|
|
102
117
|
|
|
103
|
-
async upsert(
|
|
104
|
-
const params = this.normalizeArgs<UpsertVectorParams>('upsert', args);
|
|
105
|
-
|
|
106
|
-
const { indexName, vectors, metadata, ids } = params;
|
|
118
|
+
async upsert({ indexName, vectors, metadata, ids }: UpsertVectorParams): Promise<string[]> {
|
|
107
119
|
const tx = await this.turso.transaction('write');
|
|
108
120
|
|
|
109
121
|
try {
|
|
122
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
110
123
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
111
124
|
|
|
112
125
|
for (let i = 0; i < vectors.length; i++) {
|
|
113
126
|
const query = `
|
|
114
|
-
INSERT INTO ${
|
|
127
|
+
INSERT INTO ${parsedIndexName} (vector_id, embedding, metadata)
|
|
115
128
|
VALUES (?, vector32(?), ?)
|
|
116
129
|
ON CONFLICT(vector_id) DO UPDATE SET
|
|
117
130
|
embedding = vector32(?),
|
|
@@ -156,23 +169,18 @@ export class LibSQLVector extends MastraVector {
|
|
|
156
169
|
}
|
|
157
170
|
}
|
|
158
171
|
|
|
159
|
-
async createIndex(
|
|
160
|
-
const params = this.normalizeArgs<CreateIndexParams>('createIndex', args);
|
|
161
|
-
|
|
162
|
-
const { indexName, dimension } = params;
|
|
172
|
+
async createIndex({ indexName, dimension }: CreateIndexParams): Promise<void> {
|
|
163
173
|
try {
|
|
164
174
|
// Validate inputs
|
|
165
|
-
if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
|
|
166
|
-
throw new Error('Invalid index name format');
|
|
167
|
-
}
|
|
168
175
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
169
176
|
throw new Error('Dimension must be a positive integer');
|
|
170
177
|
}
|
|
178
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
171
179
|
|
|
172
180
|
// Create the table with explicit schema
|
|
173
181
|
await this.turso.execute({
|
|
174
182
|
sql: `
|
|
175
|
-
CREATE TABLE IF NOT EXISTS ${
|
|
183
|
+
CREATE TABLE IF NOT EXISTS ${parsedIndexName} (
|
|
176
184
|
id SERIAL PRIMARY KEY,
|
|
177
185
|
vector_id TEXT UNIQUE NOT NULL,
|
|
178
186
|
embedding F32_BLOB(${dimension}),
|
|
@@ -184,8 +192,8 @@ export class LibSQLVector extends MastraVector {
|
|
|
184
192
|
|
|
185
193
|
await this.turso.execute({
|
|
186
194
|
sql: `
|
|
187
|
-
CREATE INDEX IF NOT EXISTS ${
|
|
188
|
-
ON ${
|
|
195
|
+
CREATE INDEX IF NOT EXISTS ${parsedIndexName}_vector_idx
|
|
196
|
+
ON ${parsedIndexName} (libsql_vector_idx(embedding))
|
|
189
197
|
`,
|
|
190
198
|
args: [],
|
|
191
199
|
});
|
|
@@ -197,11 +205,12 @@ export class LibSQLVector extends MastraVector {
|
|
|
197
205
|
}
|
|
198
206
|
}
|
|
199
207
|
|
|
200
|
-
async deleteIndex(indexName:
|
|
208
|
+
async deleteIndex({ indexName }: DeleteIndexParams): Promise<void> {
|
|
201
209
|
try {
|
|
210
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
202
211
|
// Drop the table
|
|
203
212
|
await this.turso.execute({
|
|
204
|
-
sql: `DROP TABLE IF EXISTS ${
|
|
213
|
+
sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
|
|
205
214
|
args: [],
|
|
206
215
|
});
|
|
207
216
|
} catch (error: any) {
|
|
@@ -229,8 +238,15 @@ export class LibSQLVector extends MastraVector {
|
|
|
229
238
|
}
|
|
230
239
|
}
|
|
231
240
|
|
|
232
|
-
|
|
241
|
+
/**
|
|
242
|
+
* Retrieves statistics about a vector index.
|
|
243
|
+
*
|
|
244
|
+
* @param {string} indexName - The name of the index to describe
|
|
245
|
+
* @returns A promise that resolves to the index statistics including dimension, count and metric
|
|
246
|
+
*/
|
|
247
|
+
async describeIndex({ indexName }: DescribeIndexParams): Promise<IndexStats> {
|
|
233
248
|
try {
|
|
249
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
234
250
|
// Get table info including column info
|
|
235
251
|
const tableInfoQuery = `
|
|
236
252
|
SELECT sql
|
|
@@ -240,11 +256,11 @@ export class LibSQLVector extends MastraVector {
|
|
|
240
256
|
`;
|
|
241
257
|
const tableInfo = await this.turso.execute({
|
|
242
258
|
sql: tableInfoQuery,
|
|
243
|
-
args: [
|
|
259
|
+
args: [parsedIndexName],
|
|
244
260
|
});
|
|
245
261
|
|
|
246
262
|
if (!tableInfo.rows[0]?.sql) {
|
|
247
|
-
throw new Error(`Table ${
|
|
263
|
+
throw new Error(`Table ${parsedIndexName} not found`);
|
|
248
264
|
}
|
|
249
265
|
|
|
250
266
|
// Extract dimension from F32_BLOB definition
|
|
@@ -253,7 +269,7 @@ export class LibSQLVector extends MastraVector {
|
|
|
253
269
|
// Get row count
|
|
254
270
|
const countQuery = `
|
|
255
271
|
SELECT COUNT(*) as count
|
|
256
|
-
FROM ${
|
|
272
|
+
FROM ${parsedIndexName};
|
|
257
273
|
`;
|
|
258
274
|
const countResult = await this.turso.execute({
|
|
259
275
|
sql: countQuery,
|
|
@@ -273,33 +289,9 @@ export class LibSQLVector extends MastraVector {
|
|
|
273
289
|
}
|
|
274
290
|
}
|
|
275
291
|
|
|
276
|
-
/**
|
|
277
|
-
* @deprecated Use {@link updateVector} instead. This method will be removed on May 20th, 2025.
|
|
278
|
-
*
|
|
279
|
-
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
280
|
-
* @param indexName - The name of the index containing the vector.
|
|
281
|
-
* @param id - The ID of the vector to update.
|
|
282
|
-
* @param update - An object containing the vector and/or metadata to update.
|
|
283
|
-
* @param update.vector - An optional array of numbers representing the new vector.
|
|
284
|
-
* @param update.metadata - An optional record containing the new metadata.
|
|
285
|
-
* @returns A promise that resolves when the update is complete.
|
|
286
|
-
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
287
|
-
*/
|
|
288
|
-
async updateIndexById(
|
|
289
|
-
indexName: string,
|
|
290
|
-
id: string,
|
|
291
|
-
update: { vector?: number[]; metadata?: Record<string, any> },
|
|
292
|
-
): Promise<void> {
|
|
293
|
-
this.logger.warn(
|
|
294
|
-
`Deprecation Warning: updateIndexById() is deprecated.
|
|
295
|
-
Please use updateVector() instead.
|
|
296
|
-
updateIndexById() will be removed on May 20th, 2025.`,
|
|
297
|
-
);
|
|
298
|
-
await this.updateVector(indexName, id, update);
|
|
299
|
-
}
|
|
300
|
-
|
|
301
292
|
/**
|
|
302
293
|
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
294
|
+
*
|
|
303
295
|
* @param indexName - The name of the index containing the vector.
|
|
304
296
|
* @param id - The ID of the vector to update.
|
|
305
297
|
* @param update - An object containing the vector and/or metadata to update.
|
|
@@ -308,12 +300,9 @@ export class LibSQLVector extends MastraVector {
|
|
|
308
300
|
* @returns A promise that resolves when the update is complete.
|
|
309
301
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
310
302
|
*/
|
|
311
|
-
async updateVector(
|
|
312
|
-
indexName: string,
|
|
313
|
-
id: string,
|
|
314
|
-
update: { vector?: number[]; metadata?: Record<string, any> },
|
|
315
|
-
): Promise<void> {
|
|
303
|
+
async updateVector({ indexName, id, update }: UpdateVectorParams): Promise<void> {
|
|
316
304
|
try {
|
|
305
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
317
306
|
const updates = [];
|
|
318
307
|
const args: InValue[] = [];
|
|
319
308
|
|
|
@@ -334,7 +323,7 @@ export class LibSQLVector extends MastraVector {
|
|
|
334
323
|
args.push(id);
|
|
335
324
|
|
|
336
325
|
const query = `
|
|
337
|
-
UPDATE ${
|
|
326
|
+
UPDATE ${parsedIndexName}
|
|
338
327
|
SET ${updates.join(', ')}
|
|
339
328
|
WHERE vector_id = ?;
|
|
340
329
|
`;
|
|
@@ -348,24 +337,6 @@ export class LibSQLVector extends MastraVector {
|
|
|
348
337
|
}
|
|
349
338
|
}
|
|
350
339
|
|
|
351
|
-
/**
|
|
352
|
-
* @deprecated Use {@link deleteVector} instead. This method will be removed on May 20th, 2025.
|
|
353
|
-
*
|
|
354
|
-
* Deletes a vector by its ID.
|
|
355
|
-
* @param indexName - The name of the index containing the vector.
|
|
356
|
-
* @param id - The ID of the vector to delete.
|
|
357
|
-
* @returns A promise that resolves when the deletion is complete.
|
|
358
|
-
* @throws Will throw an error if the deletion operation fails.
|
|
359
|
-
*/
|
|
360
|
-
async deleteIndexById(indexName: string, id: string): Promise<void> {
|
|
361
|
-
this.logger.warn(
|
|
362
|
-
`Deprecation Warning: deleteIndexById() is deprecated.
|
|
363
|
-
Please use deleteVector() instead.
|
|
364
|
-
deleteIndexById() will be removed on May 20th, 2025.`,
|
|
365
|
-
);
|
|
366
|
-
await this.deleteVector(indexName, id);
|
|
367
|
-
}
|
|
368
|
-
|
|
369
340
|
/**
|
|
370
341
|
* Deletes a vector by its ID.
|
|
371
342
|
* @param indexName - The name of the index containing the vector.
|
|
@@ -373,10 +344,11 @@ export class LibSQLVector extends MastraVector {
|
|
|
373
344
|
* @returns A promise that resolves when the deletion is complete.
|
|
374
345
|
* @throws Will throw an error if the deletion operation fails.
|
|
375
346
|
*/
|
|
376
|
-
async deleteVector(indexName
|
|
347
|
+
async deleteVector({ indexName, id }: DeleteVectorParams): Promise<void> {
|
|
377
348
|
try {
|
|
349
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
378
350
|
await this.turso.execute({
|
|
379
|
-
sql: `DELETE FROM ${
|
|
351
|
+
sql: `DELETE FROM ${parsedIndexName} WHERE vector_id = ?`,
|
|
380
352
|
args: [id],
|
|
381
353
|
});
|
|
382
354
|
} catch (error: any) {
|
|
@@ -384,9 +356,9 @@ export class LibSQLVector extends MastraVector {
|
|
|
384
356
|
}
|
|
385
357
|
}
|
|
386
358
|
|
|
387
|
-
async truncateIndex(indexName:
|
|
359
|
+
async truncateIndex({ indexName }: DeleteIndexParams): Promise<void> {
|
|
388
360
|
await this.turso.execute({
|
|
389
|
-
sql: `DELETE FROM ${indexName}`,
|
|
361
|
+
sql: `DELETE FROM ${parseSqlIdentifier(indexName, 'index name')}`,
|
|
390
362
|
args: [],
|
|
391
363
|
});
|
|
392
364
|
}
|