@mastra/libsql 0.0.4 → 0.0.5-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +15 -0
- package/dist/_tsup-dts-rollup.d.cts +18 -29
- package/dist/_tsup-dts-rollup.d.ts +18 -29
- package/dist/index.cjs +285 -184
- package/dist/index.js +285 -184
- package/package.json +2 -2
- package/src/storage/index.ts +18 -10
- package/src/vector/index.test.ts +154 -119
- package/src/vector/index.ts +63 -27
- package/src/vector/sql-builder.ts +243 -175
package/src/vector/index.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { createClient } from '@libsql/client';
|
|
2
2
|
import type { Client as TursoClient, InValue } from '@libsql/client';
|
|
3
|
+
|
|
4
|
+
import { parseSqlIdentifier } from '@mastra/core/utils';
|
|
3
5
|
import { MastraVector } from '@mastra/core/vector';
|
|
4
6
|
import type {
|
|
5
7
|
IndexStats,
|
|
@@ -9,6 +11,10 @@ import type {
|
|
|
9
11
|
UpsertVectorParams,
|
|
10
12
|
ParamsToArgs,
|
|
11
13
|
QueryVectorArgs,
|
|
14
|
+
DescribeIndexParams,
|
|
15
|
+
DeleteIndexParams,
|
|
16
|
+
DeleteVectorParams,
|
|
17
|
+
UpdateVectorParams,
|
|
12
18
|
} from '@mastra/core/vector';
|
|
13
19
|
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
14
20
|
import { LibSQLFilterTranslator } from './filter';
|
|
@@ -62,11 +68,21 @@ export class LibSQLVector extends MastraVector {
|
|
|
62
68
|
try {
|
|
63
69
|
const { indexName, queryVector, topK = 10, filter, includeVector = false, minScore = 0 } = params;
|
|
64
70
|
|
|
71
|
+
if (!Number.isInteger(topK) || topK <= 0) {
|
|
72
|
+
throw new Error('topK must be a positive integer');
|
|
73
|
+
}
|
|
74
|
+
if (!Array.isArray(queryVector) || !queryVector.every(x => typeof x === 'number' && Number.isFinite(x))) {
|
|
75
|
+
throw new Error('queryVector must be an array of finite numbers');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
79
|
+
|
|
65
80
|
const vectorStr = `[${queryVector.join(',')}]`;
|
|
66
81
|
|
|
67
82
|
const translatedFilter = this.transformFilter(filter);
|
|
68
83
|
const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter);
|
|
69
84
|
filterValues.push(minScore);
|
|
85
|
+
filterValues.push(topK);
|
|
70
86
|
|
|
71
87
|
const query = `
|
|
72
88
|
WITH vector_scores AS (
|
|
@@ -75,14 +91,14 @@ export class LibSQLVector extends MastraVector {
|
|
|
75
91
|
(1-vector_distance_cos(embedding, '${vectorStr}')) as score,
|
|
76
92
|
metadata
|
|
77
93
|
${includeVector ? ', vector_extract(embedding) as embedding' : ''}
|
|
78
|
-
FROM ${
|
|
94
|
+
FROM ${parsedIndexName}
|
|
79
95
|
${filterQuery}
|
|
80
96
|
)
|
|
81
97
|
SELECT *
|
|
82
98
|
FROM vector_scores
|
|
83
99
|
WHERE score > ?
|
|
84
100
|
ORDER BY score DESC
|
|
85
|
-
LIMIT
|
|
101
|
+
LIMIT ?`;
|
|
86
102
|
|
|
87
103
|
const result = await this.turso.execute({
|
|
88
104
|
sql: query,
|
|
@@ -107,11 +123,12 @@ export class LibSQLVector extends MastraVector {
|
|
|
107
123
|
const tx = await this.turso.transaction('write');
|
|
108
124
|
|
|
109
125
|
try {
|
|
126
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
110
127
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
111
128
|
|
|
112
129
|
for (let i = 0; i < vectors.length; i++) {
|
|
113
130
|
const query = `
|
|
114
|
-
INSERT INTO ${
|
|
131
|
+
INSERT INTO ${parsedIndexName} (vector_id, embedding, metadata)
|
|
115
132
|
VALUES (?, vector32(?), ?)
|
|
116
133
|
ON CONFLICT(vector_id) DO UPDATE SET
|
|
117
134
|
embedding = vector32(?),
|
|
@@ -162,17 +179,15 @@ export class LibSQLVector extends MastraVector {
|
|
|
162
179
|
const { indexName, dimension } = params;
|
|
163
180
|
try {
|
|
164
181
|
// Validate inputs
|
|
165
|
-
if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
|
|
166
|
-
throw new Error('Invalid index name format');
|
|
167
|
-
}
|
|
168
182
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
169
183
|
throw new Error('Dimension must be a positive integer');
|
|
170
184
|
}
|
|
185
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
171
186
|
|
|
172
187
|
// Create the table with explicit schema
|
|
173
188
|
await this.turso.execute({
|
|
174
189
|
sql: `
|
|
175
|
-
CREATE TABLE IF NOT EXISTS ${
|
|
190
|
+
CREATE TABLE IF NOT EXISTS ${parsedIndexName} (
|
|
176
191
|
id SERIAL PRIMARY KEY,
|
|
177
192
|
vector_id TEXT UNIQUE NOT NULL,
|
|
178
193
|
embedding F32_BLOB(${dimension}),
|
|
@@ -184,8 +199,8 @@ export class LibSQLVector extends MastraVector {
|
|
|
184
199
|
|
|
185
200
|
await this.turso.execute({
|
|
186
201
|
sql: `
|
|
187
|
-
CREATE INDEX IF NOT EXISTS ${
|
|
188
|
-
ON ${
|
|
202
|
+
CREATE INDEX IF NOT EXISTS ${parsedIndexName}_vector_idx
|
|
203
|
+
ON ${parsedIndexName} (libsql_vector_idx(embedding))
|
|
189
204
|
`,
|
|
190
205
|
args: [],
|
|
191
206
|
});
|
|
@@ -197,11 +212,15 @@ export class LibSQLVector extends MastraVector {
|
|
|
197
212
|
}
|
|
198
213
|
}
|
|
199
214
|
|
|
200
|
-
async deleteIndex(
|
|
215
|
+
async deleteIndex(...args: ParamsToArgs<DeleteIndexParams>): Promise<void> {
|
|
216
|
+
const params = this.normalizeArgs<DeleteIndexParams>('deleteIndex', args);
|
|
217
|
+
|
|
218
|
+
const { indexName } = params;
|
|
201
219
|
try {
|
|
220
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
202
221
|
// Drop the table
|
|
203
222
|
await this.turso.execute({
|
|
204
|
-
sql: `DROP TABLE IF EXISTS ${
|
|
223
|
+
sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
|
|
205
224
|
args: [],
|
|
206
225
|
});
|
|
207
226
|
} catch (error: any) {
|
|
@@ -229,8 +248,19 @@ export class LibSQLVector extends MastraVector {
|
|
|
229
248
|
}
|
|
230
249
|
}
|
|
231
250
|
|
|
232
|
-
|
|
251
|
+
/**
|
|
252
|
+
* Retrieves statistics about a vector index.
|
|
253
|
+
*
|
|
254
|
+
* @param params - The parameters for describing an index
|
|
255
|
+
* @param params.indexName - The name of the index to describe
|
|
256
|
+
* @returns A promise that resolves to the index statistics including dimension, count and metric
|
|
257
|
+
*/
|
|
258
|
+
async describeIndex(...args: ParamsToArgs<DescribeIndexParams>): Promise<IndexStats> {
|
|
259
|
+
const params = this.normalizeArgs<DescribeIndexParams>('describeIndex', args);
|
|
260
|
+
|
|
261
|
+
const { indexName } = params;
|
|
233
262
|
try {
|
|
263
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
234
264
|
// Get table info including column info
|
|
235
265
|
const tableInfoQuery = `
|
|
236
266
|
SELECT sql
|
|
@@ -240,11 +270,11 @@ export class LibSQLVector extends MastraVector {
|
|
|
240
270
|
`;
|
|
241
271
|
const tableInfo = await this.turso.execute({
|
|
242
272
|
sql: tableInfoQuery,
|
|
243
|
-
args: [
|
|
273
|
+
args: [parsedIndexName],
|
|
244
274
|
});
|
|
245
275
|
|
|
246
276
|
if (!tableInfo.rows[0]?.sql) {
|
|
247
|
-
throw new Error(`Table ${
|
|
277
|
+
throw new Error(`Table ${parsedIndexName} not found`);
|
|
248
278
|
}
|
|
249
279
|
|
|
250
280
|
// Extract dimension from F32_BLOB definition
|
|
@@ -253,7 +283,7 @@ export class LibSQLVector extends MastraVector {
|
|
|
253
283
|
// Get row count
|
|
254
284
|
const countQuery = `
|
|
255
285
|
SELECT COUNT(*) as count
|
|
256
|
-
FROM ${
|
|
286
|
+
FROM ${parsedIndexName};
|
|
257
287
|
`;
|
|
258
288
|
const countResult = await this.turso.execute({
|
|
259
289
|
sql: countQuery,
|
|
@@ -295,11 +325,12 @@ export class LibSQLVector extends MastraVector {
|
|
|
295
325
|
Please use updateVector() instead.
|
|
296
326
|
updateIndexById() will be removed on May 20th, 2025.`,
|
|
297
327
|
);
|
|
298
|
-
await this.updateVector(indexName, id, update);
|
|
328
|
+
await this.updateVector({ indexName, id, update });
|
|
299
329
|
}
|
|
300
330
|
|
|
301
331
|
/**
|
|
302
332
|
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
333
|
+
*
|
|
303
334
|
* @param indexName - The name of the index containing the vector.
|
|
304
335
|
* @param id - The ID of the vector to update.
|
|
305
336
|
* @param update - An object containing the vector and/or metadata to update.
|
|
@@ -308,12 +339,11 @@ export class LibSQLVector extends MastraVector {
|
|
|
308
339
|
* @returns A promise that resolves when the update is complete.
|
|
309
340
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
310
341
|
*/
|
|
311
|
-
async updateVector(
|
|
312
|
-
|
|
313
|
-
id
|
|
314
|
-
update: { vector?: number[]; metadata?: Record<string, any> },
|
|
315
|
-
): Promise<void> {
|
|
342
|
+
async updateVector(...args: ParamsToArgs<UpdateVectorParams>): Promise<void> {
|
|
343
|
+
const params = this.normalizeArgs<UpdateVectorParams>('updateVector', args);
|
|
344
|
+
const { indexName, id, update } = params;
|
|
316
345
|
try {
|
|
346
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
317
347
|
const updates = [];
|
|
318
348
|
const args: InValue[] = [];
|
|
319
349
|
|
|
@@ -334,7 +364,7 @@ export class LibSQLVector extends MastraVector {
|
|
|
334
364
|
args.push(id);
|
|
335
365
|
|
|
336
366
|
const query = `
|
|
337
|
-
UPDATE ${
|
|
367
|
+
UPDATE ${parsedIndexName}
|
|
338
368
|
SET ${updates.join(', ')}
|
|
339
369
|
WHERE vector_id = ?;
|
|
340
370
|
`;
|
|
@@ -363,7 +393,7 @@ export class LibSQLVector extends MastraVector {
|
|
|
363
393
|
Please use deleteVector() instead.
|
|
364
394
|
deleteIndexById() will be removed on May 20th, 2025.`,
|
|
365
395
|
);
|
|
366
|
-
await this.deleteVector(indexName, id);
|
|
396
|
+
await this.deleteVector({ indexName, id });
|
|
367
397
|
}
|
|
368
398
|
|
|
369
399
|
/**
|
|
@@ -373,10 +403,14 @@ export class LibSQLVector extends MastraVector {
|
|
|
373
403
|
* @returns A promise that resolves when the deletion is complete.
|
|
374
404
|
* @throws Will throw an error if the deletion operation fails.
|
|
375
405
|
*/
|
|
376
|
-
async deleteVector(
|
|
406
|
+
async deleteVector(...args: ParamsToArgs<DeleteVectorParams>): Promise<void> {
|
|
407
|
+
const params = this.normalizeArgs<DeleteVectorParams>('deleteVector', args);
|
|
408
|
+
|
|
409
|
+
const { indexName, id } = params;
|
|
377
410
|
try {
|
|
411
|
+
const parsedIndexName = parseSqlIdentifier(indexName, 'index name');
|
|
378
412
|
await this.turso.execute({
|
|
379
|
-
sql: `DELETE FROM ${
|
|
413
|
+
sql: `DELETE FROM ${parsedIndexName} WHERE vector_id = ?`,
|
|
380
414
|
args: [id],
|
|
381
415
|
});
|
|
382
416
|
} catch (error: any) {
|
|
@@ -384,9 +418,11 @@ export class LibSQLVector extends MastraVector {
|
|
|
384
418
|
}
|
|
385
419
|
}
|
|
386
420
|
|
|
387
|
-
async truncateIndex(
|
|
421
|
+
async truncateIndex(...args: ParamsToArgs<DeleteIndexParams>): Promise<void> {
|
|
422
|
+
const params = this.normalizeArgs<DeleteIndexParams>('truncateIndex', args);
|
|
423
|
+
const { indexName } = params;
|
|
388
424
|
await this.turso.execute({
|
|
389
|
-
sql: `DELETE FROM ${indexName}`,
|
|
425
|
+
sql: `DELETE FROM ${parseSqlIdentifier(indexName, 'index name')}`,
|
|
390
426
|
args: [],
|
|
391
427
|
});
|
|
392
428
|
}
|