@mastra/libsql 0.0.4 → 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +36 -0
- package/dist/_tsup-dts-rollup.d.cts +17 -59
- package/dist/_tsup-dts-rollup.d.ts +17 -59
- package/dist/index.cjs +277 -223
- package/dist/index.js +277 -223
- package/package.json +7 -4
- package/src/storage/index.ts +18 -10
- package/src/vector/index.test.ts +153 -216
- package/src/vector/index.ts +56 -84
- package/src/vector/sql-builder.ts +243 -175
package/dist/index.cjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var client = require('@libsql/client');
|
|
4
|
+
var utils = require('@mastra/core/utils');
|
|
4
5
|
var vector = require('@mastra/core/vector');
|
|
5
6
|
var filter = require('@mastra/core/vector/filter');
|
|
6
7
|
var storage = require('@mastra/core/storage');
|
|
@@ -72,28 +73,53 @@ var LibSQLFilterTranslator = class extends filter.BaseFilterTranslator {
|
|
|
72
73
|
// };
|
|
73
74
|
// }
|
|
74
75
|
};
|
|
75
|
-
|
|
76
|
-
// src/vector/sql-builder.ts
|
|
77
76
|
var createBasicOperator = (symbol) => {
|
|
78
|
-
return (key) =>
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
77
|
+
return (key, value) => {
|
|
78
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
79
|
+
return {
|
|
80
|
+
sql: `CASE
|
|
81
|
+
WHEN ? IS NULL THEN json_extract(metadata, '$."${jsonPathKey}"') IS ${symbol === "=" ? "" : "NOT"} NULL
|
|
82
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') ${symbol} ?
|
|
83
|
+
END`,
|
|
84
|
+
needsValue: true,
|
|
85
|
+
transformValue: () => {
|
|
86
|
+
return [value, value];
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
};
|
|
88
90
|
};
|
|
89
91
|
var createNumericOperator = (symbol) => {
|
|
90
|
-
return (key) =>
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
92
|
+
return (key) => {
|
|
93
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
94
|
+
return {
|
|
95
|
+
sql: `CAST(json_extract(metadata, '$."${jsonPathKey}"') AS NUMERIC) ${symbol} ?`,
|
|
96
|
+
needsValue: true
|
|
97
|
+
};
|
|
98
|
+
};
|
|
94
99
|
};
|
|
95
|
-
var validateJsonArray = (key) => `json_valid(json_extract(metadata, '$."${
|
|
96
|
-
AND json_type(json_extract(metadata, '$."${
|
|
100
|
+
var validateJsonArray = (key) => `json_valid(json_extract(metadata, '$."${key}"'))
|
|
101
|
+
AND json_type(json_extract(metadata, '$."${key}"')) = 'array'`;
|
|
102
|
+
var pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
103
|
+
function buildElemMatchConditions(value) {
|
|
104
|
+
const conditions = Object.entries(value).map(([field, fieldValue]) => {
|
|
105
|
+
if (field.startsWith("$")) {
|
|
106
|
+
const { sql, values } = buildCondition("elem.value", { [field]: fieldValue });
|
|
107
|
+
const elemSql = sql.replace(pattern, "elem.value");
|
|
108
|
+
return { sql: elemSql, values };
|
|
109
|
+
} else if (typeof fieldValue === "object" && !Array.isArray(fieldValue)) {
|
|
110
|
+
const { sql, values } = buildCondition(field, fieldValue);
|
|
111
|
+
const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
|
|
112
|
+
return { sql: elemSql, values };
|
|
113
|
+
} else {
|
|
114
|
+
const parsedFieldKey = utils.parseFieldKey(field);
|
|
115
|
+
return {
|
|
116
|
+
sql: `json_extract(elem.value, '$."${parsedFieldKey}"') = ?`,
|
|
117
|
+
values: [fieldValue]
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
return conditions;
|
|
122
|
+
}
|
|
97
123
|
var FILTER_OPERATORS = {
|
|
98
124
|
$eq: createBasicOperator("="),
|
|
99
125
|
$ne: createBasicOperator("!="),
|
|
@@ -102,90 +128,113 @@ var FILTER_OPERATORS = {
|
|
|
102
128
|
$lt: createNumericOperator("<"),
|
|
103
129
|
$lte: createNumericOperator("<="),
|
|
104
130
|
// Array Operators
|
|
105
|
-
$in: (key, value) =>
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
131
|
+
$in: (key, value) => {
|
|
132
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
133
|
+
const arr = Array.isArray(value) ? value : [value];
|
|
134
|
+
if (arr.length === 0) {
|
|
135
|
+
return { sql: "1 = 0", needsValue: true, transformValue: () => [] };
|
|
136
|
+
}
|
|
137
|
+
const paramPlaceholders = arr.map(() => "?").join(",");
|
|
138
|
+
return {
|
|
139
|
+
sql: `(
|
|
140
|
+
CASE
|
|
141
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
142
|
+
EXISTS (
|
|
143
|
+
SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
144
|
+
WHERE elem.value IN (SELECT value FROM json_each(?))
|
|
145
|
+
)
|
|
146
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') IN (${paramPlaceholders})
|
|
147
|
+
END
|
|
148
|
+
)`,
|
|
149
|
+
needsValue: true,
|
|
150
|
+
transformValue: () => [JSON.stringify(arr), ...arr]
|
|
151
|
+
};
|
|
152
|
+
},
|
|
153
|
+
$nin: (key, value) => {
|
|
154
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
155
|
+
const arr = Array.isArray(value) ? value : [value];
|
|
156
|
+
if (arr.length === 0) {
|
|
157
|
+
return { sql: "1 = 1", needsValue: true, transformValue: () => [] };
|
|
158
|
+
}
|
|
159
|
+
const paramPlaceholders = arr.map(() => "?").join(",");
|
|
160
|
+
return {
|
|
161
|
+
sql: `(
|
|
162
|
+
CASE
|
|
163
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
164
|
+
NOT EXISTS (
|
|
165
|
+
SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
166
|
+
WHERE elem.value IN (SELECT value FROM json_each(?))
|
|
167
|
+
)
|
|
168
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') NOT IN (${paramPlaceholders})
|
|
169
|
+
END
|
|
170
|
+
)`,
|
|
171
|
+
needsValue: true,
|
|
172
|
+
transformValue: () => [JSON.stringify(arr), ...arr]
|
|
173
|
+
};
|
|
174
|
+
},
|
|
175
|
+
$all: (key, value) => {
|
|
176
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
177
|
+
let sql;
|
|
178
|
+
const arrayValue = Array.isArray(value) ? value : [value];
|
|
179
|
+
if (arrayValue.length === 0) {
|
|
180
|
+
sql = "1 = 0";
|
|
181
|
+
} else {
|
|
182
|
+
sql = `(
|
|
183
|
+
CASE
|
|
184
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
185
|
+
NOT EXISTS (
|
|
186
|
+
SELECT value
|
|
187
|
+
FROM json_each(?)
|
|
188
|
+
WHERE value NOT IN (
|
|
189
|
+
SELECT value
|
|
190
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"'))
|
|
135
191
|
)
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
};
|
|
192
|
+
)
|
|
193
|
+
ELSE FALSE
|
|
194
|
+
END
|
|
195
|
+
)`;
|
|
141
196
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
throw new Error("$elemMatch requires an object with conditions");
|
|
149
|
-
}
|
|
150
|
-
const conditions = Object.entries(value).map(([field, fieldValue]) => {
|
|
151
|
-
if (field.startsWith("$")) {
|
|
152
|
-
const { sql, values } = buildCondition("elem.value", { [field]: fieldValue });
|
|
153
|
-
const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
154
|
-
const elemSql = sql.replace(pattern, "elem.value");
|
|
155
|
-
return { sql: elemSql, values };
|
|
156
|
-
} else if (typeof fieldValue === "object" && !Array.isArray(fieldValue)) {
|
|
157
|
-
const { sql, values } = buildCondition(field, fieldValue);
|
|
158
|
-
const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
159
|
-
const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
|
|
160
|
-
return { sql: elemSql, values };
|
|
161
|
-
} else {
|
|
162
|
-
return {
|
|
163
|
-
sql: `json_extract(elem.value, '$."${field}"') = ?`,
|
|
164
|
-
values: [fieldValue]
|
|
165
|
-
};
|
|
197
|
+
return {
|
|
198
|
+
sql,
|
|
199
|
+
needsValue: true,
|
|
200
|
+
transformValue: () => {
|
|
201
|
+
if (arrayValue.length === 0) {
|
|
202
|
+
return [];
|
|
166
203
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
WHERE ${conditions.map((c) => c.sql).join(" AND ")}
|
|
176
|
-
)
|
|
177
|
-
ELSE FALSE
|
|
178
|
-
END
|
|
179
|
-
)`,
|
|
180
|
-
values: conditions.flatMap((c) => c.values)
|
|
181
|
-
};
|
|
204
|
+
return [JSON.stringify(arrayValue)];
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
},
|
|
208
|
+
$elemMatch: (key, value) => {
|
|
209
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
210
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
211
|
+
throw new Error("$elemMatch requires an object with conditions");
|
|
182
212
|
}
|
|
183
|
-
|
|
213
|
+
const conditions = buildElemMatchConditions(value);
|
|
214
|
+
return {
|
|
215
|
+
sql: `(
|
|
216
|
+
CASE
|
|
217
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
218
|
+
EXISTS (
|
|
219
|
+
SELECT 1
|
|
220
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
221
|
+
WHERE ${conditions.map((c) => c.sql).join(" AND ")}
|
|
222
|
+
)
|
|
223
|
+
ELSE FALSE
|
|
224
|
+
END
|
|
225
|
+
)`,
|
|
226
|
+
needsValue: true,
|
|
227
|
+
transformValue: () => conditions.flatMap((c) => c.values)
|
|
228
|
+
};
|
|
229
|
+
},
|
|
184
230
|
// Element Operators
|
|
185
|
-
$exists: (key) =>
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
231
|
+
$exists: (key) => {
|
|
232
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
233
|
+
return {
|
|
234
|
+
sql: `json_extract(metadata, '$."${jsonPathKey}"') IS NOT NULL`,
|
|
235
|
+
needsValue: false
|
|
236
|
+
};
|
|
237
|
+
},
|
|
189
238
|
// Logical Operators
|
|
190
239
|
$and: (key) => ({
|
|
191
240
|
sql: `(${key})`,
|
|
@@ -200,27 +249,30 @@ var FILTER_OPERATORS = {
|
|
|
200
249
|
sql: `NOT (${key})`,
|
|
201
250
|
needsValue: false
|
|
202
251
|
}),
|
|
203
|
-
$size: (key, paramIndex) =>
|
|
204
|
-
|
|
252
|
+
$size: (key, paramIndex) => {
|
|
253
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
254
|
+
return {
|
|
255
|
+
sql: `(
|
|
205
256
|
CASE
|
|
206
|
-
WHEN json_type(json_extract(metadata, '$."${
|
|
207
|
-
json_array_length(json_extract(metadata, '$."${
|
|
257
|
+
WHEN json_type(json_extract(metadata, '$."${jsonPathKey}"')) = 'array' THEN
|
|
258
|
+
json_array_length(json_extract(metadata, '$."${jsonPathKey}"')) = $${paramIndex}
|
|
208
259
|
ELSE FALSE
|
|
209
260
|
END
|
|
210
261
|
)`,
|
|
211
|
-
|
|
212
|
-
|
|
262
|
+
needsValue: true
|
|
263
|
+
};
|
|
264
|
+
},
|
|
213
265
|
// /**
|
|
214
266
|
// * Regex Operators
|
|
215
267
|
// * Supports case insensitive and multiline
|
|
216
268
|
// */
|
|
217
269
|
// $regex: (key: string): FilterOperator => ({
|
|
218
|
-
// sql: `json_extract(metadata, '$."${
|
|
270
|
+
// sql: `json_extract(metadata, '$."${toJsonPathKey(key)}"') = ?`,
|
|
219
271
|
// needsValue: true,
|
|
220
272
|
// transformValue: (value: any) => {
|
|
221
273
|
// const pattern = typeof value === 'object' ? value.$regex : value;
|
|
222
274
|
// const options = typeof value === 'object' ? value.$options || '' : '';
|
|
223
|
-
// let sql = `json_extract(metadata, '$."${
|
|
275
|
+
// let sql = `json_extract(metadata, '$."${toJsonPathKey(key)}"')`;
|
|
224
276
|
// // Handle multiline
|
|
225
277
|
// // if (options.includes('m')) {
|
|
226
278
|
// // sql = `REPLACE(${sql}, CHAR(10), '\n')`;
|
|
@@ -269,50 +321,64 @@ var FILTER_OPERATORS = {
|
|
|
269
321
|
// };
|
|
270
322
|
// },
|
|
271
323
|
// }),
|
|
272
|
-
$contains: (key) =>
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
}
|
|
289
|
-
if (value && typeof value === "object") {
|
|
290
|
-
let traverse2 = function(obj, path = []) {
|
|
291
|
-
for (const [k, v] of Object.entries(obj)) {
|
|
292
|
-
const currentPath = [...path, k];
|
|
293
|
-
if (v && typeof v === "object" && !Array.isArray(v)) {
|
|
294
|
-
traverse2(v, currentPath);
|
|
295
|
-
} else {
|
|
296
|
-
paths.push(currentPath.join("."));
|
|
297
|
-
values.push(v);
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
};
|
|
301
|
-
const paths = [];
|
|
302
|
-
const values = [];
|
|
303
|
-
traverse2(value);
|
|
304
|
-
return {
|
|
305
|
-
sql: `(${paths.map((path) => `json_extract(metadata, '$."${handleKey(key)}"."${path}"') = ?`).join(" AND ")})`,
|
|
306
|
-
values
|
|
307
|
-
};
|
|
308
|
-
}
|
|
309
|
-
return value;
|
|
324
|
+
$contains: (key, value) => {
|
|
325
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
326
|
+
let sql;
|
|
327
|
+
if (Array.isArray(value)) {
|
|
328
|
+
sql = `(
|
|
329
|
+
SELECT ${validateJsonArray(jsonPathKey)}
|
|
330
|
+
AND EXISTS (
|
|
331
|
+
SELECT 1
|
|
332
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as m
|
|
333
|
+
WHERE m.value IN (SELECT value FROM json_each(?))
|
|
334
|
+
)
|
|
335
|
+
)`;
|
|
336
|
+
} else if (typeof value === "string") {
|
|
337
|
+
sql = `lower(json_extract(metadata, '$."${jsonPathKey}"')) LIKE '%' || lower(?) || '%' ESCAPE '\\'`;
|
|
338
|
+
} else {
|
|
339
|
+
sql = `json_extract(metadata, '$."${jsonPathKey}"') = ?`;
|
|
310
340
|
}
|
|
311
|
-
|
|
341
|
+
return {
|
|
342
|
+
sql,
|
|
343
|
+
needsValue: true,
|
|
344
|
+
transformValue: () => {
|
|
345
|
+
if (Array.isArray(value)) {
|
|
346
|
+
return [JSON.stringify(value)];
|
|
347
|
+
}
|
|
348
|
+
if (typeof value === "object" && value !== null) {
|
|
349
|
+
return [JSON.stringify(value)];
|
|
350
|
+
}
|
|
351
|
+
if (typeof value === "string") {
|
|
352
|
+
return [escapeLikePattern(value)];
|
|
353
|
+
}
|
|
354
|
+
return [value];
|
|
355
|
+
}
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* $objectContains: True JSON containment for advanced use (deep sub-object match).
|
|
360
|
+
* Usage: { field: { $objectContains: { ...subobject } } }
|
|
361
|
+
*/
|
|
362
|
+
// $objectContains: (key: string) => ({
|
|
363
|
+
// sql: '', // Will be overridden by transformValue
|
|
364
|
+
// needsValue: true,
|
|
365
|
+
// transformValue: (value: any) => ({
|
|
366
|
+
// sql: `json_type(json_extract(metadata, '$."${toJsonPathKey(key)}"')) = 'object'
|
|
367
|
+
// AND json_patch(json_extract(metadata, '$."${toJsonPathKey(key)}"'), ?) = json_extract(metadata, '$."${toJsonPathKey(key)}"')`,
|
|
368
|
+
// values: [JSON.stringify(value)],
|
|
369
|
+
// }),
|
|
370
|
+
// }),
|
|
312
371
|
};
|
|
313
|
-
|
|
314
|
-
return
|
|
372
|
+
function isFilterResult(obj) {
|
|
373
|
+
return obj && typeof obj === "object" && typeof obj.sql === "string" && Array.isArray(obj.values);
|
|
374
|
+
}
|
|
375
|
+
var parseJsonPathKey = (key) => {
|
|
376
|
+
const parsedKey = utils.parseFieldKey(key);
|
|
377
|
+
return parsedKey.replace(/\./g, '"."');
|
|
315
378
|
};
|
|
379
|
+
function escapeLikePattern(str) {
|
|
380
|
+
return str.replace(/([%_\\])/g, "\\$1");
|
|
381
|
+
}
|
|
316
382
|
function buildFilterQuery(filter) {
|
|
317
383
|
if (!filter) {
|
|
318
384
|
return { sql: "", values: [] };
|
|
@@ -405,8 +471,8 @@ var processOperator = (key, operator, operatorValue) => {
|
|
|
405
471
|
if (!operatorResult.needsValue) {
|
|
406
472
|
return { sql: operatorResult.sql, values: [] };
|
|
407
473
|
}
|
|
408
|
-
const transformed = operatorResult.transformValue ? operatorResult.transformValue(
|
|
409
|
-
if (transformed
|
|
474
|
+
const transformed = operatorResult.transformValue ? operatorResult.transformValue() : operatorValue;
|
|
475
|
+
if (isFilterResult(transformed)) {
|
|
410
476
|
return transformed;
|
|
411
477
|
}
|
|
412
478
|
return {
|
|
@@ -442,14 +508,27 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
442
508
|
const translator = new LibSQLFilterTranslator();
|
|
443
509
|
return translator.translate(filter);
|
|
444
510
|
}
|
|
445
|
-
async query(
|
|
446
|
-
|
|
511
|
+
async query({
|
|
512
|
+
indexName,
|
|
513
|
+
queryVector,
|
|
514
|
+
topK = 10,
|
|
515
|
+
filter,
|
|
516
|
+
includeVector = false,
|
|
517
|
+
minScore = 0
|
|
518
|
+
}) {
|
|
447
519
|
try {
|
|
448
|
-
|
|
520
|
+
if (!Number.isInteger(topK) || topK <= 0) {
|
|
521
|
+
throw new Error("topK must be a positive integer");
|
|
522
|
+
}
|
|
523
|
+
if (!Array.isArray(queryVector) || !queryVector.every((x) => typeof x === "number" && Number.isFinite(x))) {
|
|
524
|
+
throw new Error("queryVector must be an array of finite numbers");
|
|
525
|
+
}
|
|
526
|
+
const parsedIndexName = utils.parseSqlIdentifier(indexName, "index name");
|
|
449
527
|
const vectorStr = `[${queryVector.join(",")}]`;
|
|
450
528
|
const translatedFilter = this.transformFilter(filter);
|
|
451
529
|
const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter);
|
|
452
530
|
filterValues.push(minScore);
|
|
531
|
+
filterValues.push(topK);
|
|
453
532
|
const query = `
|
|
454
533
|
WITH vector_scores AS (
|
|
455
534
|
SELECT
|
|
@@ -457,14 +536,14 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
457
536
|
(1-vector_distance_cos(embedding, '${vectorStr}')) as score,
|
|
458
537
|
metadata
|
|
459
538
|
${includeVector ? ", vector_extract(embedding) as embedding" : ""}
|
|
460
|
-
FROM ${
|
|
539
|
+
FROM ${parsedIndexName}
|
|
461
540
|
${filterQuery}
|
|
462
541
|
)
|
|
463
542
|
SELECT *
|
|
464
543
|
FROM vector_scores
|
|
465
544
|
WHERE score > ?
|
|
466
545
|
ORDER BY score DESC
|
|
467
|
-
LIMIT
|
|
546
|
+
LIMIT ?`;
|
|
468
547
|
const result = await this.turso.execute({
|
|
469
548
|
sql: query,
|
|
470
549
|
args: filterValues
|
|
@@ -478,15 +557,14 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
478
557
|
} finally {
|
|
479
558
|
}
|
|
480
559
|
}
|
|
481
|
-
async upsert(
|
|
482
|
-
const params = this.normalizeArgs("upsert", args);
|
|
483
|
-
const { indexName, vectors, metadata, ids } = params;
|
|
560
|
+
async upsert({ indexName, vectors, metadata, ids }) {
|
|
484
561
|
const tx = await this.turso.transaction("write");
|
|
485
562
|
try {
|
|
563
|
+
const parsedIndexName = utils.parseSqlIdentifier(indexName, "index name");
|
|
486
564
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
487
565
|
for (let i = 0; i < vectors.length; i++) {
|
|
488
566
|
const query = `
|
|
489
|
-
INSERT INTO ${
|
|
567
|
+
INSERT INTO ${parsedIndexName} (vector_id, embedding, metadata)
|
|
490
568
|
VALUES (?, vector32(?), ?)
|
|
491
569
|
ON CONFLICT(vector_id) DO UPDATE SET
|
|
492
570
|
embedding = vector32(?),
|
|
@@ -520,19 +598,15 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
520
598
|
throw error;
|
|
521
599
|
}
|
|
522
600
|
}
|
|
523
|
-
async createIndex(
|
|
524
|
-
const params = this.normalizeArgs("createIndex", args);
|
|
525
|
-
const { indexName, dimension } = params;
|
|
601
|
+
async createIndex({ indexName, dimension }) {
|
|
526
602
|
try {
|
|
527
|
-
if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
|
|
528
|
-
throw new Error("Invalid index name format");
|
|
529
|
-
}
|
|
530
603
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
531
604
|
throw new Error("Dimension must be a positive integer");
|
|
532
605
|
}
|
|
606
|
+
const parsedIndexName = utils.parseSqlIdentifier(indexName, "index name");
|
|
533
607
|
await this.turso.execute({
|
|
534
608
|
sql: `
|
|
535
|
-
CREATE TABLE IF NOT EXISTS ${
|
|
609
|
+
CREATE TABLE IF NOT EXISTS ${parsedIndexName} (
|
|
536
610
|
id SERIAL PRIMARY KEY,
|
|
537
611
|
vector_id TEXT UNIQUE NOT NULL,
|
|
538
612
|
embedding F32_BLOB(${dimension}),
|
|
@@ -543,8 +617,8 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
543
617
|
});
|
|
544
618
|
await this.turso.execute({
|
|
545
619
|
sql: `
|
|
546
|
-
CREATE INDEX IF NOT EXISTS ${
|
|
547
|
-
ON ${
|
|
620
|
+
CREATE INDEX IF NOT EXISTS ${parsedIndexName}_vector_idx
|
|
621
|
+
ON ${parsedIndexName} (libsql_vector_idx(embedding))
|
|
548
622
|
`,
|
|
549
623
|
args: []
|
|
550
624
|
});
|
|
@@ -554,10 +628,11 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
554
628
|
} finally {
|
|
555
629
|
}
|
|
556
630
|
}
|
|
557
|
-
async deleteIndex(indexName) {
|
|
631
|
+
async deleteIndex({ indexName }) {
|
|
558
632
|
try {
|
|
633
|
+
const parsedIndexName = utils.parseSqlIdentifier(indexName, "index name");
|
|
559
634
|
await this.turso.execute({
|
|
560
|
-
sql: `DROP TABLE IF EXISTS ${
|
|
635
|
+
sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
|
|
561
636
|
args: []
|
|
562
637
|
});
|
|
563
638
|
} catch (error) {
|
|
@@ -582,8 +657,15 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
582
657
|
throw new Error(`Failed to list vector tables: ${error.message}`);
|
|
583
658
|
}
|
|
584
659
|
}
|
|
585
|
-
|
|
660
|
+
/**
|
|
661
|
+
* Retrieves statistics about a vector index.
|
|
662
|
+
*
|
|
663
|
+
* @param {string} indexName - The name of the index to describe
|
|
664
|
+
* @returns A promise that resolves to the index statistics including dimension, count and metric
|
|
665
|
+
*/
|
|
666
|
+
async describeIndex({ indexName }) {
|
|
586
667
|
try {
|
|
668
|
+
const parsedIndexName = utils.parseSqlIdentifier(indexName, "index name");
|
|
587
669
|
const tableInfoQuery = `
|
|
588
670
|
SELECT sql
|
|
589
671
|
FROM sqlite_master
|
|
@@ -592,15 +674,15 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
592
674
|
`;
|
|
593
675
|
const tableInfo = await this.turso.execute({
|
|
594
676
|
sql: tableInfoQuery,
|
|
595
|
-
args: [
|
|
677
|
+
args: [parsedIndexName]
|
|
596
678
|
});
|
|
597
679
|
if (!tableInfo.rows[0]?.sql) {
|
|
598
|
-
throw new Error(`Table ${
|
|
680
|
+
throw new Error(`Table ${parsedIndexName} not found`);
|
|
599
681
|
}
|
|
600
682
|
const dimension = parseInt(tableInfo.rows[0].sql.match(/F32_BLOB\((\d+)\)/)?.[1] || "0");
|
|
601
683
|
const countQuery = `
|
|
602
684
|
SELECT COUNT(*) as count
|
|
603
|
-
FROM ${
|
|
685
|
+
FROM ${parsedIndexName};
|
|
604
686
|
`;
|
|
605
687
|
const countResult = await this.turso.execute({
|
|
606
688
|
sql: countQuery,
|
|
@@ -616,28 +698,9 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
616
698
|
throw new Error(`Failed to describe vector table: ${e.message}`);
|
|
617
699
|
}
|
|
618
700
|
}
|
|
619
|
-
/**
|
|
620
|
-
* @deprecated Use {@link updateVector} instead. This method will be removed on May 20th, 2025.
|
|
621
|
-
*
|
|
622
|
-
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
623
|
-
* @param indexName - The name of the index containing the vector.
|
|
624
|
-
* @param id - The ID of the vector to update.
|
|
625
|
-
* @param update - An object containing the vector and/or metadata to update.
|
|
626
|
-
* @param update.vector - An optional array of numbers representing the new vector.
|
|
627
|
-
* @param update.metadata - An optional record containing the new metadata.
|
|
628
|
-
* @returns A promise that resolves when the update is complete.
|
|
629
|
-
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
630
|
-
*/
|
|
631
|
-
async updateIndexById(indexName, id, update) {
|
|
632
|
-
this.logger.warn(
|
|
633
|
-
`Deprecation Warning: updateIndexById() is deprecated.
|
|
634
|
-
Please use updateVector() instead.
|
|
635
|
-
updateIndexById() will be removed on May 20th, 2025.`
|
|
636
|
-
);
|
|
637
|
-
await this.updateVector(indexName, id, update);
|
|
638
|
-
}
|
|
639
701
|
/**
|
|
640
702
|
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
703
|
+
*
|
|
641
704
|
* @param indexName - The name of the index containing the vector.
|
|
642
705
|
* @param id - The ID of the vector to update.
|
|
643
706
|
* @param update - An object containing the vector and/or metadata to update.
|
|
@@ -646,8 +709,9 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
646
709
|
* @returns A promise that resolves when the update is complete.
|
|
647
710
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
648
711
|
*/
|
|
649
|
-
async updateVector(indexName, id, update) {
|
|
712
|
+
async updateVector({ indexName, id, update }) {
|
|
650
713
|
try {
|
|
714
|
+
const parsedIndexName = utils.parseSqlIdentifier(indexName, "index name");
|
|
651
715
|
const updates = [];
|
|
652
716
|
const args = [];
|
|
653
717
|
if (update.vector) {
|
|
@@ -663,7 +727,7 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
663
727
|
}
|
|
664
728
|
args.push(id);
|
|
665
729
|
const query = `
|
|
666
|
-
UPDATE ${
|
|
730
|
+
UPDATE ${parsedIndexName}
|
|
667
731
|
SET ${updates.join(", ")}
|
|
668
732
|
WHERE vector_id = ?;
|
|
669
733
|
`;
|
|
@@ -675,23 +739,6 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
675
739
|
throw new Error(`Failed to update vector by id: ${id} for index: ${indexName}: ${error.message}`);
|
|
676
740
|
}
|
|
677
741
|
}
|
|
678
|
-
/**
|
|
679
|
-
* @deprecated Use {@link deleteVector} instead. This method will be removed on May 20th, 2025.
|
|
680
|
-
*
|
|
681
|
-
* Deletes a vector by its ID.
|
|
682
|
-
* @param indexName - The name of the index containing the vector.
|
|
683
|
-
* @param id - The ID of the vector to delete.
|
|
684
|
-
* @returns A promise that resolves when the deletion is complete.
|
|
685
|
-
* @throws Will throw an error if the deletion operation fails.
|
|
686
|
-
*/
|
|
687
|
-
async deleteIndexById(indexName, id) {
|
|
688
|
-
this.logger.warn(
|
|
689
|
-
`Deprecation Warning: deleteIndexById() is deprecated.
|
|
690
|
-
Please use deleteVector() instead.
|
|
691
|
-
deleteIndexById() will be removed on May 20th, 2025.`
|
|
692
|
-
);
|
|
693
|
-
await this.deleteVector(indexName, id);
|
|
694
|
-
}
|
|
695
742
|
/**
|
|
696
743
|
* Deletes a vector by its ID.
|
|
697
744
|
* @param indexName - The name of the index containing the vector.
|
|
@@ -699,19 +746,20 @@ var LibSQLVector = class extends vector.MastraVector {
|
|
|
699
746
|
* @returns A promise that resolves when the deletion is complete.
|
|
700
747
|
* @throws Will throw an error if the deletion operation fails.
|
|
701
748
|
*/
|
|
702
|
-
async deleteVector(indexName, id) {
|
|
749
|
+
async deleteVector({ indexName, id }) {
|
|
703
750
|
try {
|
|
751
|
+
const parsedIndexName = utils.parseSqlIdentifier(indexName, "index name");
|
|
704
752
|
await this.turso.execute({
|
|
705
|
-
sql: `DELETE FROM ${
|
|
753
|
+
sql: `DELETE FROM ${parsedIndexName} WHERE vector_id = ?`,
|
|
706
754
|
args: [id]
|
|
707
755
|
});
|
|
708
756
|
} catch (error) {
|
|
709
757
|
throw new Error(`Failed to delete vector by id: ${id} for index: ${indexName}: ${error.message}`);
|
|
710
758
|
}
|
|
711
759
|
}
|
|
712
|
-
async truncateIndex(indexName) {
|
|
760
|
+
async truncateIndex({ indexName }) {
|
|
713
761
|
await this.turso.execute({
|
|
714
|
-
sql: `DELETE FROM ${indexName}`,
|
|
762
|
+
sql: `DELETE FROM ${utils.parseSqlIdentifier(indexName, "index name")}`,
|
|
715
763
|
args: []
|
|
716
764
|
});
|
|
717
765
|
}
|
|
@@ -733,22 +781,24 @@ var LibSQLStore = class extends storage.MastraStorage {
|
|
|
733
781
|
this.client = client.createClient(config);
|
|
734
782
|
}
|
|
735
783
|
getCreateTableSQL(tableName, schema) {
|
|
784
|
+
const parsedTableName = utils.parseSqlIdentifier(tableName, "table name");
|
|
736
785
|
const columns = Object.entries(schema).map(([name, col]) => {
|
|
786
|
+
const parsedColumnName = utils.parseSqlIdentifier(name, "column name");
|
|
737
787
|
let type = col.type.toUpperCase();
|
|
738
788
|
if (type === "TEXT") type = "TEXT";
|
|
739
789
|
if (type === "TIMESTAMP") type = "TEXT";
|
|
740
790
|
const nullable = col.nullable ? "" : "NOT NULL";
|
|
741
791
|
const primaryKey = col.primaryKey ? "PRIMARY KEY" : "";
|
|
742
|
-
return `${
|
|
792
|
+
return `${parsedColumnName} ${type} ${nullable} ${primaryKey}`.trim();
|
|
743
793
|
});
|
|
744
794
|
if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
|
|
745
|
-
const stmnt = `CREATE TABLE IF NOT EXISTS ${
|
|
795
|
+
const stmnt = `CREATE TABLE IF NOT EXISTS ${parsedTableName} (
|
|
746
796
|
${columns.join(",\n")},
|
|
747
797
|
PRIMARY KEY (workflow_name, run_id)
|
|
748
798
|
)`;
|
|
749
799
|
return stmnt;
|
|
750
800
|
}
|
|
751
|
-
return `CREATE TABLE IF NOT EXISTS ${
|
|
801
|
+
return `CREATE TABLE IF NOT EXISTS ${parsedTableName} (${columns.join(", ")})`;
|
|
752
802
|
}
|
|
753
803
|
async createTable({
|
|
754
804
|
tableName,
|
|
@@ -764,8 +814,9 @@ var LibSQLStore = class extends storage.MastraStorage {
|
|
|
764
814
|
}
|
|
765
815
|
}
|
|
766
816
|
async clearTable({ tableName }) {
|
|
817
|
+
const parsedTableName = utils.parseSqlIdentifier(tableName, "table name");
|
|
767
818
|
try {
|
|
768
|
-
await this.client.execute(`DELETE FROM ${
|
|
819
|
+
await this.client.execute(`DELETE FROM ${parsedTableName}`);
|
|
769
820
|
} catch (e) {
|
|
770
821
|
if (e instanceof Error) {
|
|
771
822
|
this.logger.error(e.message);
|
|
@@ -773,7 +824,8 @@ var LibSQLStore = class extends storage.MastraStorage {
|
|
|
773
824
|
}
|
|
774
825
|
}
|
|
775
826
|
prepareStatement({ tableName, record }) {
|
|
776
|
-
const
|
|
827
|
+
const parsedTableName = utils.parseSqlIdentifier(tableName, "table name");
|
|
828
|
+
const columns = Object.keys(record).map((col) => utils.parseSqlIdentifier(col, "column name"));
|
|
777
829
|
const values = Object.values(record).map((v) => {
|
|
778
830
|
if (typeof v === `undefined`) {
|
|
779
831
|
return null;
|
|
@@ -785,7 +837,7 @@ var LibSQLStore = class extends storage.MastraStorage {
|
|
|
785
837
|
});
|
|
786
838
|
const placeholders = values.map(() => "?").join(", ");
|
|
787
839
|
return {
|
|
788
|
-
sql: `INSERT OR REPLACE INTO ${
|
|
840
|
+
sql: `INSERT OR REPLACE INTO ${parsedTableName} (${columns.join(", ")}) VALUES (${placeholders})`,
|
|
789
841
|
args: values
|
|
790
842
|
};
|
|
791
843
|
}
|
|
@@ -813,10 +865,12 @@ var LibSQLStore = class extends storage.MastraStorage {
|
|
|
813
865
|
}
|
|
814
866
|
}
|
|
815
867
|
async load({ tableName, keys }) {
|
|
816
|
-
const
|
|
868
|
+
const parsedTableName = utils.parseSqlIdentifier(tableName, "table name");
|
|
869
|
+
const parsedKeys = Object.keys(keys).map((key) => utils.parseSqlIdentifier(key, "column name"));
|
|
870
|
+
const conditions = parsedKeys.map((key) => `${key} = ?`).join(" AND ");
|
|
817
871
|
const values = Object.values(keys);
|
|
818
872
|
const result = await this.client.execute({
|
|
819
|
-
sql: `SELECT * FROM ${
|
|
873
|
+
sql: `SELECT * FROM ${parsedTableName} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
820
874
|
args: values
|
|
821
875
|
});
|
|
822
876
|
if (!result.rows || result.rows.length === 0) {
|