@mastra/libsql 0.0.4 → 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +36 -0
- package/dist/_tsup-dts-rollup.d.cts +17 -59
- package/dist/_tsup-dts-rollup.d.ts +17 -59
- package/dist/index.cjs +277 -223
- package/dist/index.js +277 -223
- package/package.json +7 -4
- package/src/storage/index.ts +18 -10
- package/src/vector/index.test.ts +153 -216
- package/src/vector/index.ts +56 -84
- package/src/vector/sql-builder.ts +243 -175
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createClient } from '@libsql/client';
|
|
2
|
+
import { parseSqlIdentifier, parseFieldKey } from '@mastra/core/utils';
|
|
2
3
|
import { MastraVector } from '@mastra/core/vector';
|
|
3
4
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
4
5
|
import { MastraStorage, TABLE_WORKFLOW_SNAPSHOT, TABLE_THREADS, TABLE_MESSAGES, TABLE_EVALS, TABLE_TRACES } from '@mastra/core/storage';
|
|
@@ -70,28 +71,53 @@ var LibSQLFilterTranslator = class extends BaseFilterTranslator {
|
|
|
70
71
|
// };
|
|
71
72
|
// }
|
|
72
73
|
};
|
|
73
|
-
|
|
74
|
-
// src/vector/sql-builder.ts
|
|
75
74
|
var createBasicOperator = (symbol) => {
|
|
76
|
-
return (key) =>
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
75
|
+
return (key, value) => {
|
|
76
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
77
|
+
return {
|
|
78
|
+
sql: `CASE
|
|
79
|
+
WHEN ? IS NULL THEN json_extract(metadata, '$."${jsonPathKey}"') IS ${symbol === "=" ? "" : "NOT"} NULL
|
|
80
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') ${symbol} ?
|
|
81
|
+
END`,
|
|
82
|
+
needsValue: true,
|
|
83
|
+
transformValue: () => {
|
|
84
|
+
return [value, value];
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
};
|
|
86
88
|
};
|
|
87
89
|
var createNumericOperator = (symbol) => {
|
|
88
|
-
return (key) =>
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
return (key) => {
|
|
91
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
92
|
+
return {
|
|
93
|
+
sql: `CAST(json_extract(metadata, '$."${jsonPathKey}"') AS NUMERIC) ${symbol} ?`,
|
|
94
|
+
needsValue: true
|
|
95
|
+
};
|
|
96
|
+
};
|
|
92
97
|
};
|
|
93
|
-
var validateJsonArray = (key) => `json_valid(json_extract(metadata, '$."${
|
|
94
|
-
AND json_type(json_extract(metadata, '$."${
|
|
98
|
+
var validateJsonArray = (key) => `json_valid(json_extract(metadata, '$."${key}"'))
|
|
99
|
+
AND json_type(json_extract(metadata, '$."${key}"')) = 'array'`;
|
|
100
|
+
var pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
101
|
+
function buildElemMatchConditions(value) {
|
|
102
|
+
const conditions = Object.entries(value).map(([field, fieldValue]) => {
|
|
103
|
+
if (field.startsWith("$")) {
|
|
104
|
+
const { sql, values } = buildCondition("elem.value", { [field]: fieldValue });
|
|
105
|
+
const elemSql = sql.replace(pattern, "elem.value");
|
|
106
|
+
return { sql: elemSql, values };
|
|
107
|
+
} else if (typeof fieldValue === "object" && !Array.isArray(fieldValue)) {
|
|
108
|
+
const { sql, values } = buildCondition(field, fieldValue);
|
|
109
|
+
const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
|
|
110
|
+
return { sql: elemSql, values };
|
|
111
|
+
} else {
|
|
112
|
+
const parsedFieldKey = parseFieldKey(field);
|
|
113
|
+
return {
|
|
114
|
+
sql: `json_extract(elem.value, '$."${parsedFieldKey}"') = ?`,
|
|
115
|
+
values: [fieldValue]
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
return conditions;
|
|
120
|
+
}
|
|
95
121
|
var FILTER_OPERATORS = {
|
|
96
122
|
$eq: createBasicOperator("="),
|
|
97
123
|
$ne: createBasicOperator("!="),
|
|
@@ -100,90 +126,113 @@ var FILTER_OPERATORS = {
|
|
|
100
126
|
$lt: createNumericOperator("<"),
|
|
101
127
|
$lte: createNumericOperator("<="),
|
|
102
128
|
// Array Operators
|
|
103
|
-
$in: (key, value) =>
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
129
|
+
$in: (key, value) => {
|
|
130
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
131
|
+
const arr = Array.isArray(value) ? value : [value];
|
|
132
|
+
if (arr.length === 0) {
|
|
133
|
+
return { sql: "1 = 0", needsValue: true, transformValue: () => [] };
|
|
134
|
+
}
|
|
135
|
+
const paramPlaceholders = arr.map(() => "?").join(",");
|
|
136
|
+
return {
|
|
137
|
+
sql: `(
|
|
138
|
+
CASE
|
|
139
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
140
|
+
EXISTS (
|
|
141
|
+
SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
142
|
+
WHERE elem.value IN (SELECT value FROM json_each(?))
|
|
143
|
+
)
|
|
144
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') IN (${paramPlaceholders})
|
|
145
|
+
END
|
|
146
|
+
)`,
|
|
147
|
+
needsValue: true,
|
|
148
|
+
transformValue: () => [JSON.stringify(arr), ...arr]
|
|
149
|
+
};
|
|
150
|
+
},
|
|
151
|
+
$nin: (key, value) => {
|
|
152
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
153
|
+
const arr = Array.isArray(value) ? value : [value];
|
|
154
|
+
if (arr.length === 0) {
|
|
155
|
+
return { sql: "1 = 1", needsValue: true, transformValue: () => [] };
|
|
156
|
+
}
|
|
157
|
+
const paramPlaceholders = arr.map(() => "?").join(",");
|
|
158
|
+
return {
|
|
159
|
+
sql: `(
|
|
160
|
+
CASE
|
|
161
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
162
|
+
NOT EXISTS (
|
|
163
|
+
SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
164
|
+
WHERE elem.value IN (SELECT value FROM json_each(?))
|
|
165
|
+
)
|
|
166
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') NOT IN (${paramPlaceholders})
|
|
167
|
+
END
|
|
168
|
+
)`,
|
|
169
|
+
needsValue: true,
|
|
170
|
+
transformValue: () => [JSON.stringify(arr), ...arr]
|
|
171
|
+
};
|
|
172
|
+
},
|
|
173
|
+
$all: (key, value) => {
|
|
174
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
175
|
+
let sql;
|
|
176
|
+
const arrayValue = Array.isArray(value) ? value : [value];
|
|
177
|
+
if (arrayValue.length === 0) {
|
|
178
|
+
sql = "1 = 0";
|
|
179
|
+
} else {
|
|
180
|
+
sql = `(
|
|
181
|
+
CASE
|
|
182
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
183
|
+
NOT EXISTS (
|
|
184
|
+
SELECT value
|
|
185
|
+
FROM json_each(?)
|
|
186
|
+
WHERE value NOT IN (
|
|
187
|
+
SELECT value
|
|
188
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"'))
|
|
133
189
|
)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
};
|
|
190
|
+
)
|
|
191
|
+
ELSE FALSE
|
|
192
|
+
END
|
|
193
|
+
)`;
|
|
139
194
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
throw new Error("$elemMatch requires an object with conditions");
|
|
147
|
-
}
|
|
148
|
-
const conditions = Object.entries(value).map(([field, fieldValue]) => {
|
|
149
|
-
if (field.startsWith("$")) {
|
|
150
|
-
const { sql, values } = buildCondition("elem.value", { [field]: fieldValue });
|
|
151
|
-
const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
152
|
-
const elemSql = sql.replace(pattern, "elem.value");
|
|
153
|
-
return { sql: elemSql, values };
|
|
154
|
-
} else if (typeof fieldValue === "object" && !Array.isArray(fieldValue)) {
|
|
155
|
-
const { sql, values } = buildCondition(field, fieldValue);
|
|
156
|
-
const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
157
|
-
const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
|
|
158
|
-
return { sql: elemSql, values };
|
|
159
|
-
} else {
|
|
160
|
-
return {
|
|
161
|
-
sql: `json_extract(elem.value, '$."${field}"') = ?`,
|
|
162
|
-
values: [fieldValue]
|
|
163
|
-
};
|
|
195
|
+
return {
|
|
196
|
+
sql,
|
|
197
|
+
needsValue: true,
|
|
198
|
+
transformValue: () => {
|
|
199
|
+
if (arrayValue.length === 0) {
|
|
200
|
+
return [];
|
|
164
201
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
WHERE ${conditions.map((c) => c.sql).join(" AND ")}
|
|
174
|
-
)
|
|
175
|
-
ELSE FALSE
|
|
176
|
-
END
|
|
177
|
-
)`,
|
|
178
|
-
values: conditions.flatMap((c) => c.values)
|
|
179
|
-
};
|
|
202
|
+
return [JSON.stringify(arrayValue)];
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
},
|
|
206
|
+
$elemMatch: (key, value) => {
|
|
207
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
208
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
209
|
+
throw new Error("$elemMatch requires an object with conditions");
|
|
180
210
|
}
|
|
181
|
-
|
|
211
|
+
const conditions = buildElemMatchConditions(value);
|
|
212
|
+
return {
|
|
213
|
+
sql: `(
|
|
214
|
+
CASE
|
|
215
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
216
|
+
EXISTS (
|
|
217
|
+
SELECT 1
|
|
218
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
219
|
+
WHERE ${conditions.map((c) => c.sql).join(" AND ")}
|
|
220
|
+
)
|
|
221
|
+
ELSE FALSE
|
|
222
|
+
END
|
|
223
|
+
)`,
|
|
224
|
+
needsValue: true,
|
|
225
|
+
transformValue: () => conditions.flatMap((c) => c.values)
|
|
226
|
+
};
|
|
227
|
+
},
|
|
182
228
|
// Element Operators
|
|
183
|
-
$exists: (key) =>
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
229
|
+
$exists: (key) => {
|
|
230
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
231
|
+
return {
|
|
232
|
+
sql: `json_extract(metadata, '$."${jsonPathKey}"') IS NOT NULL`,
|
|
233
|
+
needsValue: false
|
|
234
|
+
};
|
|
235
|
+
},
|
|
187
236
|
// Logical Operators
|
|
188
237
|
$and: (key) => ({
|
|
189
238
|
sql: `(${key})`,
|
|
@@ -198,27 +247,30 @@ var FILTER_OPERATORS = {
|
|
|
198
247
|
sql: `NOT (${key})`,
|
|
199
248
|
needsValue: false
|
|
200
249
|
}),
|
|
201
|
-
$size: (key, paramIndex) =>
|
|
202
|
-
|
|
250
|
+
$size: (key, paramIndex) => {
|
|
251
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
252
|
+
return {
|
|
253
|
+
sql: `(
|
|
203
254
|
CASE
|
|
204
|
-
WHEN json_type(json_extract(metadata, '$."${
|
|
205
|
-
json_array_length(json_extract(metadata, '$."${
|
|
255
|
+
WHEN json_type(json_extract(metadata, '$."${jsonPathKey}"')) = 'array' THEN
|
|
256
|
+
json_array_length(json_extract(metadata, '$."${jsonPathKey}"')) = $${paramIndex}
|
|
206
257
|
ELSE FALSE
|
|
207
258
|
END
|
|
208
259
|
)`,
|
|
209
|
-
|
|
210
|
-
|
|
260
|
+
needsValue: true
|
|
261
|
+
};
|
|
262
|
+
},
|
|
211
263
|
// /**
|
|
212
264
|
// * Regex Operators
|
|
213
265
|
// * Supports case insensitive and multiline
|
|
214
266
|
// */
|
|
215
267
|
// $regex: (key: string): FilterOperator => ({
|
|
216
|
-
// sql: `json_extract(metadata, '$."${
|
|
268
|
+
// sql: `json_extract(metadata, '$."${toJsonPathKey(key)}"') = ?`,
|
|
217
269
|
// needsValue: true,
|
|
218
270
|
// transformValue: (value: any) => {
|
|
219
271
|
// const pattern = typeof value === 'object' ? value.$regex : value;
|
|
220
272
|
// const options = typeof value === 'object' ? value.$options || '' : '';
|
|
221
|
-
// let sql = `json_extract(metadata, '$."${
|
|
273
|
+
// let sql = `json_extract(metadata, '$."${toJsonPathKey(key)}"')`;
|
|
222
274
|
// // Handle multiline
|
|
223
275
|
// // if (options.includes('m')) {
|
|
224
276
|
// // sql = `REPLACE(${sql}, CHAR(10), '\n')`;
|
|
@@ -267,50 +319,64 @@ var FILTER_OPERATORS = {
|
|
|
267
319
|
// };
|
|
268
320
|
// },
|
|
269
321
|
// }),
|
|
270
|
-
$contains: (key) =>
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
}
|
|
287
|
-
if (value && typeof value === "object") {
|
|
288
|
-
let traverse2 = function(obj, path = []) {
|
|
289
|
-
for (const [k, v] of Object.entries(obj)) {
|
|
290
|
-
const currentPath = [...path, k];
|
|
291
|
-
if (v && typeof v === "object" && !Array.isArray(v)) {
|
|
292
|
-
traverse2(v, currentPath);
|
|
293
|
-
} else {
|
|
294
|
-
paths.push(currentPath.join("."));
|
|
295
|
-
values.push(v);
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
};
|
|
299
|
-
const paths = [];
|
|
300
|
-
const values = [];
|
|
301
|
-
traverse2(value);
|
|
302
|
-
return {
|
|
303
|
-
sql: `(${paths.map((path) => `json_extract(metadata, '$."${handleKey(key)}"."${path}"') = ?`).join(" AND ")})`,
|
|
304
|
-
values
|
|
305
|
-
};
|
|
306
|
-
}
|
|
307
|
-
return value;
|
|
322
|
+
$contains: (key, value) => {
|
|
323
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
324
|
+
let sql;
|
|
325
|
+
if (Array.isArray(value)) {
|
|
326
|
+
sql = `(
|
|
327
|
+
SELECT ${validateJsonArray(jsonPathKey)}
|
|
328
|
+
AND EXISTS (
|
|
329
|
+
SELECT 1
|
|
330
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as m
|
|
331
|
+
WHERE m.value IN (SELECT value FROM json_each(?))
|
|
332
|
+
)
|
|
333
|
+
)`;
|
|
334
|
+
} else if (typeof value === "string") {
|
|
335
|
+
sql = `lower(json_extract(metadata, '$."${jsonPathKey}"')) LIKE '%' || lower(?) || '%' ESCAPE '\\'`;
|
|
336
|
+
} else {
|
|
337
|
+
sql = `json_extract(metadata, '$."${jsonPathKey}"') = ?`;
|
|
308
338
|
}
|
|
309
|
-
|
|
339
|
+
return {
|
|
340
|
+
sql,
|
|
341
|
+
needsValue: true,
|
|
342
|
+
transformValue: () => {
|
|
343
|
+
if (Array.isArray(value)) {
|
|
344
|
+
return [JSON.stringify(value)];
|
|
345
|
+
}
|
|
346
|
+
if (typeof value === "object" && value !== null) {
|
|
347
|
+
return [JSON.stringify(value)];
|
|
348
|
+
}
|
|
349
|
+
if (typeof value === "string") {
|
|
350
|
+
return [escapeLikePattern(value)];
|
|
351
|
+
}
|
|
352
|
+
return [value];
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* $objectContains: True JSON containment for advanced use (deep sub-object match).
|
|
358
|
+
* Usage: { field: { $objectContains: { ...subobject } } }
|
|
359
|
+
*/
|
|
360
|
+
// $objectContains: (key: string) => ({
|
|
361
|
+
// sql: '', // Will be overridden by transformValue
|
|
362
|
+
// needsValue: true,
|
|
363
|
+
// transformValue: (value: any) => ({
|
|
364
|
+
// sql: `json_type(json_extract(metadata, '$."${toJsonPathKey(key)}"')) = 'object'
|
|
365
|
+
// AND json_patch(json_extract(metadata, '$."${toJsonPathKey(key)}"'), ?) = json_extract(metadata, '$."${toJsonPathKey(key)}"')`,
|
|
366
|
+
// values: [JSON.stringify(value)],
|
|
367
|
+
// }),
|
|
368
|
+
// }),
|
|
310
369
|
};
|
|
311
|
-
|
|
312
|
-
return
|
|
370
|
+
function isFilterResult(obj) {
|
|
371
|
+
return obj && typeof obj === "object" && typeof obj.sql === "string" && Array.isArray(obj.values);
|
|
372
|
+
}
|
|
373
|
+
var parseJsonPathKey = (key) => {
|
|
374
|
+
const parsedKey = parseFieldKey(key);
|
|
375
|
+
return parsedKey.replace(/\./g, '"."');
|
|
313
376
|
};
|
|
377
|
+
function escapeLikePattern(str) {
|
|
378
|
+
return str.replace(/([%_\\])/g, "\\$1");
|
|
379
|
+
}
|
|
314
380
|
function buildFilterQuery(filter) {
|
|
315
381
|
if (!filter) {
|
|
316
382
|
return { sql: "", values: [] };
|
|
@@ -403,8 +469,8 @@ var processOperator = (key, operator, operatorValue) => {
|
|
|
403
469
|
if (!operatorResult.needsValue) {
|
|
404
470
|
return { sql: operatorResult.sql, values: [] };
|
|
405
471
|
}
|
|
406
|
-
const transformed = operatorResult.transformValue ? operatorResult.transformValue(
|
|
407
|
-
if (transformed
|
|
472
|
+
const transformed = operatorResult.transformValue ? operatorResult.transformValue() : operatorValue;
|
|
473
|
+
if (isFilterResult(transformed)) {
|
|
408
474
|
return transformed;
|
|
409
475
|
}
|
|
410
476
|
return {
|
|
@@ -440,14 +506,27 @@ var LibSQLVector = class extends MastraVector {
|
|
|
440
506
|
const translator = new LibSQLFilterTranslator();
|
|
441
507
|
return translator.translate(filter);
|
|
442
508
|
}
|
|
443
|
-
async query(
|
|
444
|
-
|
|
509
|
+
async query({
|
|
510
|
+
indexName,
|
|
511
|
+
queryVector,
|
|
512
|
+
topK = 10,
|
|
513
|
+
filter,
|
|
514
|
+
includeVector = false,
|
|
515
|
+
minScore = 0
|
|
516
|
+
}) {
|
|
445
517
|
try {
|
|
446
|
-
|
|
518
|
+
if (!Number.isInteger(topK) || topK <= 0) {
|
|
519
|
+
throw new Error("topK must be a positive integer");
|
|
520
|
+
}
|
|
521
|
+
if (!Array.isArray(queryVector) || !queryVector.every((x) => typeof x === "number" && Number.isFinite(x))) {
|
|
522
|
+
throw new Error("queryVector must be an array of finite numbers");
|
|
523
|
+
}
|
|
524
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
447
525
|
const vectorStr = `[${queryVector.join(",")}]`;
|
|
448
526
|
const translatedFilter = this.transformFilter(filter);
|
|
449
527
|
const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter);
|
|
450
528
|
filterValues.push(minScore);
|
|
529
|
+
filterValues.push(topK);
|
|
451
530
|
const query = `
|
|
452
531
|
WITH vector_scores AS (
|
|
453
532
|
SELECT
|
|
@@ -455,14 +534,14 @@ var LibSQLVector = class extends MastraVector {
|
|
|
455
534
|
(1-vector_distance_cos(embedding, '${vectorStr}')) as score,
|
|
456
535
|
metadata
|
|
457
536
|
${includeVector ? ", vector_extract(embedding) as embedding" : ""}
|
|
458
|
-
FROM ${
|
|
537
|
+
FROM ${parsedIndexName}
|
|
459
538
|
${filterQuery}
|
|
460
539
|
)
|
|
461
540
|
SELECT *
|
|
462
541
|
FROM vector_scores
|
|
463
542
|
WHERE score > ?
|
|
464
543
|
ORDER BY score DESC
|
|
465
|
-
LIMIT
|
|
544
|
+
LIMIT ?`;
|
|
466
545
|
const result = await this.turso.execute({
|
|
467
546
|
sql: query,
|
|
468
547
|
args: filterValues
|
|
@@ -476,15 +555,14 @@ var LibSQLVector = class extends MastraVector {
|
|
|
476
555
|
} finally {
|
|
477
556
|
}
|
|
478
557
|
}
|
|
479
|
-
async upsert(
|
|
480
|
-
const params = this.normalizeArgs("upsert", args);
|
|
481
|
-
const { indexName, vectors, metadata, ids } = params;
|
|
558
|
+
async upsert({ indexName, vectors, metadata, ids }) {
|
|
482
559
|
const tx = await this.turso.transaction("write");
|
|
483
560
|
try {
|
|
561
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
484
562
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
485
563
|
for (let i = 0; i < vectors.length; i++) {
|
|
486
564
|
const query = `
|
|
487
|
-
INSERT INTO ${
|
|
565
|
+
INSERT INTO ${parsedIndexName} (vector_id, embedding, metadata)
|
|
488
566
|
VALUES (?, vector32(?), ?)
|
|
489
567
|
ON CONFLICT(vector_id) DO UPDATE SET
|
|
490
568
|
embedding = vector32(?),
|
|
@@ -518,19 +596,15 @@ var LibSQLVector = class extends MastraVector {
|
|
|
518
596
|
throw error;
|
|
519
597
|
}
|
|
520
598
|
}
|
|
521
|
-
async createIndex(
|
|
522
|
-
const params = this.normalizeArgs("createIndex", args);
|
|
523
|
-
const { indexName, dimension } = params;
|
|
599
|
+
async createIndex({ indexName, dimension }) {
|
|
524
600
|
try {
|
|
525
|
-
if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
|
|
526
|
-
throw new Error("Invalid index name format");
|
|
527
|
-
}
|
|
528
601
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
529
602
|
throw new Error("Dimension must be a positive integer");
|
|
530
603
|
}
|
|
604
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
531
605
|
await this.turso.execute({
|
|
532
606
|
sql: `
|
|
533
|
-
CREATE TABLE IF NOT EXISTS ${
|
|
607
|
+
CREATE TABLE IF NOT EXISTS ${parsedIndexName} (
|
|
534
608
|
id SERIAL PRIMARY KEY,
|
|
535
609
|
vector_id TEXT UNIQUE NOT NULL,
|
|
536
610
|
embedding F32_BLOB(${dimension}),
|
|
@@ -541,8 +615,8 @@ var LibSQLVector = class extends MastraVector {
|
|
|
541
615
|
});
|
|
542
616
|
await this.turso.execute({
|
|
543
617
|
sql: `
|
|
544
|
-
CREATE INDEX IF NOT EXISTS ${
|
|
545
|
-
ON ${
|
|
618
|
+
CREATE INDEX IF NOT EXISTS ${parsedIndexName}_vector_idx
|
|
619
|
+
ON ${parsedIndexName} (libsql_vector_idx(embedding))
|
|
546
620
|
`,
|
|
547
621
|
args: []
|
|
548
622
|
});
|
|
@@ -552,10 +626,11 @@ var LibSQLVector = class extends MastraVector {
|
|
|
552
626
|
} finally {
|
|
553
627
|
}
|
|
554
628
|
}
|
|
555
|
-
async deleteIndex(indexName) {
|
|
629
|
+
async deleteIndex({ indexName }) {
|
|
556
630
|
try {
|
|
631
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
557
632
|
await this.turso.execute({
|
|
558
|
-
sql: `DROP TABLE IF EXISTS ${
|
|
633
|
+
sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
|
|
559
634
|
args: []
|
|
560
635
|
});
|
|
561
636
|
} catch (error) {
|
|
@@ -580,8 +655,15 @@ var LibSQLVector = class extends MastraVector {
|
|
|
580
655
|
throw new Error(`Failed to list vector tables: ${error.message}`);
|
|
581
656
|
}
|
|
582
657
|
}
|
|
583
|
-
|
|
658
|
+
/**
|
|
659
|
+
* Retrieves statistics about a vector index.
|
|
660
|
+
*
|
|
661
|
+
* @param {string} indexName - The name of the index to describe
|
|
662
|
+
* @returns A promise that resolves to the index statistics including dimension, count and metric
|
|
663
|
+
*/
|
|
664
|
+
async describeIndex({ indexName }) {
|
|
584
665
|
try {
|
|
666
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
585
667
|
const tableInfoQuery = `
|
|
586
668
|
SELECT sql
|
|
587
669
|
FROM sqlite_master
|
|
@@ -590,15 +672,15 @@ var LibSQLVector = class extends MastraVector {
|
|
|
590
672
|
`;
|
|
591
673
|
const tableInfo = await this.turso.execute({
|
|
592
674
|
sql: tableInfoQuery,
|
|
593
|
-
args: [
|
|
675
|
+
args: [parsedIndexName]
|
|
594
676
|
});
|
|
595
677
|
if (!tableInfo.rows[0]?.sql) {
|
|
596
|
-
throw new Error(`Table ${
|
|
678
|
+
throw new Error(`Table ${parsedIndexName} not found`);
|
|
597
679
|
}
|
|
598
680
|
const dimension = parseInt(tableInfo.rows[0].sql.match(/F32_BLOB\((\d+)\)/)?.[1] || "0");
|
|
599
681
|
const countQuery = `
|
|
600
682
|
SELECT COUNT(*) as count
|
|
601
|
-
FROM ${
|
|
683
|
+
FROM ${parsedIndexName};
|
|
602
684
|
`;
|
|
603
685
|
const countResult = await this.turso.execute({
|
|
604
686
|
sql: countQuery,
|
|
@@ -614,28 +696,9 @@ var LibSQLVector = class extends MastraVector {
|
|
|
614
696
|
throw new Error(`Failed to describe vector table: ${e.message}`);
|
|
615
697
|
}
|
|
616
698
|
}
|
|
617
|
-
/**
|
|
618
|
-
* @deprecated Use {@link updateVector} instead. This method will be removed on May 20th, 2025.
|
|
619
|
-
*
|
|
620
|
-
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
621
|
-
* @param indexName - The name of the index containing the vector.
|
|
622
|
-
* @param id - The ID of the vector to update.
|
|
623
|
-
* @param update - An object containing the vector and/or metadata to update.
|
|
624
|
-
* @param update.vector - An optional array of numbers representing the new vector.
|
|
625
|
-
* @param update.metadata - An optional record containing the new metadata.
|
|
626
|
-
* @returns A promise that resolves when the update is complete.
|
|
627
|
-
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
628
|
-
*/
|
|
629
|
-
async updateIndexById(indexName, id, update) {
|
|
630
|
-
this.logger.warn(
|
|
631
|
-
`Deprecation Warning: updateIndexById() is deprecated.
|
|
632
|
-
Please use updateVector() instead.
|
|
633
|
-
updateIndexById() will be removed on May 20th, 2025.`
|
|
634
|
-
);
|
|
635
|
-
await this.updateVector(indexName, id, update);
|
|
636
|
-
}
|
|
637
699
|
/**
|
|
638
700
|
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
701
|
+
*
|
|
639
702
|
* @param indexName - The name of the index containing the vector.
|
|
640
703
|
* @param id - The ID of the vector to update.
|
|
641
704
|
* @param update - An object containing the vector and/or metadata to update.
|
|
@@ -644,8 +707,9 @@ var LibSQLVector = class extends MastraVector {
|
|
|
644
707
|
* @returns A promise that resolves when the update is complete.
|
|
645
708
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
646
709
|
*/
|
|
647
|
-
async updateVector(indexName, id, update) {
|
|
710
|
+
async updateVector({ indexName, id, update }) {
|
|
648
711
|
try {
|
|
712
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
649
713
|
const updates = [];
|
|
650
714
|
const args = [];
|
|
651
715
|
if (update.vector) {
|
|
@@ -661,7 +725,7 @@ var LibSQLVector = class extends MastraVector {
|
|
|
661
725
|
}
|
|
662
726
|
args.push(id);
|
|
663
727
|
const query = `
|
|
664
|
-
UPDATE ${
|
|
728
|
+
UPDATE ${parsedIndexName}
|
|
665
729
|
SET ${updates.join(", ")}
|
|
666
730
|
WHERE vector_id = ?;
|
|
667
731
|
`;
|
|
@@ -673,23 +737,6 @@ var LibSQLVector = class extends MastraVector {
|
|
|
673
737
|
throw new Error(`Failed to update vector by id: ${id} for index: ${indexName}: ${error.message}`);
|
|
674
738
|
}
|
|
675
739
|
}
|
|
676
|
-
/**
|
|
677
|
-
* @deprecated Use {@link deleteVector} instead. This method will be removed on May 20th, 2025.
|
|
678
|
-
*
|
|
679
|
-
* Deletes a vector by its ID.
|
|
680
|
-
* @param indexName - The name of the index containing the vector.
|
|
681
|
-
* @param id - The ID of the vector to delete.
|
|
682
|
-
* @returns A promise that resolves when the deletion is complete.
|
|
683
|
-
* @throws Will throw an error if the deletion operation fails.
|
|
684
|
-
*/
|
|
685
|
-
async deleteIndexById(indexName, id) {
|
|
686
|
-
this.logger.warn(
|
|
687
|
-
`Deprecation Warning: deleteIndexById() is deprecated.
|
|
688
|
-
Please use deleteVector() instead.
|
|
689
|
-
deleteIndexById() will be removed on May 20th, 2025.`
|
|
690
|
-
);
|
|
691
|
-
await this.deleteVector(indexName, id);
|
|
692
|
-
}
|
|
693
740
|
/**
|
|
694
741
|
* Deletes a vector by its ID.
|
|
695
742
|
* @param indexName - The name of the index containing the vector.
|
|
@@ -697,19 +744,20 @@ var LibSQLVector = class extends MastraVector {
|
|
|
697
744
|
* @returns A promise that resolves when the deletion is complete.
|
|
698
745
|
* @throws Will throw an error if the deletion operation fails.
|
|
699
746
|
*/
|
|
700
|
-
async deleteVector(indexName, id) {
|
|
747
|
+
async deleteVector({ indexName, id }) {
|
|
701
748
|
try {
|
|
749
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
702
750
|
await this.turso.execute({
|
|
703
|
-
sql: `DELETE FROM ${
|
|
751
|
+
sql: `DELETE FROM ${parsedIndexName} WHERE vector_id = ?`,
|
|
704
752
|
args: [id]
|
|
705
753
|
});
|
|
706
754
|
} catch (error) {
|
|
707
755
|
throw new Error(`Failed to delete vector by id: ${id} for index: ${indexName}: ${error.message}`);
|
|
708
756
|
}
|
|
709
757
|
}
|
|
710
|
-
async truncateIndex(indexName) {
|
|
758
|
+
async truncateIndex({ indexName }) {
|
|
711
759
|
await this.turso.execute({
|
|
712
|
-
sql: `DELETE FROM ${indexName}`,
|
|
760
|
+
sql: `DELETE FROM ${parseSqlIdentifier(indexName, "index name")}`,
|
|
713
761
|
args: []
|
|
714
762
|
});
|
|
715
763
|
}
|
|
@@ -731,22 +779,24 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
731
779
|
this.client = createClient(config);
|
|
732
780
|
}
|
|
733
781
|
getCreateTableSQL(tableName, schema) {
|
|
782
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
734
783
|
const columns = Object.entries(schema).map(([name, col]) => {
|
|
784
|
+
const parsedColumnName = parseSqlIdentifier(name, "column name");
|
|
735
785
|
let type = col.type.toUpperCase();
|
|
736
786
|
if (type === "TEXT") type = "TEXT";
|
|
737
787
|
if (type === "TIMESTAMP") type = "TEXT";
|
|
738
788
|
const nullable = col.nullable ? "" : "NOT NULL";
|
|
739
789
|
const primaryKey = col.primaryKey ? "PRIMARY KEY" : "";
|
|
740
|
-
return `${
|
|
790
|
+
return `${parsedColumnName} ${type} ${nullable} ${primaryKey}`.trim();
|
|
741
791
|
});
|
|
742
792
|
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
743
|
-
const stmnt = `CREATE TABLE IF NOT EXISTS ${
|
|
793
|
+
const stmnt = `CREATE TABLE IF NOT EXISTS ${parsedTableName} (
|
|
744
794
|
${columns.join(",\n")},
|
|
745
795
|
PRIMARY KEY (workflow_name, run_id)
|
|
746
796
|
)`;
|
|
747
797
|
return stmnt;
|
|
748
798
|
}
|
|
749
|
-
return `CREATE TABLE IF NOT EXISTS ${
|
|
799
|
+
return `CREATE TABLE IF NOT EXISTS ${parsedTableName} (${columns.join(", ")})`;
|
|
750
800
|
}
|
|
751
801
|
async createTable({
|
|
752
802
|
tableName,
|
|
@@ -762,8 +812,9 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
762
812
|
}
|
|
763
813
|
}
|
|
764
814
|
async clearTable({ tableName }) {
|
|
815
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
765
816
|
try {
|
|
766
|
-
await this.client.execute(`DELETE FROM ${
|
|
817
|
+
await this.client.execute(`DELETE FROM ${parsedTableName}`);
|
|
767
818
|
} catch (e) {
|
|
768
819
|
if (e instanceof Error) {
|
|
769
820
|
this.logger.error(e.message);
|
|
@@ -771,7 +822,8 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
771
822
|
}
|
|
772
823
|
}
|
|
773
824
|
prepareStatement({ tableName, record }) {
|
|
774
|
-
const
|
|
825
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
826
|
+
const columns = Object.keys(record).map((col) => parseSqlIdentifier(col, "column name"));
|
|
775
827
|
const values = Object.values(record).map((v) => {
|
|
776
828
|
if (typeof v === `undefined`) {
|
|
777
829
|
return null;
|
|
@@ -783,7 +835,7 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
783
835
|
});
|
|
784
836
|
const placeholders = values.map(() => "?").join(", ");
|
|
785
837
|
return {
|
|
786
|
-
sql: `INSERT OR REPLACE INTO ${
|
|
838
|
+
sql: `INSERT OR REPLACE INTO ${parsedTableName} (${columns.join(", ")}) VALUES (${placeholders})`,
|
|
787
839
|
args: values
|
|
788
840
|
};
|
|
789
841
|
}
|
|
@@ -811,10 +863,12 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
811
863
|
}
|
|
812
864
|
}
|
|
813
865
|
async load({ tableName, keys }) {
|
|
814
|
-
const
|
|
866
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
867
|
+
const parsedKeys = Object.keys(keys).map((key) => parseSqlIdentifier(key, "column name"));
|
|
868
|
+
const conditions = parsedKeys.map((key) => `${key} = ?`).join(" AND ");
|
|
815
869
|
const values = Object.values(keys);
|
|
816
870
|
const result = await this.client.execute({
|
|
817
|
-
sql: `SELECT * FROM ${
|
|
871
|
+
sql: `SELECT * FROM ${parsedTableName} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
818
872
|
args: values
|
|
819
873
|
});
|
|
820
874
|
if (!result.rows || result.rows.length === 0) {
|