@mastra/libsql 0.0.4-alpha.4 → 0.0.5-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +36 -0
- package/dist/_tsup-dts-rollup.d.cts +18 -29
- package/dist/_tsup-dts-rollup.d.ts +18 -29
- package/dist/index.cjs +285 -184
- package/dist/index.js +285 -184
- package/package.json +3 -3
- package/src/storage/index.ts +18 -10
- package/src/vector/index.test.ts +154 -119
- package/src/vector/index.ts +63 -27
- package/src/vector/sql-builder.ts +243 -175
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createClient } from '@libsql/client';
|
|
2
|
+
import { parseSqlIdentifier, parseFieldKey } from '@mastra/core/utils';
|
|
2
3
|
import { MastraVector } from '@mastra/core/vector';
|
|
3
4
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
4
5
|
import { MastraStorage, TABLE_WORKFLOW_SNAPSHOT, TABLE_THREADS, TABLE_MESSAGES, TABLE_EVALS, TABLE_TRACES } from '@mastra/core/storage';
|
|
@@ -70,28 +71,53 @@ var LibSQLFilterTranslator = class extends BaseFilterTranslator {
|
|
|
70
71
|
// };
|
|
71
72
|
// }
|
|
72
73
|
};
|
|
73
|
-
|
|
74
|
-
// src/vector/sql-builder.ts
|
|
75
74
|
var createBasicOperator = (symbol) => {
|
|
76
|
-
return (key) =>
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
75
|
+
return (key, value) => {
|
|
76
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
77
|
+
return {
|
|
78
|
+
sql: `CASE
|
|
79
|
+
WHEN ? IS NULL THEN json_extract(metadata, '$."${jsonPathKey}"') IS ${symbol === "=" ? "" : "NOT"} NULL
|
|
80
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') ${symbol} ?
|
|
81
|
+
END`,
|
|
82
|
+
needsValue: true,
|
|
83
|
+
transformValue: () => {
|
|
84
|
+
return [value, value];
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
};
|
|
86
88
|
};
|
|
87
89
|
var createNumericOperator = (symbol) => {
|
|
88
|
-
return (key) =>
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
return (key) => {
|
|
91
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
92
|
+
return {
|
|
93
|
+
sql: `CAST(json_extract(metadata, '$."${jsonPathKey}"') AS NUMERIC) ${symbol} ?`,
|
|
94
|
+
needsValue: true
|
|
95
|
+
};
|
|
96
|
+
};
|
|
92
97
|
};
|
|
93
|
-
var validateJsonArray = (key) => `json_valid(json_extract(metadata, '$."${
|
|
94
|
-
AND json_type(json_extract(metadata, '$."${
|
|
98
|
+
var validateJsonArray = (key) => `json_valid(json_extract(metadata, '$."${key}"'))
|
|
99
|
+
AND json_type(json_extract(metadata, '$."${key}"')) = 'array'`;
|
|
100
|
+
var pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
101
|
+
function buildElemMatchConditions(value) {
|
|
102
|
+
const conditions = Object.entries(value).map(([field, fieldValue]) => {
|
|
103
|
+
if (field.startsWith("$")) {
|
|
104
|
+
const { sql, values } = buildCondition("elem.value", { [field]: fieldValue });
|
|
105
|
+
const elemSql = sql.replace(pattern, "elem.value");
|
|
106
|
+
return { sql: elemSql, values };
|
|
107
|
+
} else if (typeof fieldValue === "object" && !Array.isArray(fieldValue)) {
|
|
108
|
+
const { sql, values } = buildCondition(field, fieldValue);
|
|
109
|
+
const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
|
|
110
|
+
return { sql: elemSql, values };
|
|
111
|
+
} else {
|
|
112
|
+
const parsedFieldKey = parseFieldKey(field);
|
|
113
|
+
return {
|
|
114
|
+
sql: `json_extract(elem.value, '$."${parsedFieldKey}"') = ?`,
|
|
115
|
+
values: [fieldValue]
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
return conditions;
|
|
120
|
+
}
|
|
95
121
|
var FILTER_OPERATORS = {
|
|
96
122
|
$eq: createBasicOperator("="),
|
|
97
123
|
$ne: createBasicOperator("!="),
|
|
@@ -100,90 +126,113 @@ var FILTER_OPERATORS = {
|
|
|
100
126
|
$lt: createNumericOperator("<"),
|
|
101
127
|
$lte: createNumericOperator("<="),
|
|
102
128
|
// Array Operators
|
|
103
|
-
$in: (key, value) =>
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
129
|
+
$in: (key, value) => {
|
|
130
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
131
|
+
const arr = Array.isArray(value) ? value : [value];
|
|
132
|
+
if (arr.length === 0) {
|
|
133
|
+
return { sql: "1 = 0", needsValue: true, transformValue: () => [] };
|
|
134
|
+
}
|
|
135
|
+
const paramPlaceholders = arr.map(() => "?").join(",");
|
|
136
|
+
return {
|
|
137
|
+
sql: `(
|
|
138
|
+
CASE
|
|
139
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
140
|
+
EXISTS (
|
|
141
|
+
SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
142
|
+
WHERE elem.value IN (SELECT value FROM json_each(?))
|
|
143
|
+
)
|
|
144
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') IN (${paramPlaceholders})
|
|
145
|
+
END
|
|
146
|
+
)`,
|
|
147
|
+
needsValue: true,
|
|
148
|
+
transformValue: () => [JSON.stringify(arr), ...arr]
|
|
149
|
+
};
|
|
150
|
+
},
|
|
151
|
+
$nin: (key, value) => {
|
|
152
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
153
|
+
const arr = Array.isArray(value) ? value : [value];
|
|
154
|
+
if (arr.length === 0) {
|
|
155
|
+
return { sql: "1 = 1", needsValue: true, transformValue: () => [] };
|
|
156
|
+
}
|
|
157
|
+
const paramPlaceholders = arr.map(() => "?").join(",");
|
|
158
|
+
return {
|
|
159
|
+
sql: `(
|
|
160
|
+
CASE
|
|
161
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
162
|
+
NOT EXISTS (
|
|
163
|
+
SELECT 1 FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
164
|
+
WHERE elem.value IN (SELECT value FROM json_each(?))
|
|
165
|
+
)
|
|
166
|
+
ELSE json_extract(metadata, '$."${jsonPathKey}"') NOT IN (${paramPlaceholders})
|
|
167
|
+
END
|
|
168
|
+
)`,
|
|
169
|
+
needsValue: true,
|
|
170
|
+
transformValue: () => [JSON.stringify(arr), ...arr]
|
|
171
|
+
};
|
|
172
|
+
},
|
|
173
|
+
$all: (key, value) => {
|
|
174
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
175
|
+
let sql;
|
|
176
|
+
const arrayValue = Array.isArray(value) ? value : [value];
|
|
177
|
+
if (arrayValue.length === 0) {
|
|
178
|
+
sql = "1 = 0";
|
|
179
|
+
} else {
|
|
180
|
+
sql = `(
|
|
181
|
+
CASE
|
|
182
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
183
|
+
NOT EXISTS (
|
|
184
|
+
SELECT value
|
|
185
|
+
FROM json_each(?)
|
|
186
|
+
WHERE value NOT IN (
|
|
187
|
+
SELECT value
|
|
188
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"'))
|
|
133
189
|
)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
};
|
|
190
|
+
)
|
|
191
|
+
ELSE FALSE
|
|
192
|
+
END
|
|
193
|
+
)`;
|
|
139
194
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
throw new Error("$elemMatch requires an object with conditions");
|
|
147
|
-
}
|
|
148
|
-
const conditions = Object.entries(value).map(([field, fieldValue]) => {
|
|
149
|
-
if (field.startsWith("$")) {
|
|
150
|
-
const { sql, values } = buildCondition("elem.value", { [field]: fieldValue });
|
|
151
|
-
const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
152
|
-
const elemSql = sql.replace(pattern, "elem.value");
|
|
153
|
-
return { sql: elemSql, values };
|
|
154
|
-
} else if (typeof fieldValue === "object" && !Array.isArray(fieldValue)) {
|
|
155
|
-
const { sql, values } = buildCondition(field, fieldValue);
|
|
156
|
-
const pattern = /json_extract\(metadata, '\$\."[^"]*"(\."[^"]*")*'\)/g;
|
|
157
|
-
const elemSql = sql.replace(pattern, `json_extract(elem.value, '$."${field}"')`);
|
|
158
|
-
return { sql: elemSql, values };
|
|
159
|
-
} else {
|
|
160
|
-
return {
|
|
161
|
-
sql: `json_extract(elem.value, '$."${field}"') = ?`,
|
|
162
|
-
values: [fieldValue]
|
|
163
|
-
};
|
|
195
|
+
return {
|
|
196
|
+
sql,
|
|
197
|
+
needsValue: true,
|
|
198
|
+
transformValue: () => {
|
|
199
|
+
if (arrayValue.length === 0) {
|
|
200
|
+
return [];
|
|
164
201
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
WHERE ${conditions.map((c) => c.sql).join(" AND ")}
|
|
174
|
-
)
|
|
175
|
-
ELSE FALSE
|
|
176
|
-
END
|
|
177
|
-
)`,
|
|
178
|
-
values: conditions.flatMap((c) => c.values)
|
|
179
|
-
};
|
|
202
|
+
return [JSON.stringify(arrayValue)];
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
},
|
|
206
|
+
$elemMatch: (key, value) => {
|
|
207
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
208
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
209
|
+
throw new Error("$elemMatch requires an object with conditions");
|
|
180
210
|
}
|
|
181
|
-
|
|
211
|
+
const conditions = buildElemMatchConditions(value);
|
|
212
|
+
return {
|
|
213
|
+
sql: `(
|
|
214
|
+
CASE
|
|
215
|
+
WHEN ${validateJsonArray(jsonPathKey)} THEN
|
|
216
|
+
EXISTS (
|
|
217
|
+
SELECT 1
|
|
218
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as elem
|
|
219
|
+
WHERE ${conditions.map((c) => c.sql).join(" AND ")}
|
|
220
|
+
)
|
|
221
|
+
ELSE FALSE
|
|
222
|
+
END
|
|
223
|
+
)`,
|
|
224
|
+
needsValue: true,
|
|
225
|
+
transformValue: () => conditions.flatMap((c) => c.values)
|
|
226
|
+
};
|
|
227
|
+
},
|
|
182
228
|
// Element Operators
|
|
183
|
-
$exists: (key) =>
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
229
|
+
$exists: (key) => {
|
|
230
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
231
|
+
return {
|
|
232
|
+
sql: `json_extract(metadata, '$."${jsonPathKey}"') IS NOT NULL`,
|
|
233
|
+
needsValue: false
|
|
234
|
+
};
|
|
235
|
+
},
|
|
187
236
|
// Logical Operators
|
|
188
237
|
$and: (key) => ({
|
|
189
238
|
sql: `(${key})`,
|
|
@@ -198,27 +247,30 @@ var FILTER_OPERATORS = {
|
|
|
198
247
|
sql: `NOT (${key})`,
|
|
199
248
|
needsValue: false
|
|
200
249
|
}),
|
|
201
|
-
$size: (key, paramIndex) =>
|
|
202
|
-
|
|
250
|
+
$size: (key, paramIndex) => {
|
|
251
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
252
|
+
return {
|
|
253
|
+
sql: `(
|
|
203
254
|
CASE
|
|
204
|
-
WHEN json_type(json_extract(metadata, '$."${
|
|
205
|
-
json_array_length(json_extract(metadata, '$."${
|
|
255
|
+
WHEN json_type(json_extract(metadata, '$."${jsonPathKey}"')) = 'array' THEN
|
|
256
|
+
json_array_length(json_extract(metadata, '$."${jsonPathKey}"')) = $${paramIndex}
|
|
206
257
|
ELSE FALSE
|
|
207
258
|
END
|
|
208
259
|
)`,
|
|
209
|
-
|
|
210
|
-
|
|
260
|
+
needsValue: true
|
|
261
|
+
};
|
|
262
|
+
},
|
|
211
263
|
// /**
|
|
212
264
|
// * Regex Operators
|
|
213
265
|
// * Supports case insensitive and multiline
|
|
214
266
|
// */
|
|
215
267
|
// $regex: (key: string): FilterOperator => ({
|
|
216
|
-
// sql: `json_extract(metadata, '$."${
|
|
268
|
+
// sql: `json_extract(metadata, '$."${toJsonPathKey(key)}"') = ?`,
|
|
217
269
|
// needsValue: true,
|
|
218
270
|
// transformValue: (value: any) => {
|
|
219
271
|
// const pattern = typeof value === 'object' ? value.$regex : value;
|
|
220
272
|
// const options = typeof value === 'object' ? value.$options || '' : '';
|
|
221
|
-
// let sql = `json_extract(metadata, '$."${
|
|
273
|
+
// let sql = `json_extract(metadata, '$."${toJsonPathKey(key)}"')`;
|
|
222
274
|
// // Handle multiline
|
|
223
275
|
// // if (options.includes('m')) {
|
|
224
276
|
// // sql = `REPLACE(${sql}, CHAR(10), '\n')`;
|
|
@@ -267,50 +319,64 @@ var FILTER_OPERATORS = {
|
|
|
267
319
|
// };
|
|
268
320
|
// },
|
|
269
321
|
// }),
|
|
270
|
-
$contains: (key) =>
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
}
|
|
287
|
-
if (value && typeof value === "object") {
|
|
288
|
-
let traverse2 = function(obj, path = []) {
|
|
289
|
-
for (const [k, v] of Object.entries(obj)) {
|
|
290
|
-
const currentPath = [...path, k];
|
|
291
|
-
if (v && typeof v === "object" && !Array.isArray(v)) {
|
|
292
|
-
traverse2(v, currentPath);
|
|
293
|
-
} else {
|
|
294
|
-
paths.push(currentPath.join("."));
|
|
295
|
-
values.push(v);
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
};
|
|
299
|
-
const paths = [];
|
|
300
|
-
const values = [];
|
|
301
|
-
traverse2(value);
|
|
302
|
-
return {
|
|
303
|
-
sql: `(${paths.map((path) => `json_extract(metadata, '$."${handleKey(key)}"."${path}"') = ?`).join(" AND ")})`,
|
|
304
|
-
values
|
|
305
|
-
};
|
|
306
|
-
}
|
|
307
|
-
return value;
|
|
322
|
+
$contains: (key, value) => {
|
|
323
|
+
const jsonPathKey = parseJsonPathKey(key);
|
|
324
|
+
let sql;
|
|
325
|
+
if (Array.isArray(value)) {
|
|
326
|
+
sql = `(
|
|
327
|
+
SELECT ${validateJsonArray(jsonPathKey)}
|
|
328
|
+
AND EXISTS (
|
|
329
|
+
SELECT 1
|
|
330
|
+
FROM json_each(json_extract(metadata, '$."${jsonPathKey}"')) as m
|
|
331
|
+
WHERE m.value IN (SELECT value FROM json_each(?))
|
|
332
|
+
)
|
|
333
|
+
)`;
|
|
334
|
+
} else if (typeof value === "string") {
|
|
335
|
+
sql = `lower(json_extract(metadata, '$."${jsonPathKey}"')) LIKE '%' || lower(?) || '%' ESCAPE '\\'`;
|
|
336
|
+
} else {
|
|
337
|
+
sql = `json_extract(metadata, '$."${jsonPathKey}"') = ?`;
|
|
308
338
|
}
|
|
309
|
-
|
|
339
|
+
return {
|
|
340
|
+
sql,
|
|
341
|
+
needsValue: true,
|
|
342
|
+
transformValue: () => {
|
|
343
|
+
if (Array.isArray(value)) {
|
|
344
|
+
return [JSON.stringify(value)];
|
|
345
|
+
}
|
|
346
|
+
if (typeof value === "object" && value !== null) {
|
|
347
|
+
return [JSON.stringify(value)];
|
|
348
|
+
}
|
|
349
|
+
if (typeof value === "string") {
|
|
350
|
+
return [escapeLikePattern(value)];
|
|
351
|
+
}
|
|
352
|
+
return [value];
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* $objectContains: True JSON containment for advanced use (deep sub-object match).
|
|
358
|
+
* Usage: { field: { $objectContains: { ...subobject } } }
|
|
359
|
+
*/
|
|
360
|
+
// $objectContains: (key: string) => ({
|
|
361
|
+
// sql: '', // Will be overridden by transformValue
|
|
362
|
+
// needsValue: true,
|
|
363
|
+
// transformValue: (value: any) => ({
|
|
364
|
+
// sql: `json_type(json_extract(metadata, '$."${toJsonPathKey(key)}"')) = 'object'
|
|
365
|
+
// AND json_patch(json_extract(metadata, '$."${toJsonPathKey(key)}"'), ?) = json_extract(metadata, '$."${toJsonPathKey(key)}"')`,
|
|
366
|
+
// values: [JSON.stringify(value)],
|
|
367
|
+
// }),
|
|
368
|
+
// }),
|
|
310
369
|
};
|
|
311
|
-
|
|
312
|
-
return
|
|
370
|
+
function isFilterResult(obj) {
|
|
371
|
+
return obj && typeof obj === "object" && typeof obj.sql === "string" && Array.isArray(obj.values);
|
|
372
|
+
}
|
|
373
|
+
var parseJsonPathKey = (key) => {
|
|
374
|
+
const parsedKey = parseFieldKey(key);
|
|
375
|
+
return parsedKey.replace(/\./g, '"."');
|
|
313
376
|
};
|
|
377
|
+
function escapeLikePattern(str) {
|
|
378
|
+
return str.replace(/([%_\\])/g, "\\$1");
|
|
379
|
+
}
|
|
314
380
|
function buildFilterQuery(filter) {
|
|
315
381
|
if (!filter) {
|
|
316
382
|
return { sql: "", values: [] };
|
|
@@ -403,8 +469,8 @@ var processOperator = (key, operator, operatorValue) => {
|
|
|
403
469
|
if (!operatorResult.needsValue) {
|
|
404
470
|
return { sql: operatorResult.sql, values: [] };
|
|
405
471
|
}
|
|
406
|
-
const transformed = operatorResult.transformValue ? operatorResult.transformValue(
|
|
407
|
-
if (transformed
|
|
472
|
+
const transformed = operatorResult.transformValue ? operatorResult.transformValue() : operatorValue;
|
|
473
|
+
if (isFilterResult(transformed)) {
|
|
408
474
|
return transformed;
|
|
409
475
|
}
|
|
410
476
|
return {
|
|
@@ -444,10 +510,18 @@ var LibSQLVector = class extends MastraVector {
|
|
|
444
510
|
const params = this.normalizeArgs("query", args, ["minScore"]);
|
|
445
511
|
try {
|
|
446
512
|
const { indexName, queryVector, topK = 10, filter, includeVector = false, minScore = 0 } = params;
|
|
513
|
+
if (!Number.isInteger(topK) || topK <= 0) {
|
|
514
|
+
throw new Error("topK must be a positive integer");
|
|
515
|
+
}
|
|
516
|
+
if (!Array.isArray(queryVector) || !queryVector.every((x) => typeof x === "number" && Number.isFinite(x))) {
|
|
517
|
+
throw new Error("queryVector must be an array of finite numbers");
|
|
518
|
+
}
|
|
519
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
447
520
|
const vectorStr = `[${queryVector.join(",")}]`;
|
|
448
521
|
const translatedFilter = this.transformFilter(filter);
|
|
449
522
|
const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter);
|
|
450
523
|
filterValues.push(minScore);
|
|
524
|
+
filterValues.push(topK);
|
|
451
525
|
const query = `
|
|
452
526
|
WITH vector_scores AS (
|
|
453
527
|
SELECT
|
|
@@ -455,14 +529,14 @@ var LibSQLVector = class extends MastraVector {
|
|
|
455
529
|
(1-vector_distance_cos(embedding, '${vectorStr}')) as score,
|
|
456
530
|
metadata
|
|
457
531
|
${includeVector ? ", vector_extract(embedding) as embedding" : ""}
|
|
458
|
-
FROM ${
|
|
532
|
+
FROM ${parsedIndexName}
|
|
459
533
|
${filterQuery}
|
|
460
534
|
)
|
|
461
535
|
SELECT *
|
|
462
536
|
FROM vector_scores
|
|
463
537
|
WHERE score > ?
|
|
464
538
|
ORDER BY score DESC
|
|
465
|
-
LIMIT
|
|
539
|
+
LIMIT ?`;
|
|
466
540
|
const result = await this.turso.execute({
|
|
467
541
|
sql: query,
|
|
468
542
|
args: filterValues
|
|
@@ -481,10 +555,11 @@ var LibSQLVector = class extends MastraVector {
|
|
|
481
555
|
const { indexName, vectors, metadata, ids } = params;
|
|
482
556
|
const tx = await this.turso.transaction("write");
|
|
483
557
|
try {
|
|
558
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
484
559
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
485
560
|
for (let i = 0; i < vectors.length; i++) {
|
|
486
561
|
const query = `
|
|
487
|
-
INSERT INTO ${
|
|
562
|
+
INSERT INTO ${parsedIndexName} (vector_id, embedding, metadata)
|
|
488
563
|
VALUES (?, vector32(?), ?)
|
|
489
564
|
ON CONFLICT(vector_id) DO UPDATE SET
|
|
490
565
|
embedding = vector32(?),
|
|
@@ -522,15 +597,13 @@ var LibSQLVector = class extends MastraVector {
|
|
|
522
597
|
const params = this.normalizeArgs("createIndex", args);
|
|
523
598
|
const { indexName, dimension } = params;
|
|
524
599
|
try {
|
|
525
|
-
if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
|
|
526
|
-
throw new Error("Invalid index name format");
|
|
527
|
-
}
|
|
528
600
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
529
601
|
throw new Error("Dimension must be a positive integer");
|
|
530
602
|
}
|
|
603
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
531
604
|
await this.turso.execute({
|
|
532
605
|
sql: `
|
|
533
|
-
CREATE TABLE IF NOT EXISTS ${
|
|
606
|
+
CREATE TABLE IF NOT EXISTS ${parsedIndexName} (
|
|
534
607
|
id SERIAL PRIMARY KEY,
|
|
535
608
|
vector_id TEXT UNIQUE NOT NULL,
|
|
536
609
|
embedding F32_BLOB(${dimension}),
|
|
@@ -541,8 +614,8 @@ var LibSQLVector = class extends MastraVector {
|
|
|
541
614
|
});
|
|
542
615
|
await this.turso.execute({
|
|
543
616
|
sql: `
|
|
544
|
-
CREATE INDEX IF NOT EXISTS ${
|
|
545
|
-
ON ${
|
|
617
|
+
CREATE INDEX IF NOT EXISTS ${parsedIndexName}_vector_idx
|
|
618
|
+
ON ${parsedIndexName} (libsql_vector_idx(embedding))
|
|
546
619
|
`,
|
|
547
620
|
args: []
|
|
548
621
|
});
|
|
@@ -552,10 +625,13 @@ var LibSQLVector = class extends MastraVector {
|
|
|
552
625
|
} finally {
|
|
553
626
|
}
|
|
554
627
|
}
|
|
555
|
-
async deleteIndex(
|
|
628
|
+
async deleteIndex(...args) {
|
|
629
|
+
const params = this.normalizeArgs("deleteIndex", args);
|
|
630
|
+
const { indexName } = params;
|
|
556
631
|
try {
|
|
632
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
557
633
|
await this.turso.execute({
|
|
558
|
-
sql: `DROP TABLE IF EXISTS ${
|
|
634
|
+
sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
|
|
559
635
|
args: []
|
|
560
636
|
});
|
|
561
637
|
} catch (error) {
|
|
@@ -580,8 +656,18 @@ var LibSQLVector = class extends MastraVector {
|
|
|
580
656
|
throw new Error(`Failed to list vector tables: ${error.message}`);
|
|
581
657
|
}
|
|
582
658
|
}
|
|
583
|
-
|
|
659
|
+
/**
|
|
660
|
+
* Retrieves statistics about a vector index.
|
|
661
|
+
*
|
|
662
|
+
* @param params - The parameters for describing an index
|
|
663
|
+
* @param params.indexName - The name of the index to describe
|
|
664
|
+
* @returns A promise that resolves to the index statistics including dimension, count and metric
|
|
665
|
+
*/
|
|
666
|
+
async describeIndex(...args) {
|
|
667
|
+
const params = this.normalizeArgs("describeIndex", args);
|
|
668
|
+
const { indexName } = params;
|
|
584
669
|
try {
|
|
670
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
585
671
|
const tableInfoQuery = `
|
|
586
672
|
SELECT sql
|
|
587
673
|
FROM sqlite_master
|
|
@@ -590,15 +676,15 @@ var LibSQLVector = class extends MastraVector {
|
|
|
590
676
|
`;
|
|
591
677
|
const tableInfo = await this.turso.execute({
|
|
592
678
|
sql: tableInfoQuery,
|
|
593
|
-
args: [
|
|
679
|
+
args: [parsedIndexName]
|
|
594
680
|
});
|
|
595
681
|
if (!tableInfo.rows[0]?.sql) {
|
|
596
|
-
throw new Error(`Table ${
|
|
682
|
+
throw new Error(`Table ${parsedIndexName} not found`);
|
|
597
683
|
}
|
|
598
684
|
const dimension = parseInt(tableInfo.rows[0].sql.match(/F32_BLOB\((\d+)\)/)?.[1] || "0");
|
|
599
685
|
const countQuery = `
|
|
600
686
|
SELECT COUNT(*) as count
|
|
601
|
-
FROM ${
|
|
687
|
+
FROM ${parsedIndexName};
|
|
602
688
|
`;
|
|
603
689
|
const countResult = await this.turso.execute({
|
|
604
690
|
sql: countQuery,
|
|
@@ -632,10 +718,11 @@ var LibSQLVector = class extends MastraVector {
|
|
|
632
718
|
Please use updateVector() instead.
|
|
633
719
|
updateIndexById() will be removed on May 20th, 2025.`
|
|
634
720
|
);
|
|
635
|
-
await this.updateVector(indexName, id, update);
|
|
721
|
+
await this.updateVector({ indexName, id, update });
|
|
636
722
|
}
|
|
637
723
|
/**
|
|
638
724
|
* Updates a vector by its ID with the provided vector and/or metadata.
|
|
725
|
+
*
|
|
639
726
|
* @param indexName - The name of the index containing the vector.
|
|
640
727
|
* @param id - The ID of the vector to update.
|
|
641
728
|
* @param update - An object containing the vector and/or metadata to update.
|
|
@@ -644,30 +731,33 @@ var LibSQLVector = class extends MastraVector {
|
|
|
644
731
|
* @returns A promise that resolves when the update is complete.
|
|
645
732
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
646
733
|
*/
|
|
647
|
-
async updateVector(
|
|
734
|
+
async updateVector(...args) {
|
|
735
|
+
const params = this.normalizeArgs("updateVector", args);
|
|
736
|
+
const { indexName, id, update } = params;
|
|
648
737
|
try {
|
|
738
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
649
739
|
const updates = [];
|
|
650
|
-
const
|
|
740
|
+
const args2 = [];
|
|
651
741
|
if (update.vector) {
|
|
652
742
|
updates.push("embedding = vector32(?)");
|
|
653
|
-
|
|
743
|
+
args2.push(JSON.stringify(update.vector));
|
|
654
744
|
}
|
|
655
745
|
if (update.metadata) {
|
|
656
746
|
updates.push("metadata = ?");
|
|
657
|
-
|
|
747
|
+
args2.push(JSON.stringify(update.metadata));
|
|
658
748
|
}
|
|
659
749
|
if (updates.length === 0) {
|
|
660
750
|
throw new Error("No updates provided");
|
|
661
751
|
}
|
|
662
|
-
|
|
752
|
+
args2.push(id);
|
|
663
753
|
const query = `
|
|
664
|
-
UPDATE ${
|
|
754
|
+
UPDATE ${parsedIndexName}
|
|
665
755
|
SET ${updates.join(", ")}
|
|
666
756
|
WHERE vector_id = ?;
|
|
667
757
|
`;
|
|
668
758
|
await this.turso.execute({
|
|
669
759
|
sql: query,
|
|
670
|
-
args
|
|
760
|
+
args: args2
|
|
671
761
|
});
|
|
672
762
|
} catch (error) {
|
|
673
763
|
throw new Error(`Failed to update vector by id: ${id} for index: ${indexName}: ${error.message}`);
|
|
@@ -688,7 +778,7 @@ var LibSQLVector = class extends MastraVector {
|
|
|
688
778
|
Please use deleteVector() instead.
|
|
689
779
|
deleteIndexById() will be removed on May 20th, 2025.`
|
|
690
780
|
);
|
|
691
|
-
await this.deleteVector(indexName, id);
|
|
781
|
+
await this.deleteVector({ indexName, id });
|
|
692
782
|
}
|
|
693
783
|
/**
|
|
694
784
|
* Deletes a vector by its ID.
|
|
@@ -697,19 +787,24 @@ var LibSQLVector = class extends MastraVector {
|
|
|
697
787
|
* @returns A promise that resolves when the deletion is complete.
|
|
698
788
|
* @throws Will throw an error if the deletion operation fails.
|
|
699
789
|
*/
|
|
700
|
-
async deleteVector(
|
|
790
|
+
async deleteVector(...args) {
|
|
791
|
+
const params = this.normalizeArgs("deleteVector", args);
|
|
792
|
+
const { indexName, id } = params;
|
|
701
793
|
try {
|
|
794
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
702
795
|
await this.turso.execute({
|
|
703
|
-
sql: `DELETE FROM ${
|
|
796
|
+
sql: `DELETE FROM ${parsedIndexName} WHERE vector_id = ?`,
|
|
704
797
|
args: [id]
|
|
705
798
|
});
|
|
706
799
|
} catch (error) {
|
|
707
800
|
throw new Error(`Failed to delete vector by id: ${id} for index: ${indexName}: ${error.message}`);
|
|
708
801
|
}
|
|
709
802
|
}
|
|
710
|
-
async truncateIndex(
|
|
803
|
+
async truncateIndex(...args) {
|
|
804
|
+
const params = this.normalizeArgs("truncateIndex", args);
|
|
805
|
+
const { indexName } = params;
|
|
711
806
|
await this.turso.execute({
|
|
712
|
-
sql: `DELETE FROM ${indexName}`,
|
|
807
|
+
sql: `DELETE FROM ${parseSqlIdentifier(indexName, "index name")}`,
|
|
713
808
|
args: []
|
|
714
809
|
});
|
|
715
810
|
}
|
|
@@ -731,22 +826,24 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
731
826
|
this.client = createClient(config);
|
|
732
827
|
}
|
|
733
828
|
getCreateTableSQL(tableName, schema) {
|
|
829
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
734
830
|
const columns = Object.entries(schema).map(([name, col]) => {
|
|
831
|
+
const parsedColumnName = parseSqlIdentifier(name, "column name");
|
|
735
832
|
let type = col.type.toUpperCase();
|
|
736
833
|
if (type === "TEXT") type = "TEXT";
|
|
737
834
|
if (type === "TIMESTAMP") type = "TEXT";
|
|
738
835
|
const nullable = col.nullable ? "" : "NOT NULL";
|
|
739
836
|
const primaryKey = col.primaryKey ? "PRIMARY KEY" : "";
|
|
740
|
-
return `${
|
|
837
|
+
return `${parsedColumnName} ${type} ${nullable} ${primaryKey}`.trim();
|
|
741
838
|
});
|
|
742
839
|
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
743
|
-
const stmnt = `CREATE TABLE IF NOT EXISTS ${
|
|
840
|
+
const stmnt = `CREATE TABLE IF NOT EXISTS ${parsedTableName} (
|
|
744
841
|
${columns.join(",\n")},
|
|
745
842
|
PRIMARY KEY (workflow_name, run_id)
|
|
746
843
|
)`;
|
|
747
844
|
return stmnt;
|
|
748
845
|
}
|
|
749
|
-
return `CREATE TABLE IF NOT EXISTS ${
|
|
846
|
+
return `CREATE TABLE IF NOT EXISTS ${parsedTableName} (${columns.join(", ")})`;
|
|
750
847
|
}
|
|
751
848
|
async createTable({
|
|
752
849
|
tableName,
|
|
@@ -762,8 +859,9 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
762
859
|
}
|
|
763
860
|
}
|
|
764
861
|
async clearTable({ tableName }) {
|
|
862
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
765
863
|
try {
|
|
766
|
-
await this.client.execute(`DELETE FROM ${
|
|
864
|
+
await this.client.execute(`DELETE FROM ${parsedTableName}`);
|
|
767
865
|
} catch (e) {
|
|
768
866
|
if (e instanceof Error) {
|
|
769
867
|
this.logger.error(e.message);
|
|
@@ -771,7 +869,8 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
771
869
|
}
|
|
772
870
|
}
|
|
773
871
|
prepareStatement({ tableName, record }) {
|
|
774
|
-
const
|
|
872
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
873
|
+
const columns = Object.keys(record).map((col) => parseSqlIdentifier(col, "column name"));
|
|
775
874
|
const values = Object.values(record).map((v) => {
|
|
776
875
|
if (typeof v === `undefined`) {
|
|
777
876
|
return null;
|
|
@@ -783,7 +882,7 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
783
882
|
});
|
|
784
883
|
const placeholders = values.map(() => "?").join(", ");
|
|
785
884
|
return {
|
|
786
|
-
sql: `INSERT OR REPLACE INTO ${
|
|
885
|
+
sql: `INSERT OR REPLACE INTO ${parsedTableName} (${columns.join(", ")}) VALUES (${placeholders})`,
|
|
787
886
|
args: values
|
|
788
887
|
};
|
|
789
888
|
}
|
|
@@ -811,10 +910,12 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
811
910
|
}
|
|
812
911
|
}
|
|
813
912
|
async load({ tableName, keys }) {
|
|
814
|
-
const
|
|
913
|
+
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
914
|
+
const parsedKeys = Object.keys(keys).map((key) => parseSqlIdentifier(key, "column name"));
|
|
915
|
+
const conditions = parsedKeys.map((key) => `${key} = ?`).join(" AND ");
|
|
815
916
|
const values = Object.values(keys);
|
|
816
917
|
const result = await this.client.execute({
|
|
817
|
-
sql: `SELECT * FROM ${
|
|
918
|
+
sql: `SELECT * FROM ${parsedTableName} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
818
919
|
args: values
|
|
819
920
|
});
|
|
820
921
|
if (!result.rows || result.rows.length === 0) {
|