@mastra/libsql 0.10.0 → 0.10.1-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +18 -0
- package/dist/_tsup-dts-rollup.d.cts +60 -20
- package/dist/_tsup-dts-rollup.d.ts +60 -20
- package/dist/index.cjs +190 -127
- package/dist/index.d.cts +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +190 -127
- package/package.json +5 -5
- package/src/storage/index.test.ts +0 -1
- package/src/storage/index.ts +102 -31
- package/src/vector/index.ts +166 -128
package/dist/index.js
CHANGED
|
@@ -482,11 +482,15 @@ var processOperator = (key, operator, operatorValue) => {
|
|
|
482
482
|
// src/vector/index.ts
|
|
483
483
|
var LibSQLVector = class extends MastraVector {
|
|
484
484
|
turso;
|
|
485
|
+
maxRetries;
|
|
486
|
+
initialBackoffMs;
|
|
485
487
|
constructor({
|
|
486
488
|
connectionUrl,
|
|
487
489
|
authToken,
|
|
488
490
|
syncUrl,
|
|
489
|
-
syncInterval
|
|
491
|
+
syncInterval,
|
|
492
|
+
maxRetries = 5,
|
|
493
|
+
initialBackoffMs = 100
|
|
490
494
|
}) {
|
|
491
495
|
super();
|
|
492
496
|
this.turso = createClient({
|
|
@@ -495,12 +499,40 @@ var LibSQLVector = class extends MastraVector {
|
|
|
495
499
|
authToken,
|
|
496
500
|
syncInterval
|
|
497
501
|
});
|
|
502
|
+
this.maxRetries = maxRetries;
|
|
503
|
+
this.initialBackoffMs = initialBackoffMs;
|
|
498
504
|
if (connectionUrl.includes(`file:`) || connectionUrl.includes(`:memory:`)) {
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
505
|
+
this.turso.execute("PRAGMA journal_mode=WAL;").then(() => this.logger.debug("LibSQLStore: PRAGMA journal_mode=WAL set.")).catch((err) => this.logger.warn("LibSQLStore: Failed to set PRAGMA journal_mode=WAL.", err));
|
|
506
|
+
this.turso.execute("PRAGMA busy_timeout = 5000;").then(() => this.logger.debug("LibSQLStore: PRAGMA busy_timeout=5000 set.")).catch((err) => this.logger.warn("LibSQLStore: Failed to set PRAGMA busy_timeout=5000.", err));
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
async executeWriteOperationWithRetry(operation, isTransaction = false) {
|
|
510
|
+
let attempts = 0;
|
|
511
|
+
let backoff = this.initialBackoffMs;
|
|
512
|
+
while (attempts < this.maxRetries) {
|
|
513
|
+
try {
|
|
514
|
+
return await operation();
|
|
515
|
+
} catch (error) {
|
|
516
|
+
if (error.code === "SQLITE_BUSY" || error.message && error.message.toLowerCase().includes("database is locked")) {
|
|
517
|
+
attempts++;
|
|
518
|
+
if (attempts >= this.maxRetries) {
|
|
519
|
+
this.logger.error(
|
|
520
|
+
`LibSQLVector: Operation failed after ${this.maxRetries} attempts due to: ${error.message}`,
|
|
521
|
+
error
|
|
522
|
+
);
|
|
523
|
+
throw error;
|
|
524
|
+
}
|
|
525
|
+
this.logger.warn(
|
|
526
|
+
`LibSQLVector: Attempt ${attempts} failed due to ${isTransaction ? "transaction " : ""}database lock. Retrying in ${backoff}ms...`
|
|
527
|
+
);
|
|
528
|
+
await new Promise((resolve) => setTimeout(resolve, backoff));
|
|
529
|
+
backoff *= 2;
|
|
530
|
+
} else {
|
|
531
|
+
throw error;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
503
534
|
}
|
|
535
|
+
throw new Error("LibSQLVector: Max retries reached, but no error was re-thrown from the loop.");
|
|
504
536
|
}
|
|
505
537
|
transformFilter(filter) {
|
|
506
538
|
const translator = new LibSQLFilterTranslator();
|
|
@@ -528,20 +560,20 @@ var LibSQLVector = class extends MastraVector {
|
|
|
528
560
|
filterValues.push(minScore);
|
|
529
561
|
filterValues.push(topK);
|
|
530
562
|
const query = `
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
563
|
+
WITH vector_scores AS (
|
|
564
|
+
SELECT
|
|
565
|
+
vector_id as id,
|
|
566
|
+
(1-vector_distance_cos(embedding, '${vectorStr}')) as score,
|
|
567
|
+
metadata
|
|
568
|
+
${includeVector ? ", vector_extract(embedding) as embedding" : ""}
|
|
569
|
+
FROM ${parsedIndexName}
|
|
570
|
+
${filterQuery}
|
|
571
|
+
)
|
|
572
|
+
SELECT *
|
|
573
|
+
FROM vector_scores
|
|
574
|
+
WHERE score > ?
|
|
575
|
+
ORDER BY score DESC
|
|
576
|
+
LIMIT ?`;
|
|
545
577
|
const result = await this.turso.execute({
|
|
546
578
|
sql: query,
|
|
547
579
|
args: filterValues
|
|
@@ -555,22 +587,24 @@ var LibSQLVector = class extends MastraVector {
|
|
|
555
587
|
} finally {
|
|
556
588
|
}
|
|
557
589
|
}
|
|
558
|
-
|
|
590
|
+
upsert(args) {
|
|
591
|
+
return this.executeWriteOperationWithRetry(() => this.doUpsert(args), true);
|
|
592
|
+
}
|
|
593
|
+
async doUpsert({ indexName, vectors, metadata, ids }) {
|
|
559
594
|
const tx = await this.turso.transaction("write");
|
|
560
595
|
try {
|
|
561
596
|
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
562
597
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
563
598
|
for (let i = 0; i < vectors.length; i++) {
|
|
564
599
|
const query = `
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
600
|
+
INSERT INTO ${parsedIndexName} (vector_id, embedding, metadata)
|
|
601
|
+
VALUES (?, vector32(?), ?)
|
|
602
|
+
ON CONFLICT(vector_id) DO UPDATE SET
|
|
603
|
+
embedding = vector32(?),
|
|
604
|
+
metadata = ?
|
|
605
|
+
`;
|
|
571
606
|
await tx.execute({
|
|
572
607
|
sql: query,
|
|
573
|
-
// @ts-ignore
|
|
574
608
|
args: [
|
|
575
609
|
vectorIds[i],
|
|
576
610
|
JSON.stringify(vectors[i]),
|
|
@@ -596,48 +630,42 @@ var LibSQLVector = class extends MastraVector {
|
|
|
596
630
|
throw error;
|
|
597
631
|
}
|
|
598
632
|
}
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
602
|
-
throw new Error("Dimension must be a positive integer");
|
|
603
|
-
}
|
|
604
|
-
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
605
|
-
await this.turso.execute({
|
|
606
|
-
sql: `
|
|
607
|
-
CREATE TABLE IF NOT EXISTS ${parsedIndexName} (
|
|
608
|
-
id SERIAL PRIMARY KEY,
|
|
609
|
-
vector_id TEXT UNIQUE NOT NULL,
|
|
610
|
-
embedding F32_BLOB(${dimension}),
|
|
611
|
-
metadata TEXT DEFAULT '{}'
|
|
612
|
-
);
|
|
613
|
-
`,
|
|
614
|
-
args: []
|
|
615
|
-
});
|
|
616
|
-
await this.turso.execute({
|
|
617
|
-
sql: `
|
|
618
|
-
CREATE INDEX IF NOT EXISTS ${parsedIndexName}_vector_idx
|
|
619
|
-
ON ${parsedIndexName} (libsql_vector_idx(embedding))
|
|
620
|
-
`,
|
|
621
|
-
args: []
|
|
622
|
-
});
|
|
623
|
-
} catch (error) {
|
|
624
|
-
console.error("Failed to create vector table:", error);
|
|
625
|
-
throw error;
|
|
626
|
-
} finally {
|
|
627
|
-
}
|
|
633
|
+
createIndex(args) {
|
|
634
|
+
return this.executeWriteOperationWithRetry(() => this.doCreateIndex(args));
|
|
628
635
|
}
|
|
629
|
-
async
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
await this.turso.execute({
|
|
633
|
-
sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
|
|
634
|
-
args: []
|
|
635
|
-
});
|
|
636
|
-
} catch (error) {
|
|
637
|
-
console.error("Failed to delete vector table:", error);
|
|
638
|
-
throw new Error(`Failed to delete vector table: ${error.message}`);
|
|
639
|
-
} finally {
|
|
636
|
+
async doCreateIndex({ indexName, dimension }) {
|
|
637
|
+
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
638
|
+
throw new Error("Dimension must be a positive integer");
|
|
640
639
|
}
|
|
640
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
641
|
+
await this.turso.execute({
|
|
642
|
+
sql: `
|
|
643
|
+
CREATE TABLE IF NOT EXISTS ${parsedIndexName} (
|
|
644
|
+
id SERIAL PRIMARY KEY,
|
|
645
|
+
vector_id TEXT UNIQUE NOT NULL,
|
|
646
|
+
embedding F32_BLOB(${dimension}),
|
|
647
|
+
metadata TEXT DEFAULT '{}'
|
|
648
|
+
);
|
|
649
|
+
`,
|
|
650
|
+
args: []
|
|
651
|
+
});
|
|
652
|
+
await this.turso.execute({
|
|
653
|
+
sql: `
|
|
654
|
+
CREATE INDEX IF NOT EXISTS ${parsedIndexName}_vector_idx
|
|
655
|
+
ON ${parsedIndexName} (libsql_vector_idx(embedding))
|
|
656
|
+
`,
|
|
657
|
+
args: []
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
deleteIndex(args) {
|
|
661
|
+
return this.executeWriteOperationWithRetry(() => this.doDeleteIndex(args));
|
|
662
|
+
}
|
|
663
|
+
async doDeleteIndex({ indexName }) {
|
|
664
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
665
|
+
await this.turso.execute({
|
|
666
|
+
sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
|
|
667
|
+
args: []
|
|
668
|
+
});
|
|
641
669
|
}
|
|
642
670
|
async listIndexes() {
|
|
643
671
|
try {
|
|
@@ -707,35 +735,34 @@ var LibSQLVector = class extends MastraVector {
|
|
|
707
735
|
* @returns A promise that resolves when the update is complete.
|
|
708
736
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
709
737
|
*/
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
738
|
+
updateVector(args) {
|
|
739
|
+
return this.executeWriteOperationWithRetry(() => this.doUpdateVector(args));
|
|
740
|
+
}
|
|
741
|
+
async doUpdateVector({ indexName, id, update }) {
|
|
742
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
743
|
+
const updates = [];
|
|
744
|
+
const args = [];
|
|
745
|
+
if (update.vector) {
|
|
746
|
+
updates.push("embedding = vector32(?)");
|
|
747
|
+
args.push(JSON.stringify(update.vector));
|
|
748
|
+
}
|
|
749
|
+
if (update.metadata) {
|
|
750
|
+
updates.push("metadata = ?");
|
|
751
|
+
args.push(JSON.stringify(update.metadata));
|
|
752
|
+
}
|
|
753
|
+
if (updates.length === 0) {
|
|
754
|
+
throw new Error("No updates provided");
|
|
755
|
+
}
|
|
756
|
+
args.push(id);
|
|
757
|
+
const query = `
|
|
728
758
|
UPDATE ${parsedIndexName}
|
|
729
759
|
SET ${updates.join(", ")}
|
|
730
760
|
WHERE vector_id = ?;
|
|
731
761
|
`;
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
} catch (error) {
|
|
737
|
-
throw new Error(`Failed to update vector by id: ${id} for index: ${indexName}: ${error.message}`);
|
|
738
|
-
}
|
|
762
|
+
await this.turso.execute({
|
|
763
|
+
sql: query,
|
|
764
|
+
args
|
|
765
|
+
});
|
|
739
766
|
}
|
|
740
767
|
/**
|
|
741
768
|
* Deletes a vector by its ID.
|
|
@@ -744,18 +771,20 @@ var LibSQLVector = class extends MastraVector {
|
|
|
744
771
|
* @returns A promise that resolves when the deletion is complete.
|
|
745
772
|
* @throws Will throw an error if the deletion operation fails.
|
|
746
773
|
*/
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
774
|
+
deleteVector(args) {
|
|
775
|
+
return this.executeWriteOperationWithRetry(() => this.doDeleteVector(args));
|
|
776
|
+
}
|
|
777
|
+
async doDeleteVector({ indexName, id }) {
|
|
778
|
+
const parsedIndexName = parseSqlIdentifier(indexName, "index name");
|
|
779
|
+
await this.turso.execute({
|
|
780
|
+
sql: `DELETE FROM ${parsedIndexName} WHERE vector_id = ?`,
|
|
781
|
+
args: [id]
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
truncateIndex(args) {
|
|
785
|
+
return this.executeWriteOperationWithRetry(() => this._doTruncateIndex(args));
|
|
757
786
|
}
|
|
758
|
-
async
|
|
787
|
+
async _doTruncateIndex({ indexName }) {
|
|
759
788
|
await this.turso.execute({
|
|
760
789
|
sql: `DELETE FROM ${parseSqlIdentifier(indexName, "index name")}`,
|
|
761
790
|
args: []
|
|
@@ -771,12 +800,20 @@ function safelyParseJSON(jsonString) {
|
|
|
771
800
|
}
|
|
772
801
|
var LibSQLStore = class extends MastraStorage {
|
|
773
802
|
client;
|
|
803
|
+
maxRetries;
|
|
804
|
+
initialBackoffMs;
|
|
774
805
|
constructor(config) {
|
|
775
806
|
super({ name: `LibSQLStore` });
|
|
807
|
+
this.maxRetries = config.maxRetries ?? 5;
|
|
808
|
+
this.initialBackoffMs = config.initialBackoffMs ?? 100;
|
|
776
809
|
if (config.url.endsWith(":memory:")) {
|
|
777
810
|
this.shouldCacheInit = false;
|
|
778
811
|
}
|
|
779
812
|
this.client = createClient(config);
|
|
813
|
+
if (config.url.startsWith("file:") || config.url.includes(":memory:")) {
|
|
814
|
+
this.client.execute("PRAGMA journal_mode=WAL;").then(() => this.logger.debug("LibSQLStore: PRAGMA journal_mode=WAL set.")).catch((err) => this.logger.warn("LibSQLStore: Failed to set PRAGMA journal_mode=WAL.", err));
|
|
815
|
+
this.client.execute("PRAGMA busy_timeout = 5000;").then(() => this.logger.debug("LibSQLStore: PRAGMA busy_timeout=5000 set.")).catch((err) => this.logger.warn("LibSQLStore: Failed to set PRAGMA busy_timeout.", err));
|
|
816
|
+
}
|
|
780
817
|
}
|
|
781
818
|
getCreateTableSQL(tableName, schema) {
|
|
782
819
|
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
@@ -839,28 +876,53 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
839
876
|
args: values
|
|
840
877
|
};
|
|
841
878
|
}
|
|
842
|
-
async
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
879
|
+
async executeWriteOperationWithRetry(operationFn, operationDescription) {
|
|
880
|
+
let retries = 0;
|
|
881
|
+
while (true) {
|
|
882
|
+
try {
|
|
883
|
+
return await operationFn();
|
|
884
|
+
} catch (error) {
|
|
885
|
+
if (error.message && (error.message.includes("SQLITE_BUSY") || error.message.includes("database is locked")) && retries < this.maxRetries) {
|
|
886
|
+
retries++;
|
|
887
|
+
const backoffTime = this.initialBackoffMs * Math.pow(2, retries - 1);
|
|
888
|
+
this.logger.warn(
|
|
889
|
+
`LibSQLStore: Encountered SQLITE_BUSY during ${operationDescription}. Retrying (${retries}/${this.maxRetries}) in ${backoffTime}ms...`
|
|
890
|
+
);
|
|
891
|
+
await new Promise((resolve) => setTimeout(resolve, backoffTime));
|
|
892
|
+
} else {
|
|
893
|
+
this.logger.error(`LibSQLStore: Error during ${operationDescription} after ${retries} retries: ${error}`);
|
|
894
|
+
throw error;
|
|
895
|
+
}
|
|
896
|
+
}
|
|
853
897
|
}
|
|
854
898
|
}
|
|
855
|
-
|
|
899
|
+
insert(args) {
|
|
900
|
+
return this.executeWriteOperationWithRetry(() => this.doInsert(args), `insert into table ${args.tableName}`);
|
|
901
|
+
}
|
|
902
|
+
async doInsert({
|
|
903
|
+
tableName,
|
|
904
|
+
record
|
|
905
|
+
}) {
|
|
906
|
+
await this.client.execute(
|
|
907
|
+
this.prepareStatement({
|
|
908
|
+
tableName,
|
|
909
|
+
record
|
|
910
|
+
})
|
|
911
|
+
);
|
|
912
|
+
}
|
|
913
|
+
batchInsert(args) {
|
|
914
|
+
return this.executeWriteOperationWithRetry(
|
|
915
|
+
() => this.doBatchInsert(args),
|
|
916
|
+
`batch insert into table ${args.tableName}`
|
|
917
|
+
);
|
|
918
|
+
}
|
|
919
|
+
async doBatchInsert({
|
|
920
|
+
tableName,
|
|
921
|
+
records
|
|
922
|
+
}) {
|
|
856
923
|
if (records.length === 0) return;
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
await this.client.batch(batchStatements, "write");
|
|
860
|
-
} catch (error) {
|
|
861
|
-
this.logger.error(`Error upserting into table ${tableName}: ${error}`);
|
|
862
|
-
throw error;
|
|
863
|
-
}
|
|
924
|
+
const batchStatements = records.map((r) => this.prepareStatement({ tableName, record: r }));
|
|
925
|
+
await this.client.batch(batchStatements, "write");
|
|
864
926
|
}
|
|
865
927
|
async load({ tableName, keys }) {
|
|
866
928
|
const parsedTableName = parseSqlIdentifier(tableName, "table name");
|
|
@@ -961,14 +1023,15 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
961
1023
|
content = JSON.parse(row.content);
|
|
962
1024
|
} catch {
|
|
963
1025
|
}
|
|
964
|
-
|
|
1026
|
+
const result = {
|
|
965
1027
|
id: row.id,
|
|
966
1028
|
content,
|
|
967
1029
|
role: row.role,
|
|
968
|
-
type: row.type,
|
|
969
1030
|
createdAt: new Date(row.createdAt),
|
|
970
1031
|
threadId: row.thread_id
|
|
971
1032
|
};
|
|
1033
|
+
if (row.type && row.type !== `v2`) result.type = row.type;
|
|
1034
|
+
return result;
|
|
972
1035
|
}
|
|
973
1036
|
async getMessages({ threadId, selectBy }) {
|
|
974
1037
|
try {
|
|
@@ -1056,7 +1119,7 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
1056
1119
|
threadId,
|
|
1057
1120
|
typeof message.content === "object" ? JSON.stringify(message.content) : message.content,
|
|
1058
1121
|
message.role,
|
|
1059
|
-
message.type,
|
|
1122
|
+
message.type || "v2",
|
|
1060
1123
|
time instanceof Date ? time.toISOString() : time
|
|
1061
1124
|
]
|
|
1062
1125
|
};
|
|
@@ -1288,8 +1351,8 @@ var LibSQLStore = class extends MastraStorage {
|
|
|
1288
1351
|
runId: row.run_id,
|
|
1289
1352
|
snapshot: parsedSnapshot,
|
|
1290
1353
|
resourceId: row.resourceId,
|
|
1291
|
-
createdAt: new Date(row.
|
|
1292
|
-
updatedAt: new Date(row.
|
|
1354
|
+
createdAt: new Date(row.createdAt),
|
|
1355
|
+
updatedAt: new Date(row.updatedAt)
|
|
1293
1356
|
};
|
|
1294
1357
|
}
|
|
1295
1358
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mastra/libsql",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.1-alpha.1",
|
|
4
4
|
"description": "Libsql provider for Mastra - includes both vector and db storage capabilities",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -29,12 +29,12 @@
|
|
|
29
29
|
"tsup": "^8.4.0",
|
|
30
30
|
"typescript": "^5.8.3",
|
|
31
31
|
"vitest": "^3.1.2",
|
|
32
|
-
"@internal/lint": "0.0.
|
|
33
|
-
"@internal/storage-test-utils": "0.0.
|
|
34
|
-
"@mastra/core": "0.10.
|
|
32
|
+
"@internal/lint": "0.0.7",
|
|
33
|
+
"@internal/storage-test-utils": "0.0.3",
|
|
34
|
+
"@mastra/core": "0.10.2-alpha.2"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
|
-
"@mastra/core": "^0.10.0"
|
|
37
|
+
"@mastra/core": "^0.10.0-alpha.0"
|
|
38
38
|
},
|
|
39
39
|
"scripts": {
|
|
40
40
|
"build": "tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting",
|
package/src/storage/index.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { createClient } from '@libsql/client';
|
|
2
2
|
import type { Client, InValue } from '@libsql/client';
|
|
3
|
+
import type { MastraMessageV2 } from '@mastra/core/agent';
|
|
3
4
|
import type { MetricResult, TestInfo } from '@mastra/core/eval';
|
|
4
|
-
import type {
|
|
5
|
+
import type { StorageThreadType } from '@mastra/core/memory';
|
|
5
6
|
import {
|
|
6
7
|
MastraStorage,
|
|
7
8
|
TABLE_MESSAGES,
|
|
@@ -32,20 +33,48 @@ function safelyParseJSON(jsonString: string): any {
|
|
|
32
33
|
export interface LibSQLConfig {
|
|
33
34
|
url: string;
|
|
34
35
|
authToken?: string;
|
|
36
|
+
/**
|
|
37
|
+
* Maximum number of retries for write operations if an SQLITE_BUSY error occurs.
|
|
38
|
+
* @default 5
|
|
39
|
+
*/
|
|
40
|
+
maxRetries?: number;
|
|
41
|
+
/**
|
|
42
|
+
* Initial backoff time in milliseconds for retrying write operations on SQLITE_BUSY.
|
|
43
|
+
* The backoff time will double with each retry (exponential backoff).
|
|
44
|
+
* @default 100
|
|
45
|
+
*/
|
|
46
|
+
initialBackoffMs?: number;
|
|
35
47
|
}
|
|
36
48
|
|
|
37
49
|
export class LibSQLStore extends MastraStorage {
|
|
38
50
|
private client: Client;
|
|
51
|
+
private readonly maxRetries: number;
|
|
52
|
+
private readonly initialBackoffMs: number;
|
|
39
53
|
|
|
40
54
|
constructor(config: LibSQLConfig) {
|
|
41
55
|
super({ name: `LibSQLStore` });
|
|
42
56
|
|
|
57
|
+
this.maxRetries = config.maxRetries ?? 5;
|
|
58
|
+
this.initialBackoffMs = config.initialBackoffMs ?? 100;
|
|
59
|
+
|
|
43
60
|
// need to re-init every time for in memory dbs or the tables might not exist
|
|
44
61
|
if (config.url.endsWith(':memory:')) {
|
|
45
62
|
this.shouldCacheInit = false;
|
|
46
63
|
}
|
|
47
64
|
|
|
48
65
|
this.client = createClient(config);
|
|
66
|
+
|
|
67
|
+
// Set PRAGMAs for better concurrency, especially for file-based databases
|
|
68
|
+
if (config.url.startsWith('file:') || config.url.includes(':memory:')) {
|
|
69
|
+
this.client
|
|
70
|
+
.execute('PRAGMA journal_mode=WAL;')
|
|
71
|
+
.then(() => this.logger.debug('LibSQLStore: PRAGMA journal_mode=WAL set.'))
|
|
72
|
+
.catch(err => this.logger.warn('LibSQLStore: Failed to set PRAGMA journal_mode=WAL.', err));
|
|
73
|
+
this.client
|
|
74
|
+
.execute('PRAGMA busy_timeout = 5000;') // 5 seconds
|
|
75
|
+
.then(() => this.logger.debug('LibSQLStore: PRAGMA busy_timeout=5000 set.'))
|
|
76
|
+
.catch(err => this.logger.warn('LibSQLStore: Failed to set PRAGMA busy_timeout.', err));
|
|
77
|
+
}
|
|
49
78
|
}
|
|
50
79
|
|
|
51
80
|
private getCreateTableSQL(tableName: TABLE_NAMES, schema: Record<string, StorageColumn>): string {
|
|
@@ -127,30 +156,71 @@ export class LibSQLStore extends MastraStorage {
|
|
|
127
156
|
};
|
|
128
157
|
}
|
|
129
158
|
|
|
130
|
-
async
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
159
|
+
private async executeWriteOperationWithRetry<T>(
|
|
160
|
+
operationFn: () => Promise<T>,
|
|
161
|
+
operationDescription: string,
|
|
162
|
+
): Promise<T> {
|
|
163
|
+
let retries = 0;
|
|
164
|
+
|
|
165
|
+
while (true) {
|
|
166
|
+
try {
|
|
167
|
+
return await operationFn();
|
|
168
|
+
} catch (error: any) {
|
|
169
|
+
if (
|
|
170
|
+
error.message &&
|
|
171
|
+
(error.message.includes('SQLITE_BUSY') || error.message.includes('database is locked')) &&
|
|
172
|
+
retries < this.maxRetries
|
|
173
|
+
) {
|
|
174
|
+
retries++;
|
|
175
|
+
const backoffTime = this.initialBackoffMs * Math.pow(2, retries - 1);
|
|
176
|
+
this.logger.warn(
|
|
177
|
+
`LibSQLStore: Encountered SQLITE_BUSY during ${operationDescription}. Retrying (${retries}/${this.maxRetries}) in ${backoffTime}ms...`,
|
|
178
|
+
);
|
|
179
|
+
await new Promise(resolve => setTimeout(resolve, backoffTime));
|
|
180
|
+
} else {
|
|
181
|
+
this.logger.error(`LibSQLStore: Error during ${operationDescription} after ${retries} retries: ${error}`);
|
|
182
|
+
throw error;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
141
185
|
}
|
|
142
186
|
}
|
|
143
187
|
|
|
144
|
-
|
|
145
|
-
|
|
188
|
+
public insert(args: { tableName: TABLE_NAMES; record: Record<string, any> }): Promise<void> {
|
|
189
|
+
return this.executeWriteOperationWithRetry(() => this.doInsert(args), `insert into table ${args.tableName}`);
|
|
190
|
+
}
|
|
146
191
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
192
|
+
private async doInsert({
|
|
193
|
+
tableName,
|
|
194
|
+
record,
|
|
195
|
+
}: {
|
|
196
|
+
tableName: TABLE_NAMES;
|
|
197
|
+
record: Record<string, any>;
|
|
198
|
+
}): Promise<void> {
|
|
199
|
+
await this.client.execute(
|
|
200
|
+
this.prepareStatement({
|
|
201
|
+
tableName,
|
|
202
|
+
record,
|
|
203
|
+
}),
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
public batchInsert(args: { tableName: TABLE_NAMES; records: Record<string, any>[] }): Promise<void> {
|
|
208
|
+
return this.executeWriteOperationWithRetry(
|
|
209
|
+
() => this.doBatchInsert(args),
|
|
210
|
+
`batch insert into table ${args.tableName}`,
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
private async doBatchInsert({
|
|
215
|
+
tableName,
|
|
216
|
+
records,
|
|
217
|
+
}: {
|
|
218
|
+
tableName: TABLE_NAMES;
|
|
219
|
+
records: Record<string, any>[];
|
|
220
|
+
}): Promise<void> {
|
|
221
|
+
if (records.length === 0) return;
|
|
222
|
+
const batchStatements = records.map(r => this.prepareStatement({ tableName, record: r }));
|
|
223
|
+
await this.client.batch(batchStatements, 'write');
|
|
154
224
|
}
|
|
155
225
|
|
|
156
226
|
async load<R>({ tableName, keys }: { tableName: TABLE_NAMES; keys: Record<string, string> }): Promise<R | null> {
|
|
@@ -274,26 +344,27 @@ export class LibSQLStore extends MastraStorage {
|
|
|
274
344
|
// Messages will be automatically deleted due to CASCADE constraint
|
|
275
345
|
}
|
|
276
346
|
|
|
277
|
-
private parseRow(row: any):
|
|
347
|
+
private parseRow(row: any): MastraMessageV2 {
|
|
278
348
|
let content = row.content;
|
|
279
349
|
try {
|
|
280
350
|
content = JSON.parse(row.content);
|
|
281
351
|
} catch {
|
|
282
352
|
// use content as is if it's not JSON
|
|
283
353
|
}
|
|
284
|
-
|
|
354
|
+
const result = {
|
|
285
355
|
id: row.id,
|
|
286
356
|
content,
|
|
287
357
|
role: row.role,
|
|
288
|
-
type: row.type,
|
|
289
358
|
createdAt: new Date(row.createdAt as string),
|
|
290
359
|
threadId: row.thread_id,
|
|
291
|
-
} as
|
|
360
|
+
} as MastraMessageV2;
|
|
361
|
+
if (row.type && row.type !== `v2`) result.type = row.type;
|
|
362
|
+
return result;
|
|
292
363
|
}
|
|
293
364
|
|
|
294
|
-
async getMessages<T extends
|
|
365
|
+
async getMessages<T extends MastraMessageV2[]>({ threadId, selectBy }: StorageGetMessagesArg): Promise<T> {
|
|
295
366
|
try {
|
|
296
|
-
const messages:
|
|
367
|
+
const messages: MastraMessageV2[] = [];
|
|
297
368
|
const limit = typeof selectBy?.last === `number` ? selectBy.last : 40;
|
|
298
369
|
|
|
299
370
|
// If we have specific messages to select
|
|
@@ -373,7 +444,7 @@ export class LibSQLStore extends MastraStorage {
|
|
|
373
444
|
}
|
|
374
445
|
}
|
|
375
446
|
|
|
376
|
-
async saveMessages({ messages }: { messages:
|
|
447
|
+
async saveMessages({ messages }: { messages: MastraMessageV2[] }): Promise<MastraMessageV2[]> {
|
|
377
448
|
if (messages.length === 0) return messages;
|
|
378
449
|
|
|
379
450
|
try {
|
|
@@ -393,7 +464,7 @@ export class LibSQLStore extends MastraStorage {
|
|
|
393
464
|
threadId,
|
|
394
465
|
typeof message.content === 'object' ? JSON.stringify(message.content) : message.content,
|
|
395
466
|
message.role,
|
|
396
|
-
message.type,
|
|
467
|
+
message.type || 'v2',
|
|
397
468
|
time instanceof Date ? time.toISOString() : time,
|
|
398
469
|
],
|
|
399
470
|
};
|
|
@@ -701,8 +772,8 @@ export class LibSQLStore extends MastraStorage {
|
|
|
701
772
|
runId: row.run_id as string,
|
|
702
773
|
snapshot: parsedSnapshot,
|
|
703
774
|
resourceId: row.resourceId as string,
|
|
704
|
-
createdAt: new Date(row.
|
|
705
|
-
updatedAt: new Date(row.
|
|
775
|
+
createdAt: new Date(row.createdAt as string),
|
|
776
|
+
updatedAt: new Date(row.updatedAt as string),
|
|
706
777
|
};
|
|
707
778
|
}
|
|
708
779
|
}
|