@mastra/libsql 1.8.2-alpha.0 → 1.9.0-alpha.1

package/dist/index.js

@@ -509,6 +509,9 @@ var LibSQLVector = class extends MastraVector {
   turso;
   maxRetries;
   initialBackoffMs;
+  overFetchMultiplier;
+  isMemoryDb;
+  vectorIndexes;
   constructor({
     url,
     authToken,
@@ -516,6 +519,7 @@ var LibSQLVector = class extends MastraVector {
     syncInterval,
     maxRetries = 5,
     initialBackoffMs = 100,
+    vectorTopKOverFetchMultiplier = 10,
     id
   }) {
     super({ id });
@@ -527,10 +531,27 @@ var LibSQLVector = class extends MastraVector {
     });
     this.maxRetries = maxRetries;
     this.initialBackoffMs = initialBackoffMs;
-    if (url.includes(`file:`) || url.includes(`:memory:`)) {
+    if (!Number.isInteger(vectorTopKOverFetchMultiplier) || vectorTopKOverFetchMultiplier < 1) {
+      throw new Error("vectorTopKOverFetchMultiplier must be a positive integer");
+    }
+    this.overFetchMultiplier = vectorTopKOverFetchMultiplier;
+    this.isMemoryDb = url.includes(":memory:");
+    if (url.includes(`file:`) || this.isMemoryDb) {
       this.turso.execute("PRAGMA journal_mode=WAL;").then(() => this.logger.debug("LibSQLStore: PRAGMA journal_mode=WAL set.")).catch((err) => this.logger.warn("LibSQLStore: Failed to set PRAGMA journal_mode=WAL.", err));
       this.turso.execute("PRAGMA busy_timeout = 5000;").then(() => this.logger.debug("LibSQLStore: PRAGMA busy_timeout=5000 set.")).catch((err) => this.logger.warn("LibSQLStore: Failed to set PRAGMA busy_timeout=5000.", err));
     }
+    this.vectorIndexes = this.isMemoryDb ? Promise.resolve(/* @__PURE__ */ new Set()) : this.discoverVectorIndexes();
+  }
+  async discoverVectorIndexes() {
+    try {
+      const result = await this.turso.execute({
+        sql: `SELECT name FROM sqlite_master WHERE type='index' AND name LIKE '%_vector_idx'`,
+        args: []
+      });
+      return new Set(result.rows.map((row) => row.name));
+    } catch {
+      return /* @__PURE__ */ new Set();
+    }
   }
   async executeWriteOperationWithRetry(operation, isTransaction = false) {
     let attempts = 0;
@@ -564,6 +585,40 @@ var LibSQLVector = class extends MastraVector {
     const translator = new LibSQLFilterTranslator();
     return translator.translate(filter);
   }
+  async hasVectorIndex(parsedIndexName) {
+    const indexes = await this.vectorIndexes;
+    return indexes.has(`${parsedIndexName}_vector_idx`);
+  }
+  async queryWithIndex(parsedIndexName, vectorStr, topK, filter, includeVector, minScore) {
+    const translatedFilter = this.transformFilter(filter);
+    const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter);
+    const hasFilter = filterQuery.length > 0;
+    const fetchCount = hasFilter ? topK * this.overFetchMultiplier : topK * 2;
+    const embeddingSelect = includeVector ? ", vector_extract(t.embedding) as embedding" : "";
+    const filterCondition = hasFilter ? filterQuery.replace(/^\s*WHERE\s+/i, "") : "";
+    const whereClause = hasFilter ? `WHERE ${filterCondition} AND score > ?` : "WHERE score > ?";
+    const query = `
+      WITH candidates AS (
+        SELECT t.vector_id AS id,
+               (1 - vector_distance_cos(t.embedding, vector32(?))) AS score,
+               t.metadata
+               ${embeddingSelect}
+        FROM vector_top_k('${parsedIndexName}_vector_idx', vector32(?), ?) AS v
+        JOIN "${parsedIndexName}" AS t ON t.rowid = v.id
+      )
+      SELECT * FROM candidates
+      ${whereClause}
+      ORDER BY score DESC
+      LIMIT ?`;
+    const args = [vectorStr, vectorStr, fetchCount, ...filterValues, minScore, topK];
+    const result = await this.turso.execute({ sql: query, args });
+    return result.rows.map(({ id, score, metadata, embedding }) => ({
+      id,
+      score,
+      metadata: JSON.parse(metadata ?? "{}"),
+      ...includeVector && embedding && { vector: JSON.parse(embedding) }
+    }));
+  }
   async query({
     indexName,
     queryVector,
@@ -594,6 +649,23 @@ var LibSQLVector = class extends MastraVector {
     try {
       const parsedIndexName = parseSqlIdentifier(indexName, "index name");
       const vectorStr = `[${queryVector.join(",")}]`;
+      if (!this.isMemoryDb && await this.hasVectorIndex(parsedIndexName)) {
+        try {
+          const indexedResults = await this.queryWithIndex(
+            parsedIndexName,
+            vectorStr,
+            topK,
+            filter,
+            includeVector,
+            minScore
+          );
+          if (!filter || indexedResults.length >= topK) {
+            return indexedResults;
+          }
+        } catch (err) {
+          this.logger.warn("LibSQLVector: indexed query failed, falling back to brute-force", err);
+        }
+      }
       const translatedFilter = this.transformFilter(filter);
       const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter);
       filterValues.push(minScore);
@@ -727,6 +799,7 @@ var LibSQLVector = class extends MastraVector {
         `,
       args: []
     });
+    void this.vectorIndexes.then((indexes) => indexes.add(`${parsedIndexName}_vector_idx`));
   }
   deleteIndex(args) {
     try {
@@ -749,6 +822,7 @@ var LibSQLVector = class extends MastraVector {
       sql: `DROP TABLE IF EXISTS ${parsedIndexName}`,
       args: []
     });
+    void this.vectorIndexes.then((indexes) => indexes.delete(`${parsedIndexName}_vector_idx`));
   }
   async listIndexes() {
     try {