@mastra/lance 1.0.3 → 1.0.4

package/dist/index.js

@@ -129,10 +129,52 @@ function resolveLanceConfig(config) {
 }
 var LanceDB = class extends MastraBase {
   client;
+  /** Cache of actual table columns: tableName -> Set<columnName> */
+  /** Cache of actual table columns: tableName -> Promise<Set<columnName>> (stores in-flight promise to coalesce concurrent calls) */
+  tableColumnsCache = /* @__PURE__ */ new Map();
   constructor({ client }) {
     super({ name: "lance-db" });
     this.client = client;
   }
+  /**
+   * Gets the set of column names that actually exist in the database table.
+   * Results are cached; the cache is invalidated when alterTable() adds new columns.
+   */
+  async getTableColumns(tableName) {
+    const cached = this.tableColumnsCache.get(tableName);
+    if (cached) return cached;
+    const promise = (async () => {
+      try {
+        const table = await this.client.openTable(tableName);
+        const schema = await table.schema();
+        const columns = new Set(schema.fields.map((f) => f.name));
+        if (columns.size === 0) {
+          this.tableColumnsCache.delete(tableName);
+        }
+        return columns;
+      } catch {
+        this.tableColumnsCache.delete(tableName);
+        return /* @__PURE__ */ new Set();
+      }
+    })();
+    this.tableColumnsCache.set(tableName, promise);
+    return promise;
+  }
+  /**
+   * Filters a record to only include columns that exist in the actual database table.
+   * Unknown columns are silently dropped to ensure forward compatibility.
+   */
+  async filterRecordToKnownColumns(tableName, record) {
+    const knownColumns = await this.getTableColumns(tableName);
+    if (knownColumns.size === 0) return record;
+    const filtered = {};
+    for (const [key, value] of Object.entries(record)) {
+      if (knownColumns.has(key)) {
+        filtered[key] = value;
+      }
+    }
+    return filtered;
+  }
   getDefaultValue(type) {
     switch (type) {
       case "text":
@@ -232,6 +274,8 @@ var LanceDB = class extends MastraBase {
         },
         error
       );
+    } finally {
+      this.tableColumnsCache.delete(tableName);
     }
   }
   async dropTable({ tableName }) {
@@ -270,6 +314,8 @@ var LanceDB = class extends MastraBase {
         },
         error
       );
+    } finally {
+      this.tableColumnsCache.delete(tableName);
     }
   }
   async alterTable({
@@ -336,6 +382,8 @@ var LanceDB = class extends MastraBase {
         },
         error
       );
+    } finally {
+      this.tableColumnsCache.delete(tableName);
     }
   }
   async clearTable({ tableName }) {
@@ -406,7 +454,9 @@ var LanceDB = class extends MastraBase {
           processedRecord[key] = JSON.stringify(processedRecord[key]);
         }
       }
-      await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
+      const filteredRecord = await this.filterRecordToKnownColumns(tableName, processedRecord);
+      if (Object.keys(filteredRecord).length === 0) return;
+      await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([filteredRecord]);
     } catch (error) {
       throw new MastraError(
         {
@@ -455,7 +505,12 @@ var LanceDB = class extends MastraBase {
         }
         return processedRecord;
       });
-      await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
+      const filteredRecords = await Promise.all(
+        processedRecords.map((r) => this.filterRecordToKnownColumns(tableName, r))
+      );
+      const nonEmptyRecords = filteredRecords.filter((r) => Object.keys(r).length > 0);
+      if (nonEmptyRecords.length === 0) return;
+      await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(nonEmptyRecords);
     } catch (error) {
       throw new MastraError(
         {
@@ -806,6 +861,20 @@ var StoreMemoryLance = class extends MemoryStorage {
         conditions.push(`\`createdAt\` ${endOp} ${endTime}`);
       }
       const whereClause = conditions.join(" AND ");
+      if (perPage === 0 && (!include || include.length === 0)) {
+        return { messages: [], total: 0, page, perPage: perPageForResponse, hasMore: false };
+      }
+      if (perPage === 0 && include && include.length > 0) {
+        const includedMessages = await this._getIncludedMessages(table, include);
+        const list2 = new MessageList().add(includedMessages, "memory");
+        return {
+          messages: this._sortMessages(list2.get.all.db(), field, direction),
+          total: 0,
+          page,
+          perPage: perPageForResponse,
+          hasMore: false
+        };
+      }
       const total = await table.countRows(whereClause);
       const query = table.query().where(whereClause);
       let allRecords = await query.toArray();
@@ -833,16 +902,7 @@ var StoreMemoryLance = class extends MemoryStorage {
       }
       const messageIds = new Set(messages.map((m) => m.id));
       if (include && include.length > 0) {
-        const threadIds2 = [...new Set(include.map((item) => item.threadId || threadId))];
-        const allThreadMessages = [];
-        for (const tid of threadIds2) {
-          const threadQuery = table.query().where(`thread_id = '${tid}'`);
-          let threadRecords = await threadQuery.toArray();
-          allThreadMessages.push(...threadRecords);
-        }
-        allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
-        const contextMessages = this.processMessagesWithContext(allThreadMessages, include);
-        const includedMessages = contextMessages.map((row) => this.normalizeMessage(row));
+        const includedMessages = await this._getIncludedMessages(table, include);
         for (const includeMsg of includedMessages) {
           if (!messageIds.has(includeMsg.id)) {
             messages.push(includeMsg);
@@ -852,17 +912,7 @@ var StoreMemoryLance = class extends MemoryStorage {
       }
       const list = new MessageList().add(messages, "memory");
       let finalMessages = list.get.all.db();
-      finalMessages = finalMessages.sort((a, b) => {
-        const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
-        const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
-        if (aValue == null && bValue == null) return 0;
-        if (aValue == null) return direction === "ASC" ? -1 : 1;
-        if (bValue == null) return direction === "ASC" ? 1 : -1;
-        if (typeof aValue === "string" && typeof bValue === "string") {
-          return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
-        }
-        return direction === "ASC" ? aValue - bValue : bValue - aValue;
-      });
+      finalMessages = this._sortMessages(finalMessages, field, direction);
       const returnedThreadMessageIds = new Set(finalMessages.filter((m) => m.threadId === threadId).map((m) => m.id));
       const allThreadMessagesReturned = returnedThreadMessageIds.size >= total;
       const fetchedAll = perPageInput === false || allThreadMessagesReturned;
@@ -1041,6 +1091,62 @@ var StoreMemoryLance = class extends MemoryStorage {
       );
     }
   }
+  _sortMessages(messages, field, direction) {
+    return messages.sort((a, b) => {
+      const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
+      const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
+      if (aValue == null && bValue == null) return 0;
+      if (aValue == null) return direction === "ASC" ? -1 : 1;
+      if (bValue == null) return direction === "ASC" ? 1 : -1;
+      if (typeof aValue === "string" && typeof bValue === "string") {
+        return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
+      }
+      return direction === "ASC" ? aValue - bValue : bValue - aValue;
+    });
+  }
+  async _getIncludedMessages(table, include) {
+    if (include.length === 0) return [];
+    const targetIds = include.map((item) => item.id);
+    const idCondition = targetIds.length === 1 ? `id = '${this.escapeSql(targetIds[0])}'` : `id IN (${targetIds.map((id) => `'${this.escapeSql(id)}'`).join(", ")})`;
+    const targetRecords = await table.query().where(idCondition).toArray();
+    const needsContext = include.some((item) => item.withPreviousMessages || item.withNextMessages);
+    if (!needsContext) {
+      return targetRecords.map((row) => this.normalizeMessage(row));
+    }
+    const threadIdsToFetch = [...new Set(targetRecords.map((r) => r.thread_id))];
+    const threadCache = /* @__PURE__ */ new Map();
+    for (const tid of threadIdsToFetch) {
+      const threadRecords = await table.query().where(`thread_id = '${this.escapeSql(tid)}'`).toArray();
+      threadRecords.sort((a, b) => a.createdAt - b.createdAt);
+      threadCache.set(tid, threadRecords);
+    }
+    const targetThreadMap = /* @__PURE__ */ new Map();
+    for (const r of targetRecords) {
+      targetThreadMap.set(r.id, r.thread_id);
+    }
+    const includeByThread = /* @__PURE__ */ new Map();
+    for (const item of include) {
+      const tid = targetThreadMap.get(item.id);
+      if (!tid) continue;
+      const items = includeByThread.get(tid) ?? [];
+      items.push(item);
+      includeByThread.set(tid, items);
+    }
+    const seen = /* @__PURE__ */ new Set();
+    const allContextMessages = [];
+    for (const [tid, threadInclude] of includeByThread) {
+      const threadMessages = threadCache.get(tid) ?? [];
+      const contextMessages = this.processMessagesWithContext(threadMessages, threadInclude);
+      for (const msg of contextMessages) {
+        if (!seen.has(msg.id)) {
+          seen.add(msg.id);
+          allContextMessages.push(msg);
+        }
+      }
+    }
+    allContextMessages.sort((a, b) => a.createdAt - b.createdAt);
+    return allContextMessages.map((row) => this.normalizeMessage(row));
+  }
   /**
    * Processes messages to include context messages based on withPreviousMessages and withNextMessages
    * @param records - The sorted array of records to process
@@ -1050,7 +1156,8 @@ var StoreMemoryLance = class extends MemoryStorage {
   processMessagesWithContext(records, include) {
     const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
     if (messagesWithContext.length === 0) {
-      return records;
+      const targetIds = new Set(include.map((item) => item.id));
+      return records.filter((record) => targetIds.has(record.id));
     }
     const messageIndexMap = /* @__PURE__ */ new Map();
     records.forEach((message, index) => {
@@ -1075,7 +1182,8 @@ var StoreMemoryLance = class extends MemoryStorage {
       }
     }
     if (additionalIndices.size === 0) {
-      return records;
+      const targetIds = new Set(include.map((item) => item.id));
+      return records.filter((record) => targetIds.has(record.id));
     }
     const originalMatchIds = new Set(include.map((item) => item.id));
     const allIndices = /* @__PURE__ */ new Set();