@mastra/lance 1.0.0 → 1.0.1-alpha.0

package/dist/docs/references/reference-vectors-lance.md

@@ -0,0 +1,220 @@
+# Lance Vector Store
+
+The LanceVectorStore class provides vector search using [LanceDB](https://lancedb.github.io/lancedb/), an embedded vector database built on the Lance columnar format. It offers efficient storage and fast similarity search for both local development and production deployments.
+
+## Factory Method
+
+The LanceVectorStore uses a factory pattern for creation. You should use the static `create()` method rather than the constructor directly.
+
+**uri:** (`string`): Path to LanceDB database or URI for cloud deployments
+
+**options?:** (`ConnectionOptions`): Additional connection options for LanceDB
+
+## Constructor Examples
+
+You can create a `LanceVectorStore` instance using the static create method:
+
+```ts
+import { LanceVectorStore } from "@mastra/lance";
+
+// Connect to a local database
+const vectorStore = await LanceVectorStore.create("/path/to/db");
+
+// Connect to a LanceDB cloud database
+const cloudStore = await LanceVectorStore.create("db://host:port");
+
+// Connect to a cloud database with options
+const s3Store = await LanceVectorStore.create("s3://bucket/db", {
+  storageOptions: { timeout: "60s" },
+});
+```
+
+## Methods
+
+### createIndex()
+
+**tableName:** (`string`): Name of the table to create index in
+
+**indexName:** (`string`): Name of the index (column name) to create
+
+**dimension:** (`number`): Vector dimension (must match your embedding model)
+
+**metric?:** (`'cosine' | 'euclidean' | 'dotproduct'`): Distance metric for similarity search (Default: `cosine`)
+
+**indexConfig?:** (`LanceIndexConfig`): Index configuration (Default: `{ type: 'hnsw' }`)
+
+#### LanceIndexConfig
+
+**type:** (`'ivfflat' | 'hnsw'`): stringivfflat:ivfflatClusters vectors into lists for approximate search.hnsw:hnswGraph-based index offering fast search times and high recall. (Default: `hnsw`)
+
+**numPartitions?:** (`number`): Number of partitions for IVF indexes (Default: `128`)
+
+**numSubVectors?:** (`number`): Number of sub-vectors for product quantization (Default: `16`)
+
+**hnsw?:** (`HNSWConfig`): objectm?:numberMaximum number of connections per node (default: 16)efConstruction?:numberBuild-time complexity (default: 100)
+
+### createTable()
+
+**tableName:** (`string`): Name of the table to create
+
+**data:** (`Record<string, unknown>[] | TableLike`): Initial data for the table
+
+**options?:** (`Partial<CreateTableOptions>`): Additional table creation options
+
+### upsert()
+
+**tableName:** (`string`): Name of the table to upsert vectors into
+
+**vectors:** (`number[][]`): Array of embedding vectors
+
+**metadata?:** (`Record<string, any>[]`): Metadata for each vector
+
+**ids?:** (`string[]`): Optional vector IDs (auto-generated if not provided)
+
+### query()
+
+**tableName:** (`string`): Name of the table to query
+
+**queryVector:** (`number[]`): Query vector
+
+**topK?:** (`number`): Number of results to return (Default: `10`)
+
+**filter?:** (`Record<string, any>`): Metadata filters
+
+**includeVector?:** (`boolean`): Whether to include the vector in the result (Default: `false`)
+
+**columns?:** (`string[]`): Specific columns to include in the result (Default: `[]`)
+
+**includeAllColumns?:** (`boolean`): Whether to include all columns in the result (Default: `false`)
+
+### listTables()
+
+Returns an array of table names as strings.
+
+```typescript
+const tables = await vectorStore.listTables();
+// ['my_vectors', 'embeddings', 'documents']
+```
+
+### getTableSchema()
+
+**tableName:** (`string`): Name of the table to describe
+
+Returns the schema of the specified table.
+
+### deleteTable()
+
+**tableName:** (`string`): Name of the table to delete
+
+### deleteAllTables()
+
+Deletes all tables in the database.
+
+### listIndexes()
+
+Returns an array of index names as strings.
+
+### describeIndex()
+
+**indexName:** (`string`): Name of the index to describe
+
+Returns information about the index:
+
+```typescript
+interface IndexStats {
+  dimension: number;
+  count: number;
+  metric: "cosine" | "euclidean" | "dotproduct";
+  type: "ivfflat" | "hnsw";
+  config: {
+    m?: number;
+    efConstruction?: number;
+    numPartitions?: number;
+    numSubVectors?: number;
+  };
+}
+```
+
+### deleteIndex()
+
+**indexName:** (`string`): Name of the index to delete
+
+### updateVector()
+
+Update a single vector by ID or by metadata filter. Either `id` or `filter` must be provided, but not both.
+
+**indexName:** (`string`): Name of the index containing the vector
+
+**id?:** (`string`): ID of the vector to update (mutually exclusive with filter)
+
+**filter?:** (`Record<string, any>`): Metadata filter to identify vector(s) to update (mutually exclusive with id)
+
+**update:** (`{ vector?: number[]; metadata?: Record<string, any>; }`): Object containing the vector and/or metadata to update
+
+### deleteVector()
+
+**indexName:** (`string`): Name of the index containing the vector
+
+**id:** (`string`): ID of the vector to delete
+
+### deleteVectors()
+
+Delete multiple vectors by IDs or by metadata filter. Either `ids` or `filter` must be provided, but not both.
+
+**indexName:** (`string`): Name of the index containing the vectors to delete
+
+**ids?:** (`string[]`): Array of vector IDs to delete (mutually exclusive with filter)
+
+**filter?:** (`Record<string, any>`): Metadata filter to identify vectors to delete (mutually exclusive with ids)
+
+### close()
+
+Closes the database connection.
+
+## Response Types
+
+Query results are returned in this format:
+
+```typescript
+interface QueryResult {
+  id: string;
+  score: number;
+  metadata: Record<string, any>;
+  vector?: number[]; // Only included if includeVector is true
+  document?: string; // Document text if available
+}
+```
+
+## Error Handling
+
+The store throws typed errors that can be caught:
+
+```typescript
+try {
+  await store.query({
+    tableName: "my_vectors",
+    queryVector: queryVector,
+  });
+} catch (error) {
+  if (error instanceof Error) {
+    console.log(error.message);
+  }
+}
+```
+
+## Best Practices
+
+- Use the appropriate index type for your use case:
+
+  - HNSW for better recall and performance when memory isn't constrained
+  - IVF for better memory efficiency with large datasets
+
+- For optimal performance with large datasets, consider adjusting `numPartitions` and `numSubVectors` values
+
+- Use `close()` method to properly close connections when done with the database
+
+- Store metadata with a consistent schema to simplify filtering operations
+
+## Related
+
+- [Metadata Filters](https://mastra.ai/reference/rag/metadata-filters)

package/dist/index.cjs

@@ -2408,6 +2408,16 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
       if (filter && Object.keys(filter).length > 0) {
         const whereClause = this.filterTranslator(filter);
         this.logger.debug(`Where clause generated: ${whereClause}`);
+        const schema = await table.schema();
+        const schemaColumns = new Set(schema.fields.map((f) => f.name));
+        const filterColumns = this.extractFilterColumns(whereClause);
+        const missingColumns = filterColumns.filter((col) => !schemaColumns.has(col));
+        if (missingColumns.length > 0) {
+          this.logger.debug(
+            `Filter references columns not in schema: ${missingColumns.join(", ")}. Returning empty results.`
+          );
+          return [];
+        }
         query = query.where(whereClause);
       }
       if (!includeAllColumns && columns.length > 0) {
@@ -2420,16 +2430,16 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
       query = query.limit(topK);
       const results = await query.toArray();
       return results.map((result) => {
-        const flatMetadata = {};
-        Object.keys(result).forEach((key) => {
-          if (key !== "id" && key !== "score" && key !== "vector" && key !== "_distance") {
-            if (key.startsWith("metadata_")) {
-              const metadataKey = key.substring("metadata_".length);
-              flatMetadata[metadataKey] = result[key];
-            }
+        let metadata = {};
+        if (result._metadata_json) {
+          try {
+            metadata = JSON.parse(result._metadata_json);
+          } catch {
+            metadata = this.extractFlatMetadata(result);
           }
-        });
-        const metadata = this.unflattenObject(flatMetadata);
+        } else {
+          metadata = this.extractFlatMetadata(result);
+        }
         return {
           id: String(result.id || ""),
           metadata,
@@ -2525,6 +2535,9 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
           Object.entries(flattenedMetadata).forEach(([key, value]) => {
             rowData[key] = value;
           });
+          rowData["_metadata_json"] = JSON.stringify(metadataItem);
+        } else {
+          rowData["_metadata_json"] = "";
         }
         return rowData;
       });
@@ -2723,7 +2736,9 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
           `Table ${resolvedTableName} does not exist. Creating empty table with dimension ${dimension}.`
         );
         const initVector = new Array(dimension).fill(0);
-        table = await this.lanceClient.createTable(resolvedTableName, [{ id: "__init__", vector: initVector }]);
+        table = await this.lanceClient.createTable(resolvedTableName, [
+          { id: "__init__", vector: initVector, _metadata_json: "" }
+        ]);
         try {
           await table.delete("id = '__init__'");
         } catch (deleteError) {
@@ -3071,6 +3086,17 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
                 Object.entries(update.metadata).forEach(([key, value]) => {
                   rowData[`metadata_${key}`] = value;
                 });
+                const hasMetadataJson = schema.fields.some((f) => f.name === "_metadata_json");
+                if (hasMetadataJson) {
+                  let existingMetadata = {};
+                  if (record._metadata_json) {
+                    try {
+                      existingMetadata = JSON.parse(record._metadata_json);
+                    } catch {
+                    }
+                  }
+                  rowData["_metadata_json"] = JSON.stringify({ ...existingMetadata, ...update.metadata });
+                }
               }
               return rowData;
             });
@@ -3162,29 +3188,27 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
     }
   }
   /**
-   * Converts a flattened object with keys using underscore notation back to a nested object.
-   * Example: { name: 'test', details_text: 'test' } → { name: 'test', details: { text: 'test' } }
+   * Extracts column names referenced in a SQL WHERE clause.
+   * Identifies metadata_* prefixed identifiers used in filter conditions.
    */
-  unflattenObject(obj) {
-    const result = {};
-    Object.keys(obj).forEach((key) => {
-      const value = obj[key];
-      const parts = key.split("_");
-      let current = result;
-      for (let i = 0; i < parts.length - 1; i++) {
-        const part = parts[i];
-        if (!part) continue;
-        if (!current[part] || typeof current[part] !== "object") {
-          current[part] = {};
+  extractFilterColumns(whereClause) {
+    const matches = whereClause.match(/metadata_\w+/g);
+    return matches ? [...new Set(matches)] : [];
+  }
+  /**
+   * Extracts metadata from flattened column names (legacy data without _metadata_json).
+   * Returns keys as-is after stripping the 'metadata_' prefix, without any unflattening.
+   */
+  extractFlatMetadata(result) {
+    const metadata = {};
+    Object.keys(result).forEach((key) => {
+      if (key !== "id" && key !== "score" && key !== "vector" && key !== "_distance" && key !== "_metadata_json") {
+        if (key.startsWith("metadata_")) {
+          metadata[key.substring("metadata_".length)] = result[key];
         }
-        current = current[part];
-      }
-      const lastPart = parts[parts.length - 1];
-      if (lastPart) {
-        current[lastPart] = value;
       }
     });
-    return result;
+    return metadata;
   }
   async deleteVectors({ indexName, filter, ids }) {
     if (ids && filter) {