@mastra/lance 1.4.0 → 1.5.0-alpha.0

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-lance
 description: Documentation for @mastra/lance. Use when working with @mastra/lance APIs, configuration, or implementation.
 metadata:
   package: "@mastra/lance"
-  version: "1.4.0"
+  version: "1.5.0-alpha.0"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.4.0",
+  "version": "1.5.0-alpha.0",
   "package": "@mastra/lance",
   "exports": {},
   "modules": {}

package/dist/docs/references/reference-vectors-lance.md

@@ -49,15 +49,17 @@ const s3Store = await LanceVectorStore.create('s3://bucket/db', {
 
 #### `LanceIndexConfig`
 
-**type** (`'ivfflat' | 'hnsw'`): Index type (Default: `hnsw`)
+**type** (`'ivfflat' | 'ivfpq' | 'hnsw'`): Index type (Default: `hnsw`)
 
-**type.ivfflat** (`ivfflat`): Clusters vectors into lists for approximate search.
+**type.ivfflat** (`ivfflat`): Clusters vectors into partitions and stores full vectors. Searches compute exact distances only within the probed partitions, so closer vectors in other partitions can be missed.
+
+**type.ivfpq** (`ivfpq`): IVF index with product quantization. Uses less memory than ivfflat with lower recall.
 
 **type.hnsw** (`hnsw`): Graph-based index offering fast search times and high recall.
 
 **numPartitions** (`number`): Number of partitions for IVF indexes (Default: `128`)
 
-**numSubVectors** (`number`): Number of sub-vectors for product quantization (Default: `16`)
+**numSubVectors** (`number`): Number of sub-vectors for product quantization. Only applies to ivfpq (Default: `16`)
 
 **hnsw** (`HNSWConfig`): HNSW configuration
 
@@ -137,16 +139,60 @@ interface IndexStats {
   dimension: number
   count: number
   metric: 'cosine' | 'euclidean' | 'dotproduct'
-  type: 'ivfflat' | 'hnsw'
-  config: {
-    m?: number
-    efConstruction?: number
-    numPartitions?: number
-    numSubVectors?: number
-  }
 }
 ```
 
+### `getIndexCoverage()`
+
+**tableName** (`string`): Table to inspect. Defaults to indexName when omitted.
+
+**indexName** (`string`): Used as the table name when tableName is omitted, matching upsert() and query().
+
+Returns one entry per index on the table:
+
+```typescript
+interface LanceIndexCoverage {
+  indexName: string
+  indexType: string
+  columns: string[]
+  numIndexedRows: number
+  numUnindexedRows: number
+}
+```
+
+### `optimize()`
+
+Runs LanceDB table maintenance. It compacts data files, adds rows written since the index was built to the existing index, and removes old table versions. Mastra never calls it automatically. Concurrent calls for the same table in one process share a single run.
+
+**tableName** (`string`): Table to optimize. Defaults to indexName when omitted.
+
+**indexName** (`string`): Used as the table name when tableName is omitted.
+
+**cleanupOlderThan** (`Date`): Remove table versions older than this date. When omitted, LanceDB keeps versions from the last 7 days.
+
+**deleteUnverified** (`boolean`): Also delete unverified files newer than 7 days. Only set this when no other process writes to the table, or the table can be corrupted. (Default: `false`)
+
+Returns LanceDB `OptimizeStats` (compaction and pruning counts).
+
+#### When to optimize
+
+`upsert()` writes rows outside the existing vector index. `query()` still returns those rows by scanning them without the index, so results stay complete, but queries get slower as unindexed rows grow. LanceDB's `fastSearch()` skips unindexed rows entirely.
+
+`optimize()` adds new rows to the existing index without retraining it. If your data distribution has changed a lot, rebuild the index with `deleteIndex()` and `createIndex()` instead.
+
+LanceDB suggests optimizing after roughly 100,000 modified rows or 20 write operations. Use `getIndexCoverage()` to build your own policy:
+
+```typescript
+const [coverage] = await store.getIndexCoverage({ indexName: 'docs' })
+const total = coverage.numIndexedRows + coverage.numUnindexedRows
+
+if (coverage.numUnindexedRows >= 1_000 && coverage.numUnindexedRows / total >= 0.1) {
+  await store.optimize({ indexName: 'docs' })
+}
+```
+
+Managed LanceDB Cloud and Enterprise maintain tables automatically, so you don't need to call `optimize()` there.
+
 ### `deleteIndex()`
 
 **indexName** (`string`): Name of the index to delete

package/dist/index.cjs

@@ -2190,8 +2190,11 @@ var LanceFilterTranslator = class extends _mastra_core_vector_filter.BaseFilterT
 };
 //#endregion
 //#region src/vector/index.ts
+const LANCE_INDEX_FINGERPRINT_KEY = "mastra.index.fingerprint";
+const LANCE_INDEX_UUID_KEY = "mastra.index.uuid";
 var LanceVectorStore = class LanceVectorStore extends _mastra_core_vector.MastraVector {
 	lanceClient;
+	optimizeInFlight = /* @__PURE__ */ new Map();
 	/**
 	* Creates a new instance of LanceVectorStore
 	* @param uri The URI to connect to LanceDB
@@ -2576,6 +2579,7 @@ var LanceVectorStore = class LanceVectorStore extends _mastra_core_vector.Mastra
 				}
 			}, err);
 		}
+		if (indexConfig.type === "ivfflat" && indexConfig.numSubVectors !== void 0) this.logger.warn(`numSubVectors is ignored for 'ivfflat' indexes. Use type 'ivfpq' for product-quantized IVF indexes.`);
 		try {
 			const tables = await this.lanceClient.tableNames();
 			let table;
@@ -2601,19 +2605,51 @@ var LanceVectorStore = class LanceVectorStore extends _mastra_core_vector.Mastra
 				this.logger.warn(`Table ${resolvedTableName} has ${rowCount} rows, which is below the 256 row minimum for index creation. Skipping index creation.`);
 				return;
 			}
-			if (indexConfig.type === "ivfflat") await table.createIndex(columnToIndex, { config: _lancedb_lancedb.Index.ivfPq({
-				numPartitions: indexConfig.numPartitions || 128,
-				numSubVectors: indexConfig.numSubVectors || 16,
+			const numPartitions = indexConfig.numPartitions || 128;
+			const buildParams = indexConfig.type === "ivfflat" ? { numPartitions } : indexConfig.type === "ivfpq" ? {
+				numPartitions,
+				numSubVectors: indexConfig.numSubVectors || 16
+			} : {
+				m: indexConfig?.hnsw?.m || 16,
+				efConstruction: indexConfig?.hnsw?.efConstruction || 100
+			};
+			const fingerprintType = indexConfig.type === "ivfflat" ? "ivf_flat" : indexConfig.type === "ivfpq" ? "ivf_pq" : "hnsw_pq";
+			const fingerprint = JSON.stringify({
+				type: fingerprintType,
+				metric: metricType,
+				...buildParams
+			});
+			const existingIndex = (await table.listIndices()).find((index) => index.columns.includes(columnToIndex));
+			if (existingIndex?.indexUuid) {
+				const field = (await table.schema()).fields.find((f) => f.name === columnToIndex);
+				if (field?.metadata.get(LANCE_INDEX_FINGERPRINT_KEY) === fingerprint && field.metadata.get(LANCE_INDEX_UUID_KEY) === existingIndex.indexUuid) {
+					this.logger.debug(`Index ${existingIndex.name} on ${resolvedTableName} is unchanged. Skipping rebuild.`);
+					return;
+				}
+			}
+			if (indexConfig.type === "ivfflat") await table.createIndex(columnToIndex, { config: _lancedb_lancedb.Index.ivfFlat({
+				numPartitions,
+				distanceType: metricType
+			}) });
+			else if (indexConfig.type === "ivfpq") await table.createIndex(columnToIndex, { config: _lancedb_lancedb.Index.ivfPq({
+				...buildParams,
 				distanceType: metricType
 			}) });
 			else {
 				this.logger.debug("Creating HNSW PQ index with config:", indexConfig);
 				await table.createIndex(columnToIndex, { config: _lancedb_lancedb.Index.hnswPq({
-					m: indexConfig?.hnsw?.m || 16,
-					efConstruction: indexConfig?.hnsw?.efConstruction || 100,
+					...buildParams,
 					distanceType: metricType
 				}) });
 			}
+			const builtUuid = (await table.listIndices()).find((index) => index.columns.includes(columnToIndex))?.indexUuid;
+			if (builtUuid && builtUuid !== existingIndex?.indexUuid) await table.updateFieldMetadata([{
+				path: columnToIndex,
+				metadata: {
+					[LANCE_INDEX_FINGERPRINT_KEY]: fingerprint,
+					[LANCE_INDEX_UUID_KEY]: builtUuid
+				}
+			}]);
 		} catch (error) {
 			throw new _mastra_core_error.MastraError({
 				id: (0, _mastra_core_storage.createVectorErrorId)("LANCE", "CREATE_INDEX", "FAILED"),
@@ -2722,6 +2758,118 @@ var LanceVectorStore = class LanceVectorStore extends _mastra_core_vector.Mastra
 			}, error);
 		}
 	}
+	async openExistingTable({ tableName, indexName }, operation) {
+		const resolvedTableName = tableName ?? indexName;
+		let validationError;
+		if (!this.lanceClient) validationError = "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance";
+		else if (!resolvedTableName) validationError = "tableName or indexName is required";
+		if (validationError || !resolvedTableName || !this.lanceClient) throw new _mastra_core_error.MastraError({
+			id: (0, _mastra_core_storage.createVectorErrorId)("LANCE", operation, "INVALID_ARGS"),
+			text: validationError,
+			domain: _mastra_core_error.ErrorDomain.STORAGE,
+			category: _mastra_core_error.ErrorCategory.USER,
+			details: { tableName: resolvedTableName ?? "" }
+		});
+		let tableExists;
+		try {
+			tableExists = (await this.lanceClient.tableNames()).includes(resolvedTableName);
+			if (tableExists) return {
+				table: await this.lanceClient.openTable(resolvedTableName),
+				resolvedTableName
+			};
+		} catch (error) {
+			throw new _mastra_core_error.MastraError({
+				id: (0, _mastra_core_storage.createVectorErrorId)("LANCE", operation, "FAILED"),
+				domain: _mastra_core_error.ErrorDomain.STORAGE,
+				category: _mastra_core_error.ErrorCategory.THIRD_PARTY,
+				details: { tableName: resolvedTableName }
+			}, error);
+		}
+		throw new _mastra_core_error.MastraError({
+			id: (0, _mastra_core_storage.createVectorErrorId)("LANCE", operation, "INVALID_ARGS"),
+			text: `Table ${resolvedTableName} does not exist`,
+			domain: _mastra_core_error.ErrorDomain.STORAGE,
+			category: _mastra_core_error.ErrorCategory.USER,
+			details: { tableName: resolvedTableName }
+		});
+	}
+	/**
+	* Runs LanceDB table maintenance: compacts fragments, adds unindexed rows to existing indexes
+	* (without retraining them), and prunes old table versions.
+	*
+	* Never runs implicitly. Concurrent calls for the same table share a single in-flight run.
+	*/
+	async optimize(params) {
+		const key = params.tableName ?? params.indexName;
+		const inFlight = key ? this.optimizeInFlight.get(key) : void 0;
+		if (inFlight) return inFlight;
+		const run = (async () => {
+			try {
+				const { table, resolvedTableName } = await this.openExistingTable(params, "OPTIMIZE");
+				try {
+					const schema = await table.schema();
+					const tracked = (await table.listIndices()).filter((index) => index.columns.some((column) => schema.fields.find((f) => f.name === column)?.metadata.get(LANCE_INDEX_UUID_KEY) === index.indexUuid));
+					const stats = await table.optimize({
+						...params.cleanupOlderThan ? { cleanupOlderThan: params.cleanupOlderThan } : {},
+						deleteUnverified: params.deleteUnverified ?? false
+					});
+					if (tracked.length) {
+						const after = await table.listIndices();
+						const updates = tracked.flatMap((before) => {
+							const current = after.find((index) => index.name === before.name);
+							return current?.indexUuid && current.indexUuid !== before.indexUuid ? before.columns.map((path) => ({
+								path,
+								metadata: { [LANCE_INDEX_UUID_KEY]: current.indexUuid }
+							})) : [];
+						});
+						if (updates.length) await table.updateFieldMetadata(updates);
+					}
+					return stats;
+				} catch (error) {
+					throw new _mastra_core_error.MastraError({
+						id: (0, _mastra_core_storage.createVectorErrorId)("LANCE", "OPTIMIZE", "FAILED"),
+						domain: _mastra_core_error.ErrorDomain.STORAGE,
+						category: _mastra_core_error.ErrorCategory.THIRD_PARTY,
+						details: { tableName: resolvedTableName }
+					}, error);
+				}
+			} finally {
+				if (key) this.optimizeInFlight.delete(key);
+			}
+		})();
+		if (key) this.optimizeInFlight.set(key, run);
+		return run;
+	}
+	/**
+	* Reports how many rows each index on a table covers. Rows written after an index was built
+	* stay unindexed until `optimize()` runs; use this to decide when to optimize.
+	*/
+	async getIndexCoverage(params) {
+		const { table, resolvedTableName } = await this.openExistingTable(params, "GET_INDEX_COVERAGE");
+		try {
+			const indices = await table.listIndices();
+			const coverage = [];
+			for (const index of indices) {
+				const stats = await table.indexStats(index.name);
+				if (!stats) continue;
+				coverage.push({
+					indexName: index.name,
+					indexType: stats.indexType,
+					columns: index.columns,
+					numIndexedRows: stats.numIndexedRows,
+					numUnindexedRows: stats.numUnindexedRows
+				});
+			}
+			return coverage;
+		} catch (error) {
+			throw new _mastra_core_error.MastraError({
+				id: (0, _mastra_core_storage.createVectorErrorId)("LANCE", "GET_INDEX_COVERAGE", "FAILED"),
+				domain: _mastra_core_error.ErrorDomain.STORAGE,
+				category: _mastra_core_error.ErrorCategory.THIRD_PARTY,
+				details: { tableName: resolvedTableName }
+			}, error);
+		}
+	}
 	/**
 	* Deletes all tables in the database
 	*/