@mastra/lance 1.4.0 → 1.5.0-alpha.0

package/dist/index.js

@@ -2189,8 +2189,11 @@ var LanceFilterTranslator = class extends BaseFilterTranslator {
 };
 //#endregion
 //#region src/vector/index.ts
+const LANCE_INDEX_FINGERPRINT_KEY = "mastra.index.fingerprint";
+const LANCE_INDEX_UUID_KEY = "mastra.index.uuid";
 var LanceVectorStore = class LanceVectorStore extends MastraVector {
 	lanceClient;
+	optimizeInFlight = /* @__PURE__ */ new Map();
 	/**
 	* Creates a new instance of LanceVectorStore
 	* @param uri The URI to connect to LanceDB
@@ -2575,6 +2578,7 @@ var LanceVectorStore = class LanceVectorStore extends MastraVector {
 				}
 			}, err);
 		}
+		if (indexConfig.type === "ivfflat" && indexConfig.numSubVectors !== void 0) this.logger.warn(`numSubVectors is ignored for 'ivfflat' indexes. Use type 'ivfpq' for product-quantized IVF indexes.`);
 		try {
 			const tables = await this.lanceClient.tableNames();
 			let table;
@@ -2600,19 +2604,51 @@ var LanceVectorStore = class LanceVectorStore extends MastraVector {
 				this.logger.warn(`Table ${resolvedTableName} has ${rowCount} rows, which is below the 256 row minimum for index creation. Skipping index creation.`);
 				return;
 			}
-			if (indexConfig.type === "ivfflat") await table.createIndex(columnToIndex, { config: Index.ivfPq({
-				numPartitions: indexConfig.numPartitions || 128,
-				numSubVectors: indexConfig.numSubVectors || 16,
+			const numPartitions = indexConfig.numPartitions || 128;
+			const buildParams = indexConfig.type === "ivfflat" ? { numPartitions } : indexConfig.type === "ivfpq" ? {
+				numPartitions,
+				numSubVectors: indexConfig.numSubVectors || 16
+			} : {
+				m: indexConfig?.hnsw?.m || 16,
+				efConstruction: indexConfig?.hnsw?.efConstruction || 100
+			};
+			const fingerprintType = indexConfig.type === "ivfflat" ? "ivf_flat" : indexConfig.type === "ivfpq" ? "ivf_pq" : "hnsw_pq";
+			const fingerprint = JSON.stringify({
+				type: fingerprintType,
+				metric: metricType,
+				...buildParams
+			});
+			const existingIndex = (await table.listIndices()).find((index) => index.columns.includes(columnToIndex));
+			if (existingIndex?.indexUuid) {
+				const field = (await table.schema()).fields.find((f) => f.name === columnToIndex);
+				if (field?.metadata.get(LANCE_INDEX_FINGERPRINT_KEY) === fingerprint && field.metadata.get(LANCE_INDEX_UUID_KEY) === existingIndex.indexUuid) {
+					this.logger.debug(`Index ${existingIndex.name} on ${resolvedTableName} is unchanged. Skipping rebuild.`);
+					return;
+				}
+			}
+			if (indexConfig.type === "ivfflat") await table.createIndex(columnToIndex, { config: Index.ivfFlat({
+				numPartitions,
+				distanceType: metricType
+			}) });
+			else if (indexConfig.type === "ivfpq") await table.createIndex(columnToIndex, { config: Index.ivfPq({
+				...buildParams,
 				distanceType: metricType
 			}) });
 			else {
 				this.logger.debug("Creating HNSW PQ index with config:", indexConfig);
 				await table.createIndex(columnToIndex, { config: Index.hnswPq({
-					m: indexConfig?.hnsw?.m || 16,
-					efConstruction: indexConfig?.hnsw?.efConstruction || 100,
+					...buildParams,
 					distanceType: metricType
 				}) });
 			}
+			const builtUuid = (await table.listIndices()).find((index) => index.columns.includes(columnToIndex))?.indexUuid;
+			if (builtUuid && builtUuid !== existingIndex?.indexUuid) await table.updateFieldMetadata([{
+				path: columnToIndex,
+				metadata: {
+					[LANCE_INDEX_FINGERPRINT_KEY]: fingerprint,
+					[LANCE_INDEX_UUID_KEY]: builtUuid
+				}
+			}]);
 		} catch (error) {
 			throw new MastraError({
 				id: createVectorErrorId("LANCE", "CREATE_INDEX", "FAILED"),
@@ -2721,6 +2757,118 @@ var LanceVectorStore = class LanceVectorStore extends MastraVector {
 			}, error);
 		}
 	}
+	async openExistingTable({ tableName, indexName }, operation) {
+		const resolvedTableName = tableName ?? indexName;
+		let validationError;
+		if (!this.lanceClient) validationError = "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance";
+		else if (!resolvedTableName) validationError = "tableName or indexName is required";
+		if (validationError || !resolvedTableName || !this.lanceClient) throw new MastraError({
+			id: createVectorErrorId("LANCE", operation, "INVALID_ARGS"),
+			text: validationError,
+			domain: ErrorDomain.STORAGE,
+			category: ErrorCategory.USER,
+			details: { tableName: resolvedTableName ?? "" }
+		});
+		let tableExists;
+		try {
+			tableExists = (await this.lanceClient.tableNames()).includes(resolvedTableName);
+			if (tableExists) return {
+				table: await this.lanceClient.openTable(resolvedTableName),
+				resolvedTableName
+			};
+		} catch (error) {
+			throw new MastraError({
+				id: createVectorErrorId("LANCE", operation, "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { tableName: resolvedTableName }
+			}, error);
+		}
+		throw new MastraError({
+			id: createVectorErrorId("LANCE", operation, "INVALID_ARGS"),
+			text: `Table ${resolvedTableName} does not exist`,
+			domain: ErrorDomain.STORAGE,
+			category: ErrorCategory.USER,
+			details: { tableName: resolvedTableName }
+		});
+	}
+	/**
+	* Runs LanceDB table maintenance: compacts fragments, adds unindexed rows to existing indexes
+	* (without retraining them), and prunes old table versions.
+	*
+	* Never runs implicitly. Concurrent calls for the same table share a single in-flight run.
+	*/
+	async optimize(params) {
+		const key = params.tableName ?? params.indexName;
+		const inFlight = key ? this.optimizeInFlight.get(key) : void 0;
+		if (inFlight) return inFlight;
+		const run = (async () => {
+			try {
+				const { table, resolvedTableName } = await this.openExistingTable(params, "OPTIMIZE");
+				try {
+					const schema = await table.schema();
+					const tracked = (await table.listIndices()).filter((index) => index.columns.some((column) => schema.fields.find((f) => f.name === column)?.metadata.get(LANCE_INDEX_UUID_KEY) === index.indexUuid));
+					const stats = await table.optimize({
+						...params.cleanupOlderThan ? { cleanupOlderThan: params.cleanupOlderThan } : {},
+						deleteUnverified: params.deleteUnverified ?? false
+					});
+					if (tracked.length) {
+						const after = await table.listIndices();
+						const updates = tracked.flatMap((before) => {
+							const current = after.find((index) => index.name === before.name);
+							return current?.indexUuid && current.indexUuid !== before.indexUuid ? before.columns.map((path) => ({
+								path,
+								metadata: { [LANCE_INDEX_UUID_KEY]: current.indexUuid }
+							})) : [];
+						});
+						if (updates.length) await table.updateFieldMetadata(updates);
+					}
+					return stats;
+				} catch (error) {
+					throw new MastraError({
+						id: createVectorErrorId("LANCE", "OPTIMIZE", "FAILED"),
+						domain: ErrorDomain.STORAGE,
+						category: ErrorCategory.THIRD_PARTY,
+						details: { tableName: resolvedTableName }
+					}, error);
+				}
+			} finally {
+				if (key) this.optimizeInFlight.delete(key);
+			}
+		})();
+		if (key) this.optimizeInFlight.set(key, run);
+		return run;
+	}
+	/**
+	* Reports how many rows each index on a table covers. Rows written after an index was built
+	* stay unindexed until `optimize()` runs; use this to decide when to optimize.
+	*/
+	async getIndexCoverage(params) {
+		const { table, resolvedTableName } = await this.openExistingTable(params, "GET_INDEX_COVERAGE");
+		try {
+			const indices = await table.listIndices();
+			const coverage = [];
+			for (const index of indices) {
+				const stats = await table.indexStats(index.name);
+				if (!stats) continue;
+				coverage.push({
+					indexName: index.name,
+					indexType: stats.indexType,
+					columns: index.columns,
+					numIndexedRows: stats.numIndexedRows,
+					numUnindexedRows: stats.numUnindexedRows
+				});
+			}
+			return coverage;
+		} catch (error) {
+			throw new MastraError({
+				id: createVectorErrorId("LANCE", "GET_INDEX_COVERAGE", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { tableName: resolvedTableName }
+			}, error);
+		}
+	}
 	/**
 	* Deletes all tables in the database
 	*/