@harperfast/harper-pro 5.0.1 → 5.0.3

package/core/resources/RocksIndexStore.ts

@@ -1,27 +1,32 @@
 import {
 	DBI,
-	Store,
-	type StoreContext,
 	type StoreIteratorOptions,
 	type StorePutOptions,
 	type StoreRemoveOptions,
+	RocksDatabase,
 } from '@harperfast/rocksdb-js';
 import { Id } from './ResourceInterface.ts';
 import { MAXIMUM_KEY } from 'ordered-binary';
 
 declare module '@harperfast/rocksdb-js' {
-	// eslint-disable-next-line no-unused-vars
 	interface DBI<T> {
 		getValuesCount(indexedValue: any): number;
 	}
 }
 
-export class RocksIndexStore extends Store {
+/**
+ * A specialized RocksDB-based index store that maintains indexed references to primary keys.
+ * This store uses composite keys consisting of indexed values and primary keys, enabling
+ * efficient range queries over indexed data. The actual data values are stored as null since
+ * this is purely an index structure pointing to primary records elsewhere. This extends
+ * RocksDatabase rather than a store because it actually alters the interface
+ */
+export class RocksIndexStore extends RocksDatabase {
 	/**
 	 * Get all entries matching the range
 	 * @param options
 	 */
-	getRange(context: StoreContext, options: StoreIteratorOptions): Iterable<any> {
+	getRange(options: StoreIteratorOptions): Iterable<any> {
 		let { start, end, exclusiveStart, inclusiveEnd, reverse } = options;
 		if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) {
 			start = [start, MAXIMUM_KEY];
@@ -30,7 +35,7 @@ export class RocksIndexStore extends Store {
 			end = [end, MAXIMUM_KEY];
 		}
 		const translatedOptions = { ...options, start, end };
-		return super.getRange(context, translatedOptions).map(({ key }) => {
+		return super.getRange(translatedOptions).map(({ key }) => {
 			return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] };
 		});
 	}
@@ -41,20 +46,20 @@ export class RocksIndexStore extends Store {
 	 * @param primaryKey
 	 * @param txnId
 	 */
-	put(context: StoreContext, indexedValue: any, primaryKey: Id, options: StorePutOptions) {
-		return super.putSync(context, [indexedValue, primaryKey], null, options);
+	put(indexedValue: any, primaryKey: Id, options: StorePutOptions) {
+		return super.putSync([indexedValue, primaryKey], null, options);
 	}
 
-	putSync(context: StoreContext, indexedValue: any, primaryKey: Id, options: StorePutOptions) {
-		return super.putSync(context, [indexedValue, primaryKey], null, options);
+	putSync(indexedValue: any, primaryKey: Id, options: StorePutOptions) {
+		return super.putSync([indexedValue, primaryKey], null, options);
 	}
 
-	remove(context: StoreContext, indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
-		return super.removeSync(context, [indexedValue, primaryKey], options);
+	remove(indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
+		return super.removeSync([indexedValue, primaryKey], options);
 	}
 
-	removeSync(context: StoreContext, indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
-		super.removeSync(context, [indexedValue, primaryKey], options);
+	removeSync(indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
+		super.removeSync([indexedValue, primaryKey], options);
 	}
 }
 
@@ -63,7 +68,7 @@ export class RocksIndexStore extends Store {
  * classes.
  */
 DBI.prototype.getValuesCount = function getValuesCount(indexedValue: any) {
-	if (this.store instanceof RocksIndexStore) {
+	if (this instanceof RocksIndexStore) {
 		return this.store.getCount(this._context, { start: indexedValue, end: [indexedValue, MAXIMUM_KEY] });
 	}
 	throw new Error('getValuesCount is only supported if dupSort=true');

package/core/resources/Table.ts

@@ -1424,23 +1424,35 @@ export function makeTable(options) {
 		 */
 		static evict(id, existingRecord, existingVersion) {
 			let entry;
-			if (hasSourceGet || audit) {
-				if (!existingRecord) return;
-				entry = primaryStore.getEntry(id);
-				if (!entry || !existingRecord) return;
-				if (entry.version !== existingVersion) return;
-			}
-			if (hasSourceGet) {
-				// if there is a resolution in-progress, abandon the eviction
-				if (primaryStore.hasLock(id, entry.version)) return;
+			let transaction = txnForContext({ transaction: new DatabaseTransaction() }).getReadTxn();
+			let options = { transaction };
+			try {
+				if (hasSourceGet || audit) {
+					if (!existingRecord) return;
+					entry = primaryStore.getEntry(id, options);
+					if (!entry || !existingRecord) return;
+					if (entry.version !== existingVersion) return;
+				}
+				if (hasSourceGet) {
+					// if there is a resolution in-progress, abandon the eviction
+					if (primaryStore.hasLock(id, entry.version)) return;
+				}
+				// evictions never go in the audit log, so we can not record a deletion entry for the eviction
+				// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
+				// removed the entry entirely, but first cleanup indices
+				if (primaryStore.ifVersion) {
+					// lmdb
+					primaryStore.ifVersion?.(id, existingVersion, () => {
+						updateIndices(id, existingRecord, null);
+					});
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
+				} else {
+					updateIndices(id, existingRecord, null, options);
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
+				}
+			} finally {
+				return transaction.commit();
 			}
-			primaryStore.ifVersion?.(id, existingVersion, () => {
-				updateIndices(id, existingRecord, null);
-			});
-			// evictions never go in the audit log, so we can not record a deletion entry for the eviction
-			// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
-			// removed the entry entirely
-			return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
 		}
 		/**
 		 * This is intended to acquire a lock on a record from the whole cluster.
@@ -1951,9 +1963,10 @@ export function makeTable(options) {
 							context.lastModified = existingEntry.version;
 						TableResource._updateResource(this, existingEntry);
 					}
-					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return; // a newer record exists locally
-					updateIndices(id, existingRecord);
-					logger.trace?.(`Deleting record with id: ${id}, txn timestamp: ${new Date(txnTime).toISOString()}`);
+					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
+						return;
+					} // a newer record exists locally
+					updateIndices(id, existingRecord, null, transaction && { transaction });
 					if (audit || trackDeletes) {
 						updateRecord(
 							id,
@@ -2262,7 +2275,9 @@ export function makeTable(options) {
 						return (entryA, entryB) => {
 							const a = getAttributeValue(entryA, order.attribute, context);
 							const b = getAttributeValue(entryB, order.attribute, context);
-							const diff = descending ? compareKeys(b, a) : compareKeys(a, b);
+							const diff = descending
+								? compareKeys(convertToComparableKeys(b), convertToComparableKeys(a))
+								: compareKeys(convertToComparableKeys(a), convertToComparableKeys(b));
 							if (diff === 0) return nextComparator?.(entryA, entryB) || 0;
 							return diff;
 						};
@@ -3509,6 +3524,7 @@ export function makeTable(options) {
 			// determine what index values need to be removed and added
 			let valuesToAdd = getIndexedValues(value, indexNulls) as any[];
 			let valuesToRemove = getIndexedValues(existingValue, indexNulls) as any[];
+			let isLMDB = !!index.prefetch;
 			if (valuesToRemove?.length > 0) {
 				// put this in a conditional so we can do a faster version for new records
 				// determine the changes/diff from new values and old values
@@ -3525,18 +3541,18 @@ export function makeTable(options) {
 						})
 					: [];
 				valuesToRemove = Array.from(setToRemove);
-				if ((valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
+				if (isLMDB && (valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
 					// prefetch any values that have been removed or added
 					const valuesToPrefetch = valuesToRemove.concat(valuesToAdd).map((v) => ({ key: v, value: id }));
-					index.prefetch?.(valuesToPrefetch, noop);
+					index.prefetch(valuesToPrefetch, noop);
 				}
 				//if the update cleared out the attribute value we need to delete it from the index
 				for (let i = 0, l = valuesToRemove.length; i < l; i++) {
 					index.remove(valuesToRemove[i], id, options);
 				}
-			} else if (valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
+			} else if (isLMDB && valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
 				// no old values, just new
-				index.prefetch?.(
+				index.prefetch(
 					valuesToAdd.map((v) => ({ key: v, value: id })),
 					noop
 				);
@@ -4122,7 +4138,7 @@ export function makeTable(options) {
 								// don't do anything if the version has changed
 								return;
 							}
-							updateIndices(id, existingRecord, updatedRecord);
+							updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
 							if (updatedRecord) {
 								if (existingEntry) {
 									context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
@@ -4572,3 +4588,12 @@ function hasOtherProcesses(store) {
 			return +line.match(/\d+/)?.[0] != pid;
 		});
 }
+function convertToComparableKeys(a) {
+	if (a instanceof Date) {
+		return a.getTime();
+	}
+	if (Array.isArray(a)) {
+		return a.map(convertToComparableKeys);
+	}
+	return a;
+}

package/core/resources/analytics/write.ts

@@ -219,15 +219,12 @@ export async function recordHostname() {
 	const nodeId = stableNodeId(hostname);
 	log.trace?.('recordHostname nodeId:', nodeId);
 	const hostnamesTable = getAnalyticsHostnameTable();
-	const record = await hostnamesTable.get(nodeId);
-	if (!record) {
-		const hostnameRecord = {
-			id: nodeId,
-			hostname,
-		};
-		log.trace?.(`recordHostname storing hostname: ${JSON.stringify(hostnameRecord)}`);
-		await hostnamesTable.put(hostnameRecord.id, hostnameRecord);
-	}
+	const hostnameRecord = {
+		id: nodeId,
+		hostname,
+	};
+	log.trace?.(`recordHostname storing hostname: ${JSON.stringify(hostnameRecord)}`);
+	await hostnamesTable.put(hostnameRecord.id, hostnameRecord);
 }
 
 export interface Metric {
@@ -514,7 +511,7 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 		await rest();
 	}
 	for (const entry of threadsToAverage) {
-		// eslint-disable-next-line @typescript-eslint/no-unused-vars,prefer-const
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
 		let { path, method, type, metric, count, total, distribution, threads, ...measures } = entry;
 		threads = threads.filter((thread) => thread);
 		for (const measureName in measures) {

package/core/resources/databases.ts

@@ -120,9 +120,13 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo
 	}
 	let db: RocksRootDatabase;
 	if (options.dupSort) {
-		db = RocksDatabase.open(new RocksIndexStore(path, options)) as RocksDatabaseEx;
+		db = new RocksIndexStore(path, options).open() as RocksDatabaseEx;
 	} else {
 		db = RocksDatabase.open(path, options) as RocksDatabaseEx;
+		// the RocksDB put and remove return promises, which masks thrown errors in non-awaiting calls to put/remove,
+		// making them unsafe to replace LMDB methods, which will synchronously throw errors if there is a problem
+		db.put = db.putSync;
+		db.remove = db.removeSync;
 		db.encoder.name = options.name;
 	}
 	db.env = {};
@@ -386,18 +390,18 @@ function initStores(
 ) {
 	const envInit = new OpenEnvironmentObject(path, false);
 	const internalDbiInit = createOpenDBIObject(false);
-	let dbisStore = rootStore.dbisDb;
-	if (!dbisStore) {
+	let attributesDbi = rootStore.dbisDb;
+	if (!attributesDbi) {
 		if (rootStore instanceof RocksDatabase) {
-			dbisStore = openRocksDatabase(rootStore.path, {
+			attributesDbi = openRocksDatabase(rootStore.path, {
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
 			}) as RocksDatabaseEx;
 		} else {
-			dbisStore = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
+			attributesDbi = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
 		}
-		rootStore.dbisDb = dbisStore;
+		rootStore.dbisDb = attributesDbi;
 	}
 
 	let auditStore = rootStore.auditStore;
@@ -428,7 +432,7 @@ function initStores(
 	definedTables.rootStore = rootStore;
 	const tablesToLoad = new Map<string, any>();
 
-	for (const result of dbisStore.getRange({ start: false })) {
+	for (const result of attributesDbi.getRange({ start: false })) {
 		const { key, value } = result as { key: string; value: any };
 		let [tableName, attribute_name] = key.toString().split('/');
 		if (attribute_name === '') {
@@ -489,16 +493,16 @@ function initStores(
 		} else {
 			tableId = primaryAttribute.tableId;
 			if (tableId) {
-				if (tableId >= (dbisStore.getSync(NEXT_TABLE_ID) || 0)) {
-					dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
+				if (tableId >= (attributesDbi.getSync(NEXT_TABLE_ID) || 0)) {
+					attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
 					logger.info(`Updating next table id (it was out of sync) to ${tableId + 1} for ${tableName}`);
 				}
 			} else {
-				primaryAttribute.tableId = tableId = dbisStore.getSync(NEXT_TABLE_ID);
+				primaryAttribute.tableId = tableId = attributesDbi.getSync(NEXT_TABLE_ID);
 				if (!tableId) tableId = 1;
 				logger.debug(`Table {tableName} missing an id, assigning {tableId}`);
-				dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
-				dbisStore.putSync(primaryAttribute.key, primaryAttribute);
+				attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
+				attributesDbi.putSync(primaryAttribute.key, primaryAttribute);
 			}
 			const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
 			dbiInit.compression = primaryAttribute.compression;
@@ -544,7 +548,18 @@ function initStores(
 			const attribute = attributes.find((attribute) => attribute.name === existingAttribute.name);
 			if (!attribute) {
 				if (existingAttribute.isPrimaryKey) {
-					logger.error('Unable to remove existing primary key attribute', existingAttribute);
+					logger.error(
+						new Error('Unable to remove existing primary key attribute'),
+						existingAttribute,
+						'from attributes',
+						existingAttributes,
+						'in',
+						tableName,
+						'requesting new attribute list',
+						attributes,
+						'full metadata list',
+						Array.from(attributesDbi.getRange({ start: false }))
+					);
 					continue;
 				}
 				if (existingAttribute.indexed) {
@@ -581,7 +596,7 @@ function initStores(
 					indices,
 					attributes,
 					schemaDefined: primaryAttribute.schemaDefined,
-					dbisDB: dbisStore,
+					dbisDB: attributesDbi,
 				})
 			);
 			table.schemaVersion = 1;

package/core/resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -130,8 +130,8 @@ export class HierarchicalNavigableSmallWorld {
 			// Generate random level for this new element
 			const level = oldNode.level ?? Math.min(Math.floor(-Math.log(Math.random()) * this.mL), MAX_LEVEL);
 			let currentLevel = entryPoint.level;
-			if (level >= currentLevel) {
-				// if we are at this level or higher, make this the new entry point
+			if (level > currentLevel) {
+				// if we are at a higher level, make this the new entry point
 				if (typeof nodeId !== 'number') {
 					throw new Error('Invalid nodeId: ' + nodeId);
 				}
@@ -142,7 +142,14 @@ export class HierarchicalNavigableSmallWorld {
 			// For each level from top to bottom
 			while (currentLevel > level) {
 				// Search for closest neighbors at current level
-				const neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, currentLevel);
+				const neighbors = this.searchLayer(
+					vector,
+					entryPointId,
+					entryPoint,
+					this.efConstruction,
+					currentLevel,
+					options
+				);
 
 				if (neighbors.length > 0) {
 					entryPointId = neighbors[0].id; // closest neighbor becomes new entry point
@@ -157,7 +164,7 @@ export class HierarchicalNavigableSmallWorld {
 
 			// Connect the new element to neighbors at its level and below
 			for (let l = Math.min(level, currentLevel); l >= 0; l--) {
-				let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l);
+				let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l, options);
 				neighbors = neighbors.slice(0, this.M << 1) as SearchResults;
 
 				if (neighbors.length === 0 && l === 0) {
@@ -232,6 +239,19 @@ export class HierarchicalNavigableSmallWorld {
 							oldNode[l] = oldConnections;
 						}
 						oldConnections.splice(oldPosition, 1);
+						// update the distance in the reverse connection if the vector changed
+						if (oldConnection.distance !== distance) {
+							const neighborNode = updateNode(id, node);
+							if (neighborNode[l]) {
+								if (Object.isFrozen(neighborNode[l])) {
+									neighborNode[l] = neighborNode[l].slice();
+								}
+								const reverseIdx = neighborNode[l].findIndex(({ id: nid }) => nid === nodeId);
+								if (reverseIdx >= 0) {
+									neighborNode[l][reverseIdx] = { id: nodeId, distance };
+								}
+							}
+						}
 					} else {
 						// add new connection since this is truly a new connection now
 						this.addConnection(id, updateNode(id, node), nodeId, l, distance, updateNode, options);
@@ -323,16 +343,16 @@ export class HierarchicalNavigableSmallWorld {
 			this.indexStore.put(id, updatedNode, options);
 		}
 		for (const [key, vector] of needsReindexing) {
-			this.index(key, vector, vector);
+			this.index(key, vector, vector, options);
 		}
 		this.checkSymmetry(nodeId, this.indexStore.getSync(nodeId, options), options);
 	}
 
-	private getEntryPoint() {
+	private getEntryPoint(options: { transaction?: any } = {}) {
 		// Get entry point
-		const entryPointId = this.indexStore.getSync(ENTRY_POINT);
+		const entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
 		if (entryPointId === undefined) return;
-		const node = this.indexStore.getSync(entryPointId);
+		const node = this.indexStore.getSync(entryPointId, options);
 		return { id: entryPointId, ...node };
 	}
 
@@ -346,6 +366,7 @@ export class HierarchicalNavigableSmallWorld {
 	 * @param ef
 	 * @param level
 	 * @param distanceFunction
+	 * @param options
 	 * @private
 	 */
 	private searchLayer(
@@ -354,13 +375,14 @@ export class HierarchicalNavigableSmallWorld {
 		entryPoint: any,
 		ef: number,
 		level: number,
+		options: { transaction?: any } = {},
 		distanceFunction = this.distance
 	): SearchResults {
 		const visited = new Set([entryPointId]);
 		const candidates = [
 			{
 				id: entryPointId,
-				distance: this.distance(queryVector, entryPoint.vector),
+				distance: distanceFunction(queryVector, entryPoint.vector),
 				node: entryPoint,
 			},
 		];
@@ -383,7 +405,7 @@ export class HierarchicalNavigableSmallWorld {
 				if (visited.has(neighborId) || neighborId === undefined) continue;
 				visited.add(neighborId);
 
-				const neighbor = this.indexStore.getSync(neighborId);
+				const neighbor = this.indexStore.getSync(neighborId, options);
 				if (!neighbor) continue;
 				this.nodesVisitedCount++;
 				const distance = distanceFunction(queryVector, neighbor.vector);
@@ -416,19 +438,22 @@ export class HierarchicalNavigableSmallWorld {
 	 * @param comparator
 	 * @param context
 	 */
-	search({
-		target,
-		value,
-		descending,
-		distance,
-		comparator,
-	}: {
-		target: number[];
-		value: number;
-		descending: boolean;
-		distance: string;
-		comparator: string;
-	}) {
+	search(
+		{
+			target,
+			value,
+			descending,
+			distance,
+			comparator,
+		}: {
+			target: number[];
+			value: number;
+			descending: boolean;
+			distance: string;
+			comparator: string;
+		},
+		context: any
+	) {
 		let limit = 0; // zero is ignored, only used if set below
 		switch (comparator) {
 			case 'lt':
@@ -449,14 +474,23 @@ export class HierarchicalNavigableSmallWorld {
 		if (!target) throw new ClientError('A target vector must be provided for an HNSW query');
 		if (!Array.isArray(target)) throw new ClientError('The target vector must be an array');
 
-		let entryPoint = this.getEntryPoint();
+		const options = context.transaction; // should have a nested RocksDB transaction
+		let entryPoint = this.getEntryPoint(options);
 		if (!entryPoint) return [];
 		let entryPointId = entryPoint.id;
 		let results: Candidate[] = [];
 		// For each level from top to bottom
 		for (let l = entryPoint.level; l >= 0; l--) {
 			// Search for closest neighbors at current level
-			results = this.searchLayer(target, entryPointId, entryPoint, this.efConstructionSearch, l, distanceFunction);
+			results = this.searchLayer(
+				target,
+				entryPointId,
+				entryPoint,
+				this.efConstructionSearch,
+				l,
+				options,
+				distanceFunction
+			);
 
 			if (results.length > 0) {
 				const neighbor = results[0]; // closest neighbor becomes new entry point
@@ -487,7 +521,7 @@ export class HierarchicalNavigableSmallWorld {
 				// verify that the connection is symmetrical
 				const symmetrical = neighborNode[l]?.find(({ id: nid }) => nid == id);
 				if (!symmetrical) {
-					logger.info?.('asymmetry detected', neighborNode[l]);
+					logger.info?.('asymmetry detected', neighborNode[l], 'does not have', id);
 				}
 			}
 			l++;
@@ -531,10 +565,13 @@ export class HierarchicalNavigableSmallWorld {
 				if (removedNode) {
 					// Remove the reverse connection if it exists
 					if (removedNode[level]) {
-						removedNode = updateNode(removed.id, removedNode);
-						removedNode[level] = removedNode[level].filter(({ id }) => id !== fromId);
-						if (level === 0 && removedNode[level].length === 0) {
-							logger.info?.('should not remove last connection', fromId, toId);
+						const filtered = removedNode[level].filter(({ id }) => id !== fromId);
+						if (level === 0 && filtered.length === 0) {
+							// don't remove the last connection at level 0 — it would orphan this node
+							logger.info?.('skipping removal of last connection', fromId, toId);
+						} else {
+							removedNode = updateNode(removed.id, removedNode);
+							removedNode[level] = filtered;
 						}
 					}
 				}

package/core/security/certificateVerification/ocspVerification.ts

@@ -95,7 +95,7 @@ export async function verifyOCSP(
 			method: cached.method || 'ocsp',
 		};
 	} catch (error) {
-		logger.error?.(`OCSP verification error: ${error}`);
+		logger.error?.(`OCSP verification error:`, error);
 
 		// Check failure mode
 		if (config.failureMode === 'fail-closed') {