@harperfast/harper 5.0.2 → 5.0.3

package/resources/Table.ts

@@ -1424,23 +1424,35 @@ export function makeTable(options) {
 		 */
 		static evict(id, existingRecord, existingVersion) {
 			let entry;
-			if (hasSourceGet || audit) {
-				if (!existingRecord) return;
-				entry = primaryStore.getEntry(id);
-				if (!entry || !existingRecord) return;
-				if (entry.version !== existingVersion) return;
-			}
-			if (hasSourceGet) {
-				// if there is a resolution in-progress, abandon the eviction
-				if (primaryStore.hasLock(id, entry.version)) return;
+			let transaction = txnForContext({ transaction: new DatabaseTransaction() }).getReadTxn();
+			let options = { transaction };
+			try {
+				if (hasSourceGet || audit) {
+					if (!existingRecord) return;
+					entry = primaryStore.getEntry(id, options);
+					if (!entry || !existingRecord) return;
+					if (entry.version !== existingVersion) return;
+				}
+				if (hasSourceGet) {
+					// if there is a resolution in-progress, abandon the eviction
+					if (primaryStore.hasLock(id, entry.version)) return;
+				}
+				// evictions never go in the audit log, so we can not record a deletion entry for the eviction
+				// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
+				// removed the entry entirely, but first cleanup indices
+				if (primaryStore.ifVersion) {
+					// lmdb
+					primaryStore.ifVersion?.(id, existingVersion, () => {
+						updateIndices(id, existingRecord, null);
+					});
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
+				} else {
+					updateIndices(id, existingRecord, null, options);
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
+				}
+			} finally {
+				return transaction.commit();
 			}
-			primaryStore.ifVersion?.(id, existingVersion, () => {
-				updateIndices(id, existingRecord, null);
-			});
-			// evictions never go in the audit log, so we can not record a deletion entry for the eviction
-			// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
-			// removed the entry entirely
-			return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
 		}
 		/**
 		 * This is intended to acquire a lock on a record from the whole cluster.
@@ -1951,9 +1963,10 @@ export function makeTable(options) {
 							context.lastModified = existingEntry.version;
 						TableResource._updateResource(this, existingEntry);
 					}
-					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return; // a newer record exists locally
-					updateIndices(id, existingRecord);
-					logger.trace?.(`Deleting record with id: ${id}, txn timestamp: ${new Date(txnTime).toISOString()}`);
+					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
+						return;
+					} // a newer record exists locally
+					updateIndices(id, existingRecord, null, transaction && { transaction });
 					if (audit || trackDeletes) {
 						updateRecord(
 							id,
@@ -3511,6 +3524,7 @@ export function makeTable(options) {
 			// determine what index values need to be removed and added
 			let valuesToAdd = getIndexedValues(value, indexNulls) as any[];
 			let valuesToRemove = getIndexedValues(existingValue, indexNulls) as any[];
+			let isLMDB = !!index.prefetch;
 			if (valuesToRemove?.length > 0) {
 				// put this in a conditional so we can do a faster version for new records
 				// determine the changes/diff from new values and old values
@@ -3527,18 +3541,18 @@ export function makeTable(options) {
 						})
 					: [];
 				valuesToRemove = Array.from(setToRemove);
-				if ((valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
+				if (isLMDB && (valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
 					// prefetch any values that have been removed or added
 					const valuesToPrefetch = valuesToRemove.concat(valuesToAdd).map((v) => ({ key: v, value: id }));
-					index.prefetch?.(valuesToPrefetch, noop);
+					index.prefetch(valuesToPrefetch, noop);
 				}
 				//if the update cleared out the attribute value we need to delete it from the index
 				for (let i = 0, l = valuesToRemove.length; i < l; i++) {
 					index.remove(valuesToRemove[i], id, options);
 				}
-			} else if (valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
+			} else if (isLMDB && valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
 				// no old values, just new
-				index.prefetch?.(
+				index.prefetch(
 					valuesToAdd.map((v) => ({ key: v, value: id })),
 					noop
 				);
@@ -4124,7 +4138,7 @@ export function makeTable(options) {
 								// don't do anything if the version has changed
 								return;
 							}
-							updateIndices(id, existingRecord, updatedRecord);
+							updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
 							if (updatedRecord) {
 								if (existingEntry) {
 									context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);

package/resources/databases.ts

@@ -120,9 +120,13 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo
 	}
 	let db: RocksRootDatabase;
 	if (options.dupSort) {
-		db = RocksDatabase.open(new RocksIndexStore(path, options)) as RocksDatabaseEx;
+		db = new RocksIndexStore(path, options).open() as RocksDatabaseEx;
 	} else {
 		db = RocksDatabase.open(path, options) as RocksDatabaseEx;
+		// the RocksDB put and remove return promises, which masks thrown errors in non-awaiting calls to put/remove,
+		// making them unsafe to replace LMDB methods, which will synchronously throw errors if there is a problem
+		db.put = db.putSync;
+		db.remove = db.removeSync;
 		db.encoder.name = options.name;
 	}
 	db.env = {};
@@ -386,18 +390,18 @@ function initStores(
 ) {
 	const envInit = new OpenEnvironmentObject(path, false);
 	const internalDbiInit = createOpenDBIObject(false);
-	let dbisStore = rootStore.dbisDb;
-	if (!dbisStore) {
+	let attributesDbi = rootStore.dbisDb;
+	if (!attributesDbi) {
 		if (rootStore instanceof RocksDatabase) {
-			dbisStore = openRocksDatabase(rootStore.path, {
+			attributesDbi = openRocksDatabase(rootStore.path, {
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
 			}) as RocksDatabaseEx;
 		} else {
-			dbisStore = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
+			attributesDbi = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
 		}
-		rootStore.dbisDb = dbisStore;
+		rootStore.dbisDb = attributesDbi;
 	}
 
 	let auditStore = rootStore.auditStore;
@@ -428,7 +432,7 @@ function initStores(
 	definedTables.rootStore = rootStore;
 	const tablesToLoad = new Map<string, any>();
 
-	for (const result of dbisStore.getRange({ start: false })) {
+	for (const result of attributesDbi.getRange({ start: false })) {
 		const { key, value } = result as { key: string; value: any };
 		let [tableName, attribute_name] = key.toString().split('/');
 		if (attribute_name === '') {
@@ -489,16 +493,16 @@ function initStores(
 		} else {
 			tableId = primaryAttribute.tableId;
 			if (tableId) {
-				if (tableId >= (dbisStore.getSync(NEXT_TABLE_ID) || 0)) {
-					dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
+				if (tableId >= (attributesDbi.getSync(NEXT_TABLE_ID) || 0)) {
+					attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
 					logger.info(`Updating next table id (it was out of sync) to ${tableId + 1} for ${tableName}`);
 				}
 			} else {
-				primaryAttribute.tableId = tableId = dbisStore.getSync(NEXT_TABLE_ID);
+				primaryAttribute.tableId = tableId = attributesDbi.getSync(NEXT_TABLE_ID);
 				if (!tableId) tableId = 1;
 				logger.debug(`Table {tableName} missing an id, assigning {tableId}`);
-				dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
-				dbisStore.putSync(primaryAttribute.key, primaryAttribute);
+				attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
+				attributesDbi.putSync(primaryAttribute.key, primaryAttribute);
 			}
 			const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
 			dbiInit.compression = primaryAttribute.compression;
@@ -544,7 +548,18 @@ function initStores(
 			const attribute = attributes.find((attribute) => attribute.name === existingAttribute.name);
 			if (!attribute) {
 				if (existingAttribute.isPrimaryKey) {
-					logger.error('Unable to remove existing primary key attribute', existingAttribute);
+					logger.error(
+						new Error('Unable to remove existing primary key attribute'),
+						existingAttribute,
+						'from attributes',
+						existingAttributes,
+						'in',
+						tableName,
+						'requesting new attribute list',
+						attributes,
+						'full metadata list',
+						Array.from(attributesDbi.getRange({ start: false }))
+					);
 					continue;
 				}
 				if (existingAttribute.indexed) {
@@ -581,7 +596,7 @@ function initStores(
 					indices,
 					attributes,
 					schemaDefined: primaryAttribute.schemaDefined,
-					dbisDB: dbisStore,
+					dbisDB: attributesDbi,
 				})
 			);
 			table.schemaVersion = 1;

package/resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -142,7 +142,14 @@ export class HierarchicalNavigableSmallWorld {
 			// For each level from top to bottom
 			while (currentLevel > level) {
 				// Search for closest neighbors at current level
-				const neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, currentLevel);
+				const neighbors = this.searchLayer(
+					vector,
+					entryPointId,
+					entryPoint,
+					this.efConstruction,
+					currentLevel,
+					options
+				);
 
 				if (neighbors.length > 0) {
 					entryPointId = neighbors[0].id; // closest neighbor becomes new entry point
@@ -157,7 +164,7 @@ export class HierarchicalNavigableSmallWorld {
 
 			// Connect the new element to neighbors at its level and below
 			for (let l = Math.min(level, currentLevel); l >= 0; l--) {
-				let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l);
+				let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l, options);
 				neighbors = neighbors.slice(0, this.M << 1) as SearchResults;
 
 				if (neighbors.length === 0 && l === 0) {
@@ -336,16 +343,16 @@ export class HierarchicalNavigableSmallWorld {
 			this.indexStore.put(id, updatedNode, options);
 		}
 		for (const [key, vector] of needsReindexing) {
-			this.index(key, vector, vector);
+			this.index(key, vector, vector, options);
 		}
 		this.checkSymmetry(nodeId, this.indexStore.getSync(nodeId, options), options);
 	}
 
-	private getEntryPoint() {
+	private getEntryPoint(options: { transaction?: any } = {}) {
 		// Get entry point
-		const entryPointId = this.indexStore.getSync(ENTRY_POINT);
+		const entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
 		if (entryPointId === undefined) return;
-		const node = this.indexStore.getSync(entryPointId);
+		const node = this.indexStore.getSync(entryPointId, options);
 		return { id: entryPointId, ...node };
 	}
 
@@ -359,6 +366,7 @@ export class HierarchicalNavigableSmallWorld {
 	 * @param ef
 	 * @param level
 	 * @param distanceFunction
+	 * @param options
 	 * @private
 	 */
 	private searchLayer(
@@ -367,6 +375,7 @@ export class HierarchicalNavigableSmallWorld {
 		entryPoint: any,
 		ef: number,
 		level: number,
+		options: { transaction?: any } = {},
 		distanceFunction = this.distance
 	): SearchResults {
 		const visited = new Set([entryPointId]);
@@ -396,7 +405,7 @@ export class HierarchicalNavigableSmallWorld {
 				if (visited.has(neighborId) || neighborId === undefined) continue;
 				visited.add(neighborId);
 
-				const neighbor = this.indexStore.getSync(neighborId);
+				const neighbor = this.indexStore.getSync(neighborId, options);
 				if (!neighbor) continue;
 				this.nodesVisitedCount++;
 				const distance = distanceFunction(queryVector, neighbor.vector);
@@ -429,19 +438,22 @@ export class HierarchicalNavigableSmallWorld {
 	 * @param comparator
 	 * @param context
 	 */
-	search({
-		target,
-		value,
-		descending,
-		distance,
-		comparator,
-	}: {
-		target: number[];
-		value: number;
-		descending: boolean;
-		distance: string;
-		comparator: string;
-	}) {
+	search(
+		{
+			target,
+			value,
+			descending,
+			distance,
+			comparator,
+		}: {
+			target: number[];
+			value: number;
+			descending: boolean;
+			distance: string;
+			comparator: string;
+		},
+		context: any
+	) {
 		let limit = 0; // zero is ignored, only used if set below
 		switch (comparator) {
 			case 'lt':
@@ -462,14 +474,23 @@ export class HierarchicalNavigableSmallWorld {
 		if (!target) throw new ClientError('A target vector must be provided for an HNSW query');
 		if (!Array.isArray(target)) throw new ClientError('The target vector must be an array');
 
-		let entryPoint = this.getEntryPoint();
+		const options = context.transaction; // should have a nested RocksDB transaction
+		let entryPoint = this.getEntryPoint(options);
 		if (!entryPoint) return [];
 		let entryPointId = entryPoint.id;
 		let results: Candidate[] = [];
 		// For each level from top to bottom
 		for (let l = entryPoint.level; l >= 0; l--) {
 			// Search for closest neighbors at current level
-			results = this.searchLayer(target, entryPointId, entryPoint, this.efConstructionSearch, l, distanceFunction);
+			results = this.searchLayer(
+				target,
+				entryPointId,
+				entryPoint,
+				this.efConstructionSearch,
+				l,
+				options,
+				distanceFunction
+			);
 
 			if (results.length > 0) {
 				const neighbor = results[0]; // closest neighbor becomes new entry point
@@ -500,7 +521,7 @@ export class HierarchicalNavigableSmallWorld {
 				// verify that the connection is symmetrical
 				const symmetrical = neighborNode[l]?.find(({ id: nid }) => nid == id);
 				if (!symmetrical) {
-					logger.info?.('asymmetry detected', neighborNode[l]);
+					logger.info?.('asymmetry detected', neighborNode[l], 'does not have', id);
 				}
 			}
 			l++;

package/security/certificateVerification/ocspVerification.ts

@@ -95,7 +95,7 @@ export async function verifyOCSP(
 			method: cached.method || 'ocsp',
 		};
 	} catch (error) {
-		logger.error?.(`OCSP verification error: ${error}`);
+		logger.error?.(`OCSP verification error:`, error);
 
 		// Check failure mode
 		if (config.failureMode === 'fail-closed') {

package/security/keys.js

@@ -125,7 +125,7 @@ function getCertTable() {
 }
 
 async function getReplicationCert() {
-	const SNICallback = createTLSSelector('operations-api');
+	const SNICallback = createTLSSelector('replication');
 	const secureTarget = {
 		secureContexts: null,
 		setSecureContext: (_ctx) => {},
@@ -238,7 +238,7 @@ function loadCertificates() {
 
 								promise = certificateTable.put({
 									name: certCn,
-									uses: config.uses ?? ['https', ...(configKey.includes('operations') ? ['operations'] : [])],
+									uses: config.uses ?? [configKey.includes('operations') ? ['operations-api'] : []],
 									ciphers: config.ciphers,
 									certificate: certificatePem,
 									private_key_name,
@@ -351,7 +351,7 @@ function certExtensions() {
 async function createCertificateTable(cert, caCert) {
 	await setCertTable({
 		name: getThisNodeName(),
-		uses: ['https', 'wss'],
+		uses: ['replication'],
 		certificate: cert,
 		private_key_name: 'privateKey.pem',
 		is_authority: false,
@@ -360,7 +360,7 @@ async function createCertificateTable(cert, caCert) {
 
 	await setCertTable({
 		name: caCert.subject.getField('CN').value,
-		uses: ['https', 'wss'],
+		uses: [],
 		certificate: pki.certificateToPem(caCert),
 		private_key_name: 'privateKey.pem',
 		is_authority: true,
@@ -600,7 +600,7 @@ async function reviewSelfSignedCert() {
 
 		await setCertTable({
 			name: hdbCa.subject.getField('CN').value,
-			uses: ['https'],
+			uses: [],
 			certificate: pki.certificateToPem(hdbCa),
 			private_key_name: keyName,
 			is_authority: true,
@@ -621,7 +621,7 @@ async function reviewSelfSignedCert() {
 		const newPublicCert = await generateCertificates(pki.privateKeyFromPem(caAndKey.private_key), publicKey, hdbCa);
 		await setCertTable({
 			name: certName,
-			uses: ['https', 'operations', 'wss'],
+			uses: ['replication'],
 			certificate: newPublicCert,
 			is_authority: false,
 			private_key_name: caAndKey.ca.private_key_name,
@@ -755,7 +755,7 @@ function createTLSSelector(type, mtlsOptions) {
 							}
 							let quality = cert.is_self_signed ? 1 : 3;
 							// prefer operations certificates for operations API
-							if (cert.uses?.includes(type)) quality += 1;
+							if (cert.uses?.includes(type)) quality += 3;
 
 							const private_key = getPrivateKeyByName(cert.private_key_name);
 

package/server/itc/serverHandlers.js

@@ -46,10 +46,6 @@ async function schemaHandler(event) {
  */
 async function syncSchemaMetadata(msg) {
 	try {
-		// reset current read transactions to ensure that we are getting the very latest data
-		harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.TABLE_TABLE_NAME);
-		harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.ATTRIBUTE_TABLE_NAME);
-		harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.SCHEMA_TABLE_NAME);
 		// TODO: Eventually should indicate which database/table changed so we don't have to scan everything
 		let databases = resetDatabases();
 		if (msg.table && msg.database)