@harperfast/harper-pro 5.0.2 → 5.0.3

package/core/bin/cliOperations.js

@@ -63,7 +63,7 @@ function buildRequest() {
  */
 async function cliOperations(req) {
 	if (!req.target) {
-		req.target = process.env.CLI_TARGET;
+		req.target = process.env.HARPER_CLI_TARGET || process.env.CLI_TARGET;
 	}
 	let target;
 	if (req.target) {
@@ -73,20 +73,22 @@ async function cliOperations(req) {
 			try {
 				target = new URL(`https://${req.target}:9925`);
 			} catch {
-				throw error; // throw the original error
+				throw error;
 			}
 		}
 		target = {
 			protocol: target.protocol,
 			hostname: target.hostname,
 			port: target.port,
-			username: req.username || target.username || process.env.CLI_TARGET_USERNAME,
-			password: req.password || target.password || process.env.CLI_TARGET_PASSWORD,
+			username: req.username || target.username || process.env.HARPER_CLI_USERNAME || process.env.CLI_TARGET_USERNAME,
+			password: req.password || target.password || process.env.HARPER_CLI_PASSWORD || process.env.CLI_TARGET_PASSWORD,
 			rejectUnauthorized: req.rejectUnauthorized,
 		};
+		console.error(`Connecting to ${target.protocol}//${target.hostname}:${target.port}`);
 	} else {
 		// if we aren't doing a targeted operation (like deploy), we initialize the config and verify that local harper
 		// is running and that we can communicate with it.
+		console.error('Connecting to local Harper Pro instance');
 		initConfig();
 		if (!getHdbPid()) {
 			console.error('Harper Pro must be running to perform this operation');

package/core/bin/copyDb.ts

@@ -2,6 +2,7 @@ import { getDatabases, getDefaultCompression, resetDatabases } from '../resource
 import { open } from 'lmdb';
 import { join } from 'path';
 import { move, remove } from 'fs-extra';
+import { existsSync, mkdirSync } from 'node:fs';
 import { get } from '../utility/environment/environmentManager.js';
 import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
 import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.js';
@@ -11,6 +12,8 @@ import { AUDIT_STORE_OPTIONS } from '../resources/auditStore.ts';
 import { describeSchema } from '../dataLayer/schemaDescribe.js';
 import { updateConfigValue } from '../config/configUtils.js';
 import * as hdbLogger from '../utility/logging/harper_logger.js';
+import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
+import { RocksIndexStore } from '../resources/RocksIndexStore.ts';
 
 export async function compactOnStart() {
 	hdbLogger.notify('Running compact on start');
@@ -278,3 +281,208 @@ export async function copyDb(sourceDatabase: string, targetDatabasePath: string)
 		targetEnv.close();
 	}
 }
+
+function openRocksDb(path: string, options: RocksDatabaseOptions & { dupSort?: boolean } = {}) {
+	options.disableWAL ??= false;
+	if (!existsSync(path)) {
+		mkdirSync(path, { recursive: true });
+	}
+	let db;
+	if (options.dupSort) {
+		db = RocksDatabase.open(new RocksIndexStore(path, options));
+	} else {
+		db = RocksDatabase.open(path, options);
+		db.encoder.name = options.name;
+	}
+	return db;
+}
+
+export async function migrateOnStart() {
+	hdbLogger.notify('Running migrate on start (LMDB to RocksDB)');
+	console.log('Running migrate on start (LMDB to RocksDB)');
+
+	const rootPath = get(CONFIG_PARAMS.ROOTPATH);
+	const databases = getDatabases();
+
+	updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false);
+
+	try {
+		for (const databaseName in databases) {
+			if (databaseName === 'system') continue;
+			if (databaseName.endsWith('-copy')) continue;
+			let rootStore;
+			for (const tableName in databases[databaseName]) {
+				const table = databases[databaseName][tableName];
+				table.primaryStore.put = noop;
+				table.primaryStore.remove = noop;
+				for (const attributeName in table.indices) {
+					const index = table.indices[attributeName];
+					index.put = noop;
+					index.remove = noop;
+				}
+				if (table.auditStore) {
+					table.auditStore.put = noop;
+					table.auditStore.remove = noop;
+				}
+				rootStore = table.primaryStore.rootStore;
+			}
+			if (!rootStore) {
+				console.log("Couldn't find any tables in database", databaseName);
+				continue;
+			}
+			if (rootStore instanceof RocksDatabase) {
+				console.log('Database', databaseName, 'is already RocksDB, skipping');
+				continue;
+			}
+
+			const targetPath = join(rootPath, DATABASES_DIR_NAME, databaseName);
+			const lmdbPath = rootStore.path;
+			const backupDest = join(rootPath, 'backup', databaseName + '.mdb');
+
+			console.log('Migrating', databaseName, 'from LMDB to RocksDB at', targetPath);
+
+			await copyDbToRocks(rootStore, databaseName, targetPath);
+
+			// Back up the original LMDB file
+			console.log('Backing up LMDB', databaseName, 'to', backupDest);
+			try {
+				await move(lmdbPath, backupDest, { overwrite: true });
+			} catch (error) {
+				console.log('Error moving database', lmdbPath, 'to', backupDest, error);
+			}
+			// Remove the lock file
+			try {
+				await remove(lmdbPath + '-lock');
+			} catch {
+				// lock file may not exist
+			}
+		}
+
+		try {
+			resetDatabases();
+		} catch (err) {
+			hdbLogger.error('Error resetting databases after migration', err);
+			console.error('Error resetting databases after migration', err);
+		}
+	} catch (err) {
+		hdbLogger.error('Error migrating database', err);
+		console.error('Error migrating database', err);
+		throw err;
+	}
+}
+
+async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath: string) {
+	console.log(`Migrating database ${sourceDatabase} to RocksDB at ${targetPath}`);
+	const sourceDbisDb = sourceRootStore.dbisDb;
+
+	const targetRootStore = openRocksDb(targetPath, { disableWAL: false });
+	const targetDbisDb = openRocksDb(targetPath, {
+		disableWAL: false,
+		name: INTERNAL_DBIS_NAME,
+	});
+
+	let written;
+	let outstandingWrites = 0;
+	const transaction = sourceDbisDb.useReadTransaction();
+	try {
+		for (const { key, value: attribute } of sourceDbisDb.getRange({ transaction })) {
+			const isPrimary = attribute.isPrimaryKey;
+			targetDbisDb.put(key, attribute);
+			if (!(isPrimary || attribute.indexed)) continue;
+
+			// Open source LMDB dbi with default encoding so values are decoded
+			const dbiInit = new OpenDBIObject(!isPrimary, isPrimary);
+			const sourceDbi = sourceRootStore.openDB(key, dbiInit);
+
+			let targetDbi;
+			if (!isPrimary) {
+				targetDbi = openRocksDb(targetPath, { dupSort: true, name: key });
+			} else {
+				targetDbi = openRocksDb(targetPath, { name: key });
+			}
+
+			console.log('migrating', key, 'from', sourceDatabase, 'to RocksDB');
+			await copyDbiToRocks(sourceDbi, targetDbi, isPrimary, transaction);
+		}
+
+		// Note: audit store is not migrated because LMDB and RocksDB use fundamentally different
+		// audit store formats (LMDB uses a custom binary encoding in a regular DB, RocksDB uses TransactionLog).
+		// A new audit store will be created automatically when the RocksDB database is opened.
+
+		await written;
+		console.log('migrated database ' + sourceDatabase + ' to RocksDB');
+	} finally {
+		transaction.done();
+		targetRootStore.close();
+	}
+
+	async function copyDbiToRocks(sourceDbi, targetDbi, isPrimary, transaction) {
+		let recordsCopied = 0;
+		let skippedRecord = 0;
+		let retries = 1000000;
+		let start = null;
+		while (retries-- > 0) {
+			try {
+				if (isPrimary) {
+					for (const { key, value, version } of sourceDbi.getRange({ start, transaction, versions: true })) {
+						try {
+							start = key;
+							if (value == null) {
+								skippedRecord++;
+								continue;
+							}
+							written = targetDbi.put(key, value, version);
+							recordsCopied++;
+							if (transaction.openTimer) transaction.openTimer = 0;
+							if (outstandingWrites++ > 5000) {
+								await written;
+								console.log('migrated', recordsCopied, 'entries, skipped', skippedRecord, 'delete records');
+								outstandingWrites = 0;
+							}
+						} catch (error) {
+							console.error(
+								'Error migrating record',
+								typeof key === 'symbol' ? 'symbol' : key,
+								'from',
+								sourceDatabase,
+								error
+							);
+						}
+					}
+				} else {
+					for (const { key, value } of sourceDbi.getRange({ start, transaction })) {
+						try {
+							start = key;
+							written = targetDbi.put(key, value);
+							recordsCopied++;
+							if (transaction.openTimer) transaction.openTimer = 0;
+							if (outstandingWrites++ > 5000) {
+								await written;
+								console.log('migrated', recordsCopied, 'index entries');
+								outstandingWrites = 0;
+							}
+						} catch (error) {
+							console.error(
+								'Error migrating index record',
+								typeof key === 'symbol' ? 'symbol' : key,
+								'from',
+								sourceDatabase,
+								error
+							);
+						}
+					}
+				}
+				console.log('finish migrating, copied', recordsCopied, 'entries, skipped', skippedRecord, 'delete records');
+				return;
+			} catch {
+				if (typeof start === 'string') {
+					if (start === 'z') {
+						return console.error('Reached end of dbi', start, 'for', sourceDatabase);
+					}
+					start = start.slice(0, -2) + 'z';
+				} else if (typeof start === 'number') start++;
+				else return console.error('Unknown key type', start, 'for', sourceDatabase);
+			}
+		}
+	}
+}

package/core/bin/restart.js

@@ -61,12 +61,6 @@ async function restart(req) {
 
 		if (envMgr.get(hdbTerms.CONFIG_PARAMS.STORAGE_COMPACTONSTART)) await compactOnStart();
 
-		if (process.env.HARPER_EXIT_ON_RESTART) {
-			// use this to exit the process so that it will be restarted by the
-			// PM/container/orchestrator.
-			hdbLogger.warn('Exiting Harper Pro process to trigger a container restart');
-			process.exit(0);
-		}
 		setTimeout(async () => {
 			// It seems like you should just be able to start the other process and kill this process and everything should
 			// be cleaned up, however that doesn't work for some reason; the socket listening fds somehow get transferred to the
@@ -79,9 +73,16 @@ async function restart(req) {
 			// remove pid file so it doesn't trip up the launch
 			await unlinkSync(path.join(envMgr.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), hdbTerms.HDB_PID_FILE), `${process.pid}`);
 			hdbLogger.debug('Starting new process...');
+			if (process.env.HARPER_EXIT_ON_RESTART) {
+				// use this to exit the process so that it will be restarted by the
+				// PM/container/orchestrator.
+				hdbLogger.warn('Exiting Harper Pro process to trigger a container restart');
+				process.exit(0);
+			}
 			// now launch the new process and exit this process
 			require('./run.js').launch(true);
-		}, 50); // can't await this because it is going to do an exit()
+		}, 50); // can't await this because it is going to do an exit(), but wait for 50ms so we give the HTTP thread a
+		// chance to return a response
 	} else {
 		// Post msg to main parent thread requesting it restart (so the main thread can process.exit())
 		parentPort.postMessage({

package/core/bin/run.js

@@ -19,7 +19,7 @@ const installation = require('../utility/installation.ts');
 const configUtils = require('../config/configUtils.js');
 const assignCMDENVVariables = require('../utility/assignCmdEnvVariables.js');
 const upgrade = require('./upgrade.js');
-const { compactOnStart } = require('./copyDb.ts');
+const { compactOnStart, migrateOnStart } = require('./copyDb.ts');
 const minimist = require('minimist');
 const keys = require('../security/keys.js');
 const { startHTTPThreads } = require('../server/threads/socketRouter.ts');
@@ -192,6 +192,7 @@ async function main(calledByInstall = false) {
 		await initialize(calledByInstall, true);
 
 		if (env.get(terms.CONFIG_PARAMS.STORAGE_COMPACTONSTART)) await compactOnStart();
+		if (env.get(terms.CONFIG_PARAMS.STORAGE_MIGRATEONSTART)) await migrateOnStart();
 
 		const isScripted = process.env.IS_SCRIPTED_SERVICE && !cmdArgs.service;
 

package/core/resources/DatabaseTransaction.ts

@@ -7,6 +7,7 @@ import * as envMngr from '../utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 import { convertToMS } from '../utility/common_utils.js';
 import { when } from '../utility/when.ts';
+import { setTimeout as delay } from 'node:timers/promises';
 import { Transaction as RocksTransaction, type Store as RocksStore } from '@harperfast/rocksdb-js';
 import type { RootDatabaseKind } from './databases.ts';
 import type { Entry } from './RecordEncoder.ts';
@@ -19,6 +20,7 @@ export const TRANSACTION_STATE = {
 	OPEN: 1, // the transaction is open and can be used for reads and writes
 	LINGERING: 2, // the transaction has completed a read, but can be used for immediate writes
 };
+const MAX_RETRIES = 40;
 let outstandingCommit, outstandingCommitStart;
 let confirmReplication;
 export function replicationConfirmation(callback) {
@@ -93,6 +95,7 @@ export class DatabaseTransaction implements Transaction {
 		if (this.open !== TRANSACTION_STATE.OPEN) return; // can not start a new read transaction as there is no future commit that will take place, just have to allow the read to latest database state
 
 		this.transaction = new RocksTransaction(this.db.store);
+
 		if (this.timestamp) {
 			this.transaction.setTimestamp(this.timestamp);
 		}
@@ -158,7 +161,10 @@ export class DatabaseTransaction implements Transaction {
 		transaction ??= this.transaction;
 		let immediateCommit = false;
 		if (!transaction) {
-			transaction = new RocksTransaction(this.db.store as RocksStore);
+			transaction = new RocksTransaction(operation.store.store as RocksStore);
+			if (operation.store.rootStore !== this.db.rootStore) {
+				harperLogger.warn?.('Created new transaction in save, but the store does match existing store', transaction.id);
+			}
 			if (this.open === TRANSACTION_STATE.OPEN) {
 				this.transaction = transaction;
 			} else {
@@ -168,9 +174,10 @@ export class DatabaseTransaction implements Transaction {
 			if (txnTime) {
 				transaction.setTimestamp(txnTime);
 			}
+		} else {
 		}
 		if (this.retries > 0) {
-			// this is marks the rocks transaction as a retry so we don't write the transaction log again
+			// This marks the Rocks transaction as a retry so we don't write the transaction log again
 			transaction.isRetry = true;
 		}
 		if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
@@ -295,6 +302,16 @@ export class DatabaseTransaction implements Transaction {
 							// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
 							// for future transactions
 							this.retries++;
+							harperLogger.debug?.('retrying', transaction.id, this.retries);
+							if (this.retries > 2) {
+								if (this.retries > MAX_RETRIES) {
+									throw new ServerError(
+										`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`
+									);
+								}
+								// start delaying, back off to try to space out transactions and avoid excessive conflicts
+								return delay(this.retries * this.retries).then(() => this.commit({ transaction }));
+							}
 							return this.commit({ transaction }); // try again
 						} else throw error;
 					}

package/core/resources/RecordEncoder.ts

@@ -688,13 +688,13 @@ export function recordUpdater(store, tableId, auditStore) {
 export function setAdditionalAuditRefs(refs: Array<{ version: number; nodeId: number }> | undefined) {
 	additionalAuditRefsNextEncoding = refs;
 }
-export function removeEntry(store: any, entry: any, existingVersion?: number) {
+export function removeEntry(store: any, entry: any, options?: any) {
 	if (!entry) return;
 	if (entry.value && entry.metadataFlags & HAS_BLOBS) {
 		// if it used to have blobs, we need to delete the old blobs
 		deleteBlobsInObject(entry.value);
 	}
-	return store.remove(entry.key, existingVersion);
+	return store.remove(entry.key, options);
 }
 export interface RecordObject {
 	getUpdatedTime(): number;

package/core/resources/ResourceInterface.ts

@@ -69,7 +69,7 @@ export interface Context {
 	/**	 The user making the request */
 	user?: User;
 	/** Check the username and password against the core user table to verify user identity */
-	login: (username: string, password: string) => Promise<string>;
+	login?: (username: string, password: string) => Promise<string>;
 	/** Describes the current cookie-based session if it is present and grants the capacity to delete it. authentication.enableSessions must be turned on in the harperdb-config.yaml  */
 	session?: Session;
 	/**	 The database transaction object */

package/core/resources/RocksIndexStore.ts

@@ -1,27 +1,32 @@
 import {
 	DBI,
-	Store,
-	type StoreContext,
 	type StoreIteratorOptions,
 	type StorePutOptions,
 	type StoreRemoveOptions,
+	RocksDatabase,
 } from '@harperfast/rocksdb-js';
 import { Id } from './ResourceInterface.ts';
 import { MAXIMUM_KEY } from 'ordered-binary';
 
 declare module '@harperfast/rocksdb-js' {
-	// eslint-disable-next-line no-unused-vars
 	interface DBI<T> {
 		getValuesCount(indexedValue: any): number;
 	}
 }
 
-export class RocksIndexStore extends Store {
+/**
+ * A specialized RocksDB-based index store that maintains indexed references to primary keys.
+ * This store uses composite keys consisting of indexed values and primary keys, enabling
+ * efficient range queries over indexed data. The actual data values are stored as null since
+ * this is purely an index structure pointing to primary records elsewhere. This extends
+ * RocksDatabase rather than a store because it actually alters the interface
+ */
+export class RocksIndexStore extends RocksDatabase {
 	/**
 	 * Get all entries matching the range
 	 * @param options
 	 */
-	getRange(context: StoreContext, options: StoreIteratorOptions): Iterable<any> {
+	getRange(options: StoreIteratorOptions): Iterable<any> {
 		let { start, end, exclusiveStart, inclusiveEnd, reverse } = options;
 		if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) {
 			start = [start, MAXIMUM_KEY];
@@ -30,7 +35,7 @@ export class RocksIndexStore extends Store {
 			end = [end, MAXIMUM_KEY];
 		}
 		const translatedOptions = { ...options, start, end };
-		return super.getRange(context, translatedOptions).map(({ key }) => {
+		return super.getRange(translatedOptions).map(({ key }) => {
 			return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] };
 		});
 	}
@@ -41,20 +46,20 @@ export class RocksIndexStore extends Store {
 	 * @param primaryKey
 	 * @param txnId
 	 */
-	put(context: StoreContext, indexedValue: any, primaryKey: Id, options: StorePutOptions) {
-		return super.putSync(context, [indexedValue, primaryKey], null, options);
+	put(indexedValue: any, primaryKey: Id, options: StorePutOptions) {
+		return super.putSync([indexedValue, primaryKey], null, options);
 	}
 
-	putSync(context: StoreContext, indexedValue: any, primaryKey: Id, options: StorePutOptions) {
-		return super.putSync(context, [indexedValue, primaryKey], null, options);
+	putSync(indexedValue: any, primaryKey: Id, options: StorePutOptions) {
+		return super.putSync([indexedValue, primaryKey], null, options);
 	}
 
-	remove(context: StoreContext, indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
-		return super.removeSync(context, [indexedValue, primaryKey], options);
+	remove(indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
+		return super.removeSync([indexedValue, primaryKey], options);
 	}
 
-	removeSync(context: StoreContext, indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
-		super.removeSync(context, [indexedValue, primaryKey], options);
+	removeSync(indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
+		super.removeSync([indexedValue, primaryKey], options);
 	}
 }
 
@@ -63,7 +68,7 @@ export class RocksIndexStore extends Store {
  * classes.
  */
 DBI.prototype.getValuesCount = function getValuesCount(indexedValue: any) {
-	if (this.store instanceof RocksIndexStore) {
+	if (this instanceof RocksIndexStore) {
 		return this.store.getCount(this._context, { start: indexedValue, end: [indexedValue, MAXIMUM_KEY] });
 	}
 	throw new Error('getValuesCount is only supported if dupSort=true');

package/core/resources/Table.ts

@@ -1424,23 +1424,35 @@ export function makeTable(options) {
 		 */
 		static evict(id, existingRecord, existingVersion) {
 			let entry;
-			if (hasSourceGet || audit) {
-				if (!existingRecord) return;
-				entry = primaryStore.getEntry(id);
-				if (!entry || !existingRecord) return;
-				if (entry.version !== existingVersion) return;
-			}
-			if (hasSourceGet) {
-				// if there is a resolution in-progress, abandon the eviction
-				if (primaryStore.hasLock(id, entry.version)) return;
+			let transaction = txnForContext({ transaction: new DatabaseTransaction() }).getReadTxn();
+			let options = { transaction };
+			try {
+				if (hasSourceGet || audit) {
+					if (!existingRecord) return;
+					entry = primaryStore.getEntry(id, options);
+					if (!entry || !existingRecord) return;
+					if (entry.version !== existingVersion) return;
+				}
+				if (hasSourceGet) {
+					// if there is a resolution in-progress, abandon the eviction
+					if (primaryStore.hasLock(id, entry.version)) return;
+				}
+				// evictions never go in the audit log, so we can not record a deletion entry for the eviction
+				// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
+				// removed the entry entirely, but first cleanup indices
+				if (primaryStore.ifVersion) {
+					// lmdb
+					primaryStore.ifVersion?.(id, existingVersion, () => {
+						updateIndices(id, existingRecord, null);
+					});
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
+				} else {
+					updateIndices(id, existingRecord, null, options);
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
+				}
+			} finally {
+				return transaction.commit();
 			}
-			primaryStore.ifVersion?.(id, existingVersion, () => {
-				updateIndices(id, existingRecord, null);
-			});
-			// evictions never go in the audit log, so we can not record a deletion entry for the eviction
-			// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
-			// removed the entry entirely
-			return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
 		}
 		/**
 		 * This is intended to acquire a lock on a record from the whole cluster.
@@ -1951,9 +1963,10 @@ export function makeTable(options) {
 							context.lastModified = existingEntry.version;
 						TableResource._updateResource(this, existingEntry);
 					}
-					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return; // a newer record exists locally
-					updateIndices(id, existingRecord);
-					logger.trace?.(`Deleting record with id: ${id}, txn timestamp: ${new Date(txnTime).toISOString()}`);
+					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
+						return;
+					} // a newer record exists locally
+					updateIndices(id, existingRecord, null, transaction && { transaction });
 					if (audit || trackDeletes) {
 						updateRecord(
 							id,
@@ -3511,6 +3524,7 @@ export function makeTable(options) {
 			// determine what index values need to be removed and added
 			let valuesToAdd = getIndexedValues(value, indexNulls) as any[];
 			let valuesToRemove = getIndexedValues(existingValue, indexNulls) as any[];
+			let isLMDB = !!index.prefetch;
 			if (valuesToRemove?.length > 0) {
 				// put this in a conditional so we can do a faster version for new records
 				// determine the changes/diff from new values and old values
@@ -3527,18 +3541,18 @@ export function makeTable(options) {
 						})
 					: [];
 				valuesToRemove = Array.from(setToRemove);
-				if ((valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
+				if (isLMDB && (valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
 					// prefetch any values that have been removed or added
 					const valuesToPrefetch = valuesToRemove.concat(valuesToAdd).map((v) => ({ key: v, value: id }));
-					index.prefetch?.(valuesToPrefetch, noop);
+					index.prefetch(valuesToPrefetch, noop);
 				}
 				//if the update cleared out the attribute value we need to delete it from the index
 				for (let i = 0, l = valuesToRemove.length; i < l; i++) {
 					index.remove(valuesToRemove[i], id, options);
 				}
-			} else if (valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
+			} else if (isLMDB && valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
 				// no old values, just new
-				index.prefetch?.(
+				index.prefetch(
 					valuesToAdd.map((v) => ({ key: v, value: id })),
 					noop
 				);
@@ -4124,7 +4138,7 @@ export function makeTable(options) {
 								// don't do anything if the version has changed
 								return;
 							}
-							updateIndices(id, existingRecord, updatedRecord);
+							updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
 							if (updatedRecord) {
 								if (existingEntry) {
 									context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);