@harperfast/harper-pro 5.0.0-beta.6 → 5.0.0-beta.8

package/core/bin/cliOperations.js

@@ -10,7 +10,7 @@ const YAML = require('yaml');
 const { packageDirectory } = require('../components/packageComponent.ts');
 const { encode } = require('cbor-x');
 const { getHdbPid } = require('../utility/processManagement/processManagement.js');
-const { initConfig } = require('../config/configUtils.js');
+const { initConfig, getConfigPath } = require('../config/configUtils.js');
 
 const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' };
 
@@ -93,7 +93,7 @@ async function cliOperations(req) {
 			process.exit(1);
 		}
 
-		if (!fs.existsSync(envMgr.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET))) {
+		if (!fs.existsSync(getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET))) {
 			console.error('No domain socket found, unable to perform this operation');
 			process.exit(1);
 		}
@@ -102,7 +102,7 @@ async function cliOperations(req) {
 	try {
 		let options = target ?? {
 			protocol: 'http:',
-			socketPath: envMgr.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET),
+			socketPath: getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET),
 		};
 		options.method = 'POST';
 		options.headers = { 'Content-Type': 'application/json' };

package/core/bin/harper.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S node --disable-warning=ExperimentalWarning --experimental-vm-modules
+#!/usr/bin/env node
 'use strict';
 
 const fs = require('node:fs');

package/core/bin/run.js

@@ -266,7 +266,7 @@ function startupLog(portResolutions) {
 			? `, TCP: ${env.get(CONFIG_PARAMS.THREADS_DEBUG_PORT)}\n`
 			: '\n';
 	}
-	const logFilePath = path.join(env.get(CONFIG_PARAMS.LOGGING_ROOT), 'hdb.log');
+	const logFilePath = path.join(configUtils.getConfigPath(CONFIG_PARAMS.LOGGING_ROOT), 'hdb.log');
 	logMsg += `${pad('Logging:')}level: ${env.get(CONFIG_PARAMS.LOGGING_LEVEL)}, location: ${
 		logFilePath + (env.get(CONFIG_PARAMS.LOGGING_STDSTREAMS) ? ', stdout/err' : '')
 	}\n`;
@@ -292,7 +292,7 @@ function startupLog(portResolutions) {
 			? `enabled for ${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST)}`
 			: 'disabled'
 	}`;
-	logMsg += `, unix socket: ${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)}\n`;
+	logMsg += `, unix socket: ${configUtils.getConfigPath(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)}\n`;
 	if (env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT)) {
 		logMsg +=
 			pad('') +

package/core/components/Application.ts

@@ -1,5 +1,5 @@
 import { type Logger } from '../utility/logging/logger.ts';
-import { getConfigObj, getConfigValue } from '../config/configUtils.js';
+import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.js';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
 import logger from '../utility/logging/harper_logger.js';
 
@@ -402,7 +402,9 @@ export class Application {
 		this.payload = payload;
 		this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
 		this.install = install;
-		this.dirPath = join(getConfigValue(CONFIG_PARAMS.COMPONENTSROOT), name);
+		const componentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
+		if (!componentsRoot) throw new Error('componentsRoot is not configured');
+		this.dirPath = join(componentsRoot, name);
 		this.logger = logger.loggerWithTag(name);
 		this.packageManagerPrefix = getConfigValue(CONFIG_PARAMS.APPLICATIONS_PACKAGEMANAGERPREFIX);
 	}
@@ -464,7 +466,8 @@ export async function installApplications() {
 
 	const config = getConfigObj();
 
-	const componentsRootDirPath = getConfigValue(CONFIG_PARAMS.COMPONENTSROOT);
+	const componentsRootDirPath = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
+	if (!componentsRootDirPath) throw new Error('componentsRoot is not configured');
 
 	// Ensure component directory exists
 	await mkdir(componentsRootDirPath, { recursive: true });

package/core/components/ApplicationScope.ts

@@ -23,6 +23,7 @@ export class ApplicationScope {
 	dependencyContainment?: boolean; // option to set this from the scope
 	verifyPath?: string;
 	config: any;
+	moduleCache: any; // used by the loader to retain a cache of modules, type is an internal detail of the loader
 	constructor(name: string, resources: Resources, server: Server, isInternal = false, verifyPath?: string) {
 		this.logger = forComponent(name, !isInternal);
 

package/core/components/componentLoader.ts

@@ -27,7 +27,7 @@ import { getHdbBasePath } from '../utility/environment/environmentManager.js';
 import * as operationsServer from '../server/operationsServer.ts';
 import * as auth from '../security/auth.ts';
 import * as mqtt from '../server/mqtt.ts';
-import { getConfigObj, resolvePath } from '../config/configUtils.js';
+import { getConfigObj, getConfigPath } from '../config/configUtils.js';
 import { createReuseportFd } from '../server/serverHelpers/Request.ts';
 import { ErrorResource } from '../resources/ErrorResource.ts';
 import { Scope } from './Scope.ts';
@@ -39,8 +39,9 @@ import { lifecycle as componentLifecycle } from './status/index.ts';
 import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts';
 import { PluginModule } from './PluginModule.ts';
 import { getEnvBuiltInComponents } from './Application.ts';
+import { pathToFileURL } from 'node:url';
 
-const CF_ROUTES_DIR = resolvePath(env.get(CONFIG_PARAMS.COMPONENTSROOT));
+const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
 let loadedComponents = new Map<any, any>();
 let watchesSetup;
 let resources;
@@ -147,20 +148,28 @@ function symlinkHarperModule(componentDirectory: string) {
 					mkdirSync(nodeModulesDir);
 				}
 
-				// validate harperdb module
+				// validate harper module
 				const harperModule = join(nodeModulesDir, 'harper');
 				if (existsSync(harperModule)) {
-					if (realpathSync(harperModule) === realpathSync(PACKAGE_ROOT)) {
-						// if it exists and correctly linked, resolve
-						return resolve();
+					if (realpathSync(harperModule) !== realpathSync(PACKAGE_ROOT)) {
+						// if it exists but is incorrectly linked, fix it
+						rmSync(harperModule, { recursive: true, force: true });
+						// create link to harper module
+						symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
 					}
-
-					// Otherwise remove it then link
-					rmSync(harperModule, { recursive: true, force: true });
+				} else {
+					// create link to harper module
+					symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
+				}
+				// if there is a harperdb module, fix that too
+				const harperdbModule = join(nodeModulesDir, 'harperdb');
+				if (existsSync(harperdbModule) && realpathSync(harperdbModule) !== realpathSync(PACKAGE_ROOT)) {
+					// if it exists but is incorrectly linked, fix it
+					rmSync(harperdbModule, { recursive: true, force: true });
+					// create link to harper module
+					symlinkSync(PACKAGE_ROOT, harperdbModule, 'dir');
 				}
 
-				// create link to harperdb module
-				symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
 				resolve();
 			} finally {
 				// finally release the lock
@@ -352,7 +361,9 @@ export async function loadComponent(
 					const plugin = TRUSTED_RESOURCE_PLUGINS[componentName];
 					extensionModule =
 						typeof plugin === 'string'
-							? await import(plugin.startsWith('@/') ? join(PACKAGE_ROOT, plugin.slice(1)) : plugin)
+							? await import(
+									plugin.startsWith('@/') ? pathToFileURL(join(PACKAGE_ROOT, plugin.slice(1))).toString() : plugin
+								)
 							: plugin;
 				}
 

package/core/components/operations.js

@@ -31,7 +31,7 @@ function customFunctionsStatus() {
 	try {
 		response = {
 			port: env.get(hdbTerms.CONFIG_PARAMS.HTTP_PORT),
-			directory: env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT),
+			directory: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT),
 			is_enabled: true,
 		};
 	} catch (err) {
@@ -54,7 +54,7 @@ function customFunctionsStatus() {
 function getCustomFunctions() {
 	log.trace(`getting custom api endpoints`);
 	let response = {};
-	const dir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const dir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 
 	try {
 		const projectFolders = fg.sync(normalize(`${dir}/*`), { onlyDirectories: true });
@@ -103,7 +103,7 @@ function getCustomFunction(req) {
 	}
 
 	log.trace(`getting custom api endpoint file content`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file } = req;
 	const fileLocation = path.join(cfDir, project, type, file + '.js');
 
@@ -141,7 +141,7 @@ async function setCustomFunction(req) {
 	}
 
 	log.trace(`setting custom function file content`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file, function_content } = req;
 
 	try {
@@ -181,7 +181,7 @@ async function dropCustomFunction(req) {
 	}
 
 	log.trace(`dropping custom function file`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file } = req;
 
 	try {
@@ -216,7 +216,7 @@ async function addComponent(req) {
 	}
 
 	log.trace(`adding component`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, install_command, install_timeout } = req;
 
 	const template = req.template || 'https://github.com/harperdb/application-template';
@@ -264,7 +264,7 @@ async function dropCustomFunctionProject(req) {
 	}
 
 	log.trace(`dropping custom function project`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project } = req;
 
 	let apps = env.get(hdbTerms.CONFIG_PARAMS.APPS);
@@ -318,7 +318,7 @@ async function packageComponent(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project } = req;
 	log.trace(`packaging component`, project);
 
@@ -507,8 +507,8 @@ async function getComponents() {
 		}
 	};
 
-	const results = await walkDir(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), {
-		name: env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
+	const results = await walkDir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), {
+		name: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
 		entries: [],
 	});
 	for (let entry of results.entries) {
@@ -557,7 +557,7 @@ async function getComponentFile(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
-	const compRoot = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const compRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const options = req.encoding ? { encoding: req.encoding } : { encoding: 'utf8' };
 
 	try {
@@ -588,7 +588,7 @@ async function setComponentFile(req) {
 	}
 
 	const options = req.encoding ? { encoding: req.encoding } : { encoding: 'utf8' };
-	const pathToComp = path.join(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project, req.file);
+	const pathToComp = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project, req.file);
 	if (req.payload !== undefined) {
 		await fs.ensureFile(pathToComp);
 		await fs.outputFile(pathToComp, req.payload, options);
@@ -613,7 +613,7 @@ async function dropComponent(req) {
 
 	const { project, file } = req;
 	const projectPath = req.file ? path.join(project, file) : project;
-	const pathToComponent = path.join(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
+	const pathToComponent = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
 
 	const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project);
 	if (await fs.pathExists(componentSymlink)) {

package/core/components/operationsValidation.js

@@ -4,9 +4,9 @@ const Joi = require('joi');
 const fs = require('fs-extra');
 const path = require('path');
 const validator = require('../validation/validationWrapper.js');
-const envMangr = require('../utility/environment/environmentManager.js');
 const hdbTerms = require('../utility/hdbTerms.ts');
 const hdbLogger = require('../utility/logging/harper_logger.js');
+const configUtils = require('../config/configUtils.js');
 const { hdbErrors } = require('../utility/errors/hdbError.js');
 const { HDB_ERROR_MSGS } = hdbErrors;
 
@@ -34,7 +34,7 @@ module.exports = {
  */
 function checkProjectExists(checkExists, project, helpers) {
 	try {
-		const cfDir = envMangr.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+		const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 		const projectDir = path.join(cfDir, project);
 
 		if (!fs.existsSync(projectDir)) {
@@ -71,7 +71,7 @@ function checkFilePath(path, helpers) {
  */
 function checkFileExists(project, type, file, helpers) {
 	try {
-		const cfDir = envMangr.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+		const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 		const filePath = path.join(cfDir, project, type, file + '.js');
 		if (!fs.existsSync(filePath)) {
 			return helpers.message(HDB_ERROR_MSGS.NO_FILE);

package/core/config/configUtils.js

@@ -57,6 +57,7 @@ exports.deleteConfigFromFile = deleteConfigFromFile;
 exports.getConfigObj = getConfigObj;
 exports.resolvePath = resolvePath;
 exports.getFlatConfigObj = getFlatConfigObj;
+exports.getConfigPath = getConfigPath;
 
 function resolvePath(relativePath) {
 	if (relativePath?.startsWith('~/')) {
@@ -70,7 +71,23 @@ function resolvePath(relativePath) {
 		return relativePath;
 	}
 }
-
+/**
+ * Get a config value and resolve it as a path relative to rootPath.
+ * Use this for any config param that represents a file/directory path.
+ * @param param
+ */
+function getConfigPath(param) {
+	const env = require('../utility/environment/environmentManager.js');
+	const value = env.get(param);
+	if (!value || typeof value !== 'string') return value;
+	if (value.startsWith('~/')) {
+		return path.join(hdbUtils.getHomeDir(), value.slice(1));
+	}
+	if (path.isAbsolute(value)) return value;
+	const rootPath = env.getHdbBasePath();
+	if (!rootPath) return value;
+	return path.resolve(rootPath, value);
+}
 /**
  * Builds the Harper config file using user inputs and default values from defaultConfig.yaml
  * @param args - any args that the user provided.
@@ -332,15 +349,14 @@ function initConfig(force = false) {
  * @param configFilePath
  */
 function checkForUpdatedConfig(configDoc, configFilePath) {
-	const rootPath = configDoc.getIn(['rootPath']);
 	let updateFile = false;
 	if (!configDoc.hasIn(['storage', 'path'])) {
-		configDoc.setIn(['storage', 'path'], path.join(rootPath, 'database'));
+		configDoc.setIn(['storage', 'path'], 'database');
 		updateFile = true;
 	}
 
 	if (!configDoc.hasIn(['logging', 'rotation', 'path'])) {
-		configDoc.setIn(['logging', 'rotation', 'path'], path.join(rootPath, 'log'));
+		configDoc.setIn(['logging', 'rotation', 'path'], 'log');
 		updateFile = true;
 	}
 

package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js

@@ -7,6 +7,7 @@ const path = require('path');
 const minimist = require('minimist');
 const fs = require('fs-extra');
 const _ = require('lodash');
+const { getConfigPath } = require('../../../../config/configUtils.js');
 env.initSync();
 
 const { CONFIG_PARAMS, DATABASES_PARAM_CONFIG, SYSTEM_SCHEMA_NAME } = hdbTerms;
@@ -25,7 +26,7 @@ function getBaseSchemaPath() {
 
 	if (env.getHdbBasePath() !== undefined) {
 		BASE_SCHEMA_PATH =
-			env.get(CONFIG_PARAMS.STORAGE_PATH) || path.join(env.getHdbBasePath(), hdbTerms.DATABASES_DIR_NAME);
+			getConfigPath(CONFIG_PARAMS.STORAGE_PATH) || path.join(env.getHdbBasePath(), hdbTerms.DATABASES_DIR_NAME);
 		return BASE_SCHEMA_PATH;
 	}
 }
@@ -52,7 +53,7 @@ function getTransactionAuditStoreBasePath() {
 
 	if (env.getHdbBasePath() !== undefined) {
 		TRANSACTION_STORE_PATH =
-			env.get(hdbTerms.CONFIG_PARAMS.STORAGE_AUDIT_PATH) ||
+			getConfigPath(hdbTerms.CONFIG_PARAMS.STORAGE_AUDIT_PATH) ||
 			path.join(env.getHdbBasePath(), hdbTerms.TRANSACTIONS_DIR_NAME);
 		return TRANSACTION_STORE_PATH;
 	}

package/core/resources/DatabaseTransaction.ts

@@ -35,6 +35,7 @@ export type CommitOptions = {
 	timestamp?: number;
 	retries?: number;
 	flush?: boolean;
+	transaction?: RocksTransaction;
 };
 
 type ReadTransaction = (LMDBTransaction | RocksTransaction) & {
@@ -116,8 +117,13 @@ export class DatabaseTransaction implements Transaction {
 		if (!this.transaction) return;
 		if (--this.readTxnsUsed === 0) {
 			trackedTxns.delete(this);
-			this.transaction?.abort();
-			this.transaction = null;
+			if (this.open === TRANSACTION_STATE.LINGERING) {
+				// if we have lingering writes, we have to call commit to finish them
+				this.commit();
+			} else {
+				this.transaction?.abort();
+				this.transaction = null;
+			}
 		}
 	}
 
@@ -139,9 +145,6 @@ export class DatabaseTransaction implements Transaction {
 	}
 
 	addWrite(operation: TransactionWrite) {
-		if (this.open === TRANSACTION_STATE.CLOSED) {
-			throw new Error('Can not use a transaction that is no longer open');
-		}
 		this.writes.push(operation);
 		if (!operation.deferSave) {
 			// Setting saved to false means to defer saving
@@ -150,61 +153,94 @@ export class DatabaseTransaction implements Transaction {
 		return operation;
 	}
 
-	save(operation: TransactionWrite, reloadEntry = false) {
+	save(operation: TransactionWrite, transaction?: RocksTransaction, reloadEntry = false) {
 		let txnTime = this.timestamp;
-		if (!this.transaction) {
-			this.transaction = new RocksTransaction(this.db.store as RocksStore);
+		transaction ??= this.transaction;
+		let immediateCommit = false;
+		if (!transaction) {
+			transaction = new RocksTransaction(this.db.store as RocksStore);
+			if (this.open === TRANSACTION_STATE.OPEN) {
+				this.transaction = transaction;
+			} else {
+				// if it is closed, we have to immediately commit, using our immediate transaction
+				immediateCommit = true;
+			}
 			if (txnTime) {
-				this.transaction.setTimestamp(txnTime);
+				transaction.setTimestamp(txnTime);
 			}
 		}
 		if (this.retries > 0) {
 			// this is marks the rocks transaction as a retry so we don't write the transaction log again
-			this.transaction.isRetry = true;
+			transaction.isRetry = true;
 		}
-		if (!txnTime) txnTime = this.timestamp = this.transaction.getTimestamp();
+		if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
 		if (reloadEntry || operation.entry === undefined) {
-			operation.entry = operation.store.getEntry(operation.key, { transaction: this.transaction });
+			operation.entry = operation.store.getEntry(operation.key, { transaction });
+		}
+		if (!operation.saved) {
+			operation.saved = true;
+			// immediately execute in this transaction
+			if (operation.validate?.(txnTime) === false) {
+				operation.commit = () => {}; // noop if we try again
+				return;
+			}
+			let result: Promise<void> = operation.before?.() as Promise<void>;
+			if (result?.then) this.completions.push(result);
+			result = operation.beforeIntermediate?.() as Promise<void>;
+			if (result?.then) this.completions.push(result);
+		}
+		operation.commit(txnTime, operation.entry, this.retries > 0, transaction);
+		if (immediateCommit) {
+			return this.commit({ transaction }); // immediately commit if the harper transaction is closed
 		}
-		operation.saved = true;
-		// immediately execute in this transaction
-		if (operation.validate?.(txnTime) === false) return;
-		let result: Promise<void> = operation.before?.() as Promise<void>;
-		if (result?.then) this.completions.push(result);
-		result = operation.beforeIntermediate?.() as Promise<void>;
-		if (result?.then) this.completions.push(result);
-		operation.commit(txnTime, operation.entry, this.retries > 0, this.transaction);
 	}
 
 	/**
 	 * Resolves with information on the timestamp and success of the commit
 	 */
 	commit(options: CommitOptions = {}): MaybePromise<CommitResolution> {
+		let transaction = options.transaction ?? this.transaction; // we need to preserve this transaction as we might to resurrect it if we have to retry
 		for (let i = 0; i < this.writes.length; i++) {
 			let operation = this.writes[i];
 			if (this.retries === 0 && operation.saved) continue;
-			this.save(operation, i < this.validated);
+			this.save(operation, transaction, i < this.validated);
 		}
 		this.validated = this.writes.length;
-		return when(this.completions.length > 0 ? Promise.all(this.completions) : null, () => {
+		const completions = this.completions;
+		if (completions.length > 0) this.completions = []; // reset
+		return when(completions.length > 0 ? Promise.all(completions) : null, () => {
+			if (this.writes.length > this.validated) {
+				// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
+				return this.commit(options);
+			}
+			this.open = TRANSACTION_STATE.CLOSED;
 			let commitResolution: MaybePromise<void>;
 			if (--this.readTxnsUsed > 0) {
 				// we still have outstanding iterators using the transaction, we can't just commit/abort it, we will still
 				// need to use it
+				if (this.writes.length > 0) {
+					// if there are outstanding writes, we have to call commit later to finish them
+					this.open = TRANSACTION_STATE.LINGERING;
+					/* TODO: This is not really the intended behavior though, we want to immediately commit writes, but continue to use
+					 * the transaction, as there is likely existing references to the transaction in other parts of the codebase,
+					 * particularly in the query iterator */
+				}
+				/*
 				commitResolution =
 					this.writes.length > 0
-						? this.transaction?.commit({ renewAfterCommit: true /* Try to use RocksDB's CommitAndTryCreateSnapshot */ })
-						: // don't abort, we still have outstanding reads to complete
+						? transaction?.commit({ renewAfterCommit: true }) // Try to use RocksDB's CommitAndTryCreateSnapshot
+			: // don't abort, we still have outstanding reads to complete
 							null;
+				*/
 			} else {
 				// no more reads need to be performed, just commit/abort based if there are any writes
 				trackedTxns.delete(this);
-				if (this.transaction) {
+				this.transaction = null; // clear transaction so any further operations operate immediately
+				if (transaction) {
 					if (this.writes.length > 0) {
-						commitResolution = this.transaction.commit();
+						commitResolution = transaction.commit();
 					} else {
-						commitResolution = this.transaction.abort();
-						this.transaction = null; // immediately clear transaction, no need to wait
+						commitResolution = transaction.abort();
 					}
 				}
 			}
@@ -220,8 +256,7 @@ export class DatabaseTransaction implements Transaction {
 				const completions = [];
 				return commitResolution.then(
 					() => {
-						this.transaction.onCommit?.();
-						this.transaction = null; // the native transaction is done (reset if needed)
+						transaction.onCommit?.();
 						if (this.next) {
 							completions.push(this.next.commit(options));
 						}
@@ -260,7 +295,7 @@ export class DatabaseTransaction implements Transaction {
 							// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
 							// for future transactions
 							this.retries++;
-							return this.commit(options); // try again
+							return this.commit({ transaction }); // try again
 						} else throw error;
 					}
 				);
@@ -318,7 +353,7 @@ export class ImmediateTransaction extends DatabaseTransaction {
 	save(transaction: ImmediateTransaction) {
 		if (this.isCommitting) {
 			// if we are in the commit, do the save and force a reload so we get a read within the transaction
-			super.save(transaction, true);
+			super.save(transaction, null, true);
 		} else {
 			this.isCommitting = true;
 			return when(this.commit(), () => {

package/core/resources/Resource.ts

@@ -683,15 +683,26 @@ function transactional(
 		}
 	}
 }
-function missingMethod(resource, method) {
-	const error = new ClientError(`The ${resource.constructor.name} does not have a ${method} method implemented`, 405);
-	error.allow = [];
+const KNOWN_METHODS = ['get', 'head', 'put', 'post', 'delete', 'patch', 'query', 'move', 'copy'];
+type ClientErrorWithMethods = ClientError & { allow: string[]; method: string };
+export function missingMethod(resource: any, method: string) {
+	const error = new ClientError(
+		`The ${resource.constructor.name} does not have a ${method} method implemented`,
+		405
+	) as ClientErrorWithMethods;
+	error.allow = allowedMethods(resource);
 	error.method = method;
-	for (const method of ['get', 'put', 'post', 'delete', 'query', 'move', 'copy']) {
-		if (typeof resource[method] === 'function') error.allow.push(method);
-	}
 	throw error;
 }
+
+export function allowedMethods(resource: any) {
+	const allow = [];
+	for (const method of KNOWN_METHODS) {
+		if (typeof resource[method] === 'function') allow.push(method);
+	}
+	return allow;
+}
+
 /**
  * This is responsible for handling a select query parameter/call that selects specific
  * properties from the returned record(s).

package/core/resources/RocksTransactionLogStore.ts

@@ -3,6 +3,7 @@ import { ExtendedIterable } from '@harperfast/extended-iterable';
 import { Decoder, readAuditEntry, ENTRY_DATAVIEW, AuditRecord, createAuditEntry } from './auditStore.ts';
 import { isMainThread } from 'node:worker_threads';
 import { EventEmitter } from 'node:events';
+import { asBinary } from 'lmdb';
 
 if (!process.env.HARPER_NO_FLUSH_ON_EXIT && isMainThread) {
 	// we want to be able to test log replay
@@ -81,7 +82,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 
 	putSync(suggestedKey: any, value: any, options: any) {
 		if (typeof suggestedKey === 'symbol') {
-			this.rootStore.putSync(suggestedKey, value, options);
+			this.rootStore.putSync(suggestedKey, asBinary(value), options);
 		} else {
 			this.put(suggestedKey, value, options);
 		}
@@ -107,6 +108,12 @@ export class RocksTransactionLogStore extends EventEmitter {
 			return this.rootStore.getSync(key);
 		}
 	}
+	getBinary(key: any) {
+		if (typeof key === 'number') {
+			throw new Error('Unsupported binary access by number');
+		}
+		return this.rootStore.getBinarySync(key);
+	}
 	getEntry() {
 		throw new Error('Not implemented');
 	}