@harperfast/harper-pro 5.0.1 → 5.0.3

package/analytics/profile.ts

@@ -6,6 +6,10 @@ import { recordAction } from '../core/resources/analytics/write.ts';
 import { getHdbBasePath } from '../core/utility/environment/environmentManager.js';
 import { PACKAGE_ROOT } from '../core/utility/packageUtils.js';
 import { realpathSync, readFileSync, readdirSync } from 'node:fs';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
 import { time as timeProfiler } from '@datadog/pprof';
 import { getWorkerIndex } from '../core/server/threads/manageThreads.js';
 import * as log from '../core/utility/logging/harper_logger.js';
@@ -42,6 +46,7 @@ export function handleApplication({ options }: Scope) {
 	}, 1000); // wait for everything to load before we start the profiler
 }
 let lastChildCpuTime = 0;
+let gpuAvailable = true;
 
 export async function captureProfile(delayToNextCapture = (capturePeriod ?? 60) * 1000): Promise<void> {
 	clearTimeout(profilerTimer);
@@ -58,6 +63,8 @@ export async function captureProfile(delayToNextCapture = (capturePeriod ?? 60)
 	const samplesByLocationId = new Map<number, number>();
 	let totalUserCount = 0;
 	let totalHarperCount = 0;
+	// Start GPU measurement early so it runs in parallel with CPU profiling work
+	const gpuPromise = getWorkerIndex() === 0 && gpuAvailable ? getGpuUtilization() : null;
 	try {
 		const profile = timeProfiler.stop(true);
 		const strings = profile.stringTable.strings;
@@ -89,6 +96,11 @@ export async function captureProfile(delayToNextCapture = (capturePeriod ?? 60)
 					recordAction(childCpuTimeInInterval, 'cpu-usage', 'user', 'child-processes');
 				lastChildCpuTime = childCpuTime;
 			}
+			// Record GPU utilization for this process and child processes
+			const gpuSeconds = await gpuPromise;
+			if (gpuSeconds !== null) {
+				recordAction(gpuSeconds, 'gpu-usage', 'user');
+			}
 		}
 	} catch (error) {
 		log.error?.('analytics profiler error:', error);
@@ -177,6 +189,41 @@ function getChildProcessCpuTime(): number | null {
 	}
 }
 
+/**
+ * Get the total SM (shader/streaming multiprocessor) utilization percentage across all GPUs
+ * for this process and all descendant processes.
+ * Uses nvidia-smi pmon for per-process GPU utilization.
+ * Only works on Linux with NVIDIA GPUs. Returns null if unavailable.
+ */
+async function getGpuUtilization(): Promise<number | null> {
+	try {
+		const currentPid = process.pid;
+		const descendants = findAllDescendants(currentPid);
+		const pidsToMonitor = new Set([currentPid, ...descendants]);
+
+		const { stdout } = await execFileAsync('nvidia-smi', ['pmon', '-c', '1', '-s', 'u']);
+
+		let totalSmPercent = 0;
+		for (const line of stdout.split('\n')) {
+			if (line.startsWith('#') || !line.trim()) continue;
+			const parts = line.trim().split(/\s+/);
+			// pmon -s u format: gpu pid type fb sm enc dec jpg ofa command
+			if (parts.length < 5) continue;
+			const pid = parseInt(parts[1], 10);
+			if (isNaN(pid) || !pidsToMonitor.has(pid)) continue;
+			const sm = parseInt(parts[4], 10); // SM utilization %
+			if (!isNaN(sm)) totalSmPercent += sm;
+		}
+
+		// Convert SM utilization % to GPU-seconds over the capture period
+		// e.g. 50% utilization over 60s = 30 GPU-seconds
+		return (totalSmPercent / 100) * (capturePeriod / 1000);
+	} catch {
+		gpuAvailable = false;
+		return null;
+	}
+}
+
 /**
  * Recursively find all descendant PIDs of the given parent PID.
  */

package/core/bin/cliOperations.js

@@ -63,7 +63,7 @@ function buildRequest() {
  */
 async function cliOperations(req) {
 	if (!req.target) {
-		req.target = process.env.CLI_TARGET;
+		req.target = process.env.HARPER_CLI_TARGET || process.env.CLI_TARGET;
 	}
 	let target;
 	if (req.target) {
@@ -73,20 +73,22 @@ async function cliOperations(req) {
 			try {
 				target = new URL(`https://${req.target}:9925`);
 			} catch {
-				throw error; // throw the original error
+				throw error;
 			}
 		}
 		target = {
 			protocol: target.protocol,
 			hostname: target.hostname,
 			port: target.port,
-			username: req.username || target.username || process.env.CLI_TARGET_USERNAME,
-			password: req.password || target.password || process.env.CLI_TARGET_PASSWORD,
+			username: req.username || target.username || process.env.HARPER_CLI_USERNAME || process.env.CLI_TARGET_USERNAME,
+			password: req.password || target.password || process.env.HARPER_CLI_PASSWORD || process.env.CLI_TARGET_PASSWORD,
 			rejectUnauthorized: req.rejectUnauthorized,
 		};
+		console.error(`Connecting to ${target.protocol}//${target.hostname}:${target.port}`);
 	} else {
 		// if we aren't doing a targeted operation (like deploy), we initialize the config and verify that local harper
 		// is running and that we can communicate with it.
+		console.error('Connecting to local Harper Pro instance');
 		initConfig();
 		if (!getHdbPid()) {
 			console.error('Harper Pro must be running to perform this operation');

package/core/bin/copyDb.ts

@@ -2,6 +2,7 @@ import { getDatabases, getDefaultCompression, resetDatabases } from '../resource
 import { open } from 'lmdb';
 import { join } from 'path';
 import { move, remove } from 'fs-extra';
+import { existsSync, mkdirSync } from 'node:fs';
 import { get } from '../utility/environment/environmentManager.js';
 import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
 import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.js';
@@ -11,6 +12,8 @@ import { AUDIT_STORE_OPTIONS } from '../resources/auditStore.ts';
 import { describeSchema } from '../dataLayer/schemaDescribe.js';
 import { updateConfigValue } from '../config/configUtils.js';
 import * as hdbLogger from '../utility/logging/harper_logger.js';
+import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
+import { RocksIndexStore } from '../resources/RocksIndexStore.ts';
 
 export async function compactOnStart() {
 	hdbLogger.notify('Running compact on start');
@@ -278,3 +281,208 @@ export async function copyDb(sourceDatabase: string, targetDatabasePath: string)
 		targetEnv.close();
 	}
 }
+
+function openRocksDb(path: string, options: RocksDatabaseOptions & { dupSort?: boolean } = {}) {
+	options.disableWAL ??= false;
+	if (!existsSync(path)) {
+		mkdirSync(path, { recursive: true });
+	}
+	let db;
+	if (options.dupSort) {
+		db = RocksDatabase.open(new RocksIndexStore(path, options));
+	} else {
+		db = RocksDatabase.open(path, options);
+		db.encoder.name = options.name;
+	}
+	return db;
+}
+
+export async function migrateOnStart() {
+	hdbLogger.notify('Running migrate on start (LMDB to RocksDB)');
+	console.log('Running migrate on start (LMDB to RocksDB)');
+
+	const rootPath = get(CONFIG_PARAMS.ROOTPATH);
+	const databases = getDatabases();
+
+	updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false);
+
+	try {
+		for (const databaseName in databases) {
+			if (databaseName === 'system') continue;
+			if (databaseName.endsWith('-copy')) continue;
+			let rootStore;
+			for (const tableName in databases[databaseName]) {
+				const table = databases[databaseName][tableName];
+				table.primaryStore.put = noop;
+				table.primaryStore.remove = noop;
+				for (const attributeName in table.indices) {
+					const index = table.indices[attributeName];
+					index.put = noop;
+					index.remove = noop;
+				}
+				if (table.auditStore) {
+					table.auditStore.put = noop;
+					table.auditStore.remove = noop;
+				}
+				rootStore = table.primaryStore.rootStore;
+			}
+			if (!rootStore) {
+				console.log("Couldn't find any tables in database", databaseName);
+				continue;
+			}
+			if (rootStore instanceof RocksDatabase) {
+				console.log('Database', databaseName, 'is already RocksDB, skipping');
+				continue;
+			}
+
+			const targetPath = join(rootPath, DATABASES_DIR_NAME, databaseName);
+			const lmdbPath = rootStore.path;
+			const backupDest = join(rootPath, 'backup', databaseName + '.mdb');
+
+			console.log('Migrating', databaseName, 'from LMDB to RocksDB at', targetPath);
+
+			await copyDbToRocks(rootStore, databaseName, targetPath);
+
+			// Back up the original LMDB file
+			console.log('Backing up LMDB', databaseName, 'to', backupDest);
+			try {
+				await move(lmdbPath, backupDest, { overwrite: true });
+			} catch (error) {
+				console.log('Error moving database', lmdbPath, 'to', backupDest, error);
+			}
+			// Remove the lock file
+			try {
+				await remove(lmdbPath + '-lock');
+			} catch {
+				// lock file may not exist
+			}
+		}
+
+		try {
+			resetDatabases();
+		} catch (err) {
+			hdbLogger.error('Error resetting databases after migration', err);
+			console.error('Error resetting databases after migration', err);
+		}
+	} catch (err) {
+		hdbLogger.error('Error migrating database', err);
+		console.error('Error migrating database', err);
+		throw err;
+	}
+}
+
+async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath: string) {
+	console.log(`Migrating database ${sourceDatabase} to RocksDB at ${targetPath}`);
+	const sourceDbisDb = sourceRootStore.dbisDb;
+
+	const targetRootStore = openRocksDb(targetPath, { disableWAL: false });
+	const targetDbisDb = openRocksDb(targetPath, {
+		disableWAL: false,
+		name: INTERNAL_DBIS_NAME,
+	});
+
+	let written;
+	let outstandingWrites = 0;
+	const transaction = sourceDbisDb.useReadTransaction();
+	try {
+		for (const { key, value: attribute } of sourceDbisDb.getRange({ transaction })) {
+			const isPrimary = attribute.isPrimaryKey;
+			targetDbisDb.put(key, attribute);
+			if (!(isPrimary || attribute.indexed)) continue;
+
+			// Open source LMDB dbi with default encoding so values are decoded
+			const dbiInit = new OpenDBIObject(!isPrimary, isPrimary);
+			const sourceDbi = sourceRootStore.openDB(key, dbiInit);
+
+			let targetDbi;
+			if (!isPrimary) {
+				targetDbi = openRocksDb(targetPath, { dupSort: true, name: key });
+			} else {
+				targetDbi = openRocksDb(targetPath, { name: key });
+			}
+
+			console.log('migrating', key, 'from', sourceDatabase, 'to RocksDB');
+			await copyDbiToRocks(sourceDbi, targetDbi, isPrimary, transaction);
+		}
+
+		// Note: audit store is not migrated because LMDB and RocksDB use fundamentally different
+		// audit store formats (LMDB uses a custom binary encoding in a regular DB, RocksDB uses TransactionLog).
+		// A new audit store will be created automatically when the RocksDB database is opened.
+
+		await written;
+		console.log('migrated database ' + sourceDatabase + ' to RocksDB');
+	} finally {
+		transaction.done();
+		targetRootStore.close();
+	}
+
+	async function copyDbiToRocks(sourceDbi, targetDbi, isPrimary, transaction) {
+		let recordsCopied = 0;
+		let skippedRecord = 0;
+		let retries = 1000000;
+		let start = null;
+		while (retries-- > 0) {
+			try {
+				if (isPrimary) {
+					for (const { key, value, version } of sourceDbi.getRange({ start, transaction, versions: true })) {
+						try {
+							start = key;
+							if (value == null) {
+								skippedRecord++;
+								continue;
+							}
+							written = targetDbi.put(key, value, version);
+							recordsCopied++;
+							if (transaction.openTimer) transaction.openTimer = 0;
+							if (outstandingWrites++ > 5000) {
+								await written;
+								console.log('migrated', recordsCopied, 'entries, skipped', skippedRecord, 'delete records');
+								outstandingWrites = 0;
+							}
+						} catch (error) {
+							console.error(
+								'Error migrating record',
+								typeof key === 'symbol' ? 'symbol' : key,
+								'from',
+								sourceDatabase,
+								error
+							);
+						}
+					}
+				} else {
+					for (const { key, value } of sourceDbi.getRange({ start, transaction })) {
+						try {
+							start = key;
+							written = targetDbi.put(key, value);
+							recordsCopied++;
+							if (transaction.openTimer) transaction.openTimer = 0;
+							if (outstandingWrites++ > 5000) {
+								await written;
+								console.log('migrated', recordsCopied, 'index entries');
+								outstandingWrites = 0;
+							}
+						} catch (error) {
+							console.error(
+								'Error migrating index record',
+								typeof key === 'symbol' ? 'symbol' : key,
+								'from',
+								sourceDatabase,
+								error
+							);
+						}
+					}
+				}
+				console.log('finish migrating, copied', recordsCopied, 'entries, skipped', skippedRecord, 'delete records');
+				return;
+			} catch {
+				if (typeof start === 'string') {
+					if (start === 'z') {
+						return console.error('Reached end of dbi', start, 'for', sourceDatabase);
+					}
+					start = start.slice(0, -2) + 'z';
+				} else if (typeof start === 'number') start++;
+				else return console.error('Unknown key type', start, 'for', sourceDatabase);
+			}
+		}
+	}
+}

package/core/bin/restart.js

@@ -61,12 +61,6 @@ async function restart(req) {
 
 		if (envMgr.get(hdbTerms.CONFIG_PARAMS.STORAGE_COMPACTONSTART)) await compactOnStart();
 
-		if (process.env.HARPER_EXIT_ON_RESTART) {
-			// use this to exit the process so that it will be restarted by the
-			// PM/container/orchestrator.
-			hdbLogger.warn('Exiting Harper Pro process to trigger a container restart');
-			process.exit(0);
-		}
 		setTimeout(async () => {
 			// It seems like you should just be able to start the other process and kill this process and everything should
 			// be cleaned up, however that doesn't work for some reason; the socket listening fds somehow get transferred to the
@@ -79,9 +73,16 @@ async function restart(req) {
 			// remove pid file so it doesn't trip up the launch
 			await unlinkSync(path.join(envMgr.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), hdbTerms.HDB_PID_FILE), `${process.pid}`);
 			hdbLogger.debug('Starting new process...');
+			if (process.env.HARPER_EXIT_ON_RESTART) {
+				// use this to exit the process so that it will be restarted by the
+				// PM/container/orchestrator.
+				hdbLogger.warn('Exiting Harper Pro process to trigger a container restart');
+				process.exit(0);
+			}
 			// now launch the new process and exit this process
 			require('./run.js').launch(true);
-		}, 50); // can't await this because it is going to do an exit()
+		}, 50); // can't await this because it is going to do an exit(), but wait for 50ms so we give the HTTP thread a
+		// chance to return a response
 	} else {
 		// Post msg to main parent thread requesting it restart (so the main thread can process.exit())
 		parentPort.postMessage({

package/core/bin/run.js

@@ -19,7 +19,7 @@ const installation = require('../utility/installation.ts');
 const configUtils = require('../config/configUtils.js');
 const assignCMDENVVariables = require('../utility/assignCmdEnvVariables.js');
 const upgrade = require('./upgrade.js');
-const { compactOnStart } = require('./copyDb.ts');
+const { compactOnStart, migrateOnStart } = require('./copyDb.ts');
 const minimist = require('minimist');
 const keys = require('../security/keys.js');
 const { startHTTPThreads } = require('../server/threads/socketRouter.ts');
@@ -192,6 +192,7 @@ async function main(calledByInstall = false) {
 		await initialize(calledByInstall, true);
 
 		if (env.get(terms.CONFIG_PARAMS.STORAGE_COMPACTONSTART)) await compactOnStart();
+		if (env.get(terms.CONFIG_PARAMS.STORAGE_MIGRATEONSTART)) await migrateOnStart();
 
 		const isScripted = process.env.IS_SCRIPTED_SERVICE && !cmdArgs.service;
 

package/core/components/Application.ts

@@ -172,15 +172,30 @@ export async function extractApplication(application: Application) {
 				}
 			}
 		} else {
-			// Given a package, resolve using `npm pack` (downloads the package as a tarball and writes the path to stdout)
-			const {
-				stdout: tarballFilePath,
-				code,
-				stderr,
-			} = await nonInteractiveSpawn(application.name, 'npm', ['pack', application.packageIdentifier], parentDirPath);
-			if (code !== 0) throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
-			tarballPath = join(parentDirPath, tarballFilePath.trim());
-			// Create a Readable from the tarball
+			// `npm pack --json` writes a JSON array describing the packed tarball(s).
+			const { stdout, code, stderr } = await nonInteractiveSpawn(
+				application.name,
+				'npm',
+				['pack', '--json', application.packageIdentifier],
+				parentDirPath
+			);
+			if (code !== 0) {
+				throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
+			}
+
+			let packResult: Array<{ filename: string }>;
+			try {
+				packResult = JSON.parse(stdout.slice(stdout.indexOf('[')));
+			} catch (err) {
+				throw new Error(
+					`Failed to parse npm pack output for ${application.packageIdentifier}: ${err.message}\nstdout: ${stdout}`
+				);
+			}
+			if (!Array.isArray(packResult) || typeof packResult[0]?.filename !== 'string') {
+				throw new Error(`Unexpected npm pack output for ${application.packageIdentifier}:\n${stdout}`);
+			}
+
+			tarballPath = join(parentDirPath, packResult[0].filename);
 			tarball = createReadStream(tarballPath);
 		}
 	}

package/core/components/ApplicationScope.ts

@@ -21,10 +21,10 @@ export class ApplicationScope {
 	server: Server;
 	mode?: 'native' | 'vm' | 'vm-current-context' | 'compartment'; // option to set this from the scope
 	dependencyLoader?: 'native' | 'app' | 'auto'; // option to set this from the scope
-	verifyPath?: string;
+	allowedPath?: string;
 	config: any;
 	moduleCache: any; // used by the loader to retain a cache of modules, type is an internal detail of the loader
-	constructor(name: string, resources: Resources, server: Server, isInternal = false, verifyPath?: string) {
+	constructor(name: string, resources: Resources, server: Server, isInternal = false) {
 		this.logger = forComponent(name, !isInternal);
 
 		this.resources = resources;
@@ -32,7 +32,6 @@ export class ApplicationScope {
 
 		this.mode = env.get(CONFIG_PARAMS.APPLICATIONS_MODULELOADER) ?? 'vm';
 		this.dependencyLoader = env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYLOADER);
-		this.verifyPath = verifyPath;
 	}
 
 	/**

package/core/components/componentLoader.ts

@@ -274,7 +274,7 @@ export async function loadComponent(
 		autoReload,
 		appName,
 	} = options;
-	applicationScope.verifyPath ??= componentDirectory;
+	applicationScope.allowedPath ??= realpathSync(componentDirectory);
 	if (providedLoadedComponents) loadedComponents = providedLoadedComponents;
 	try {
 		let config;
@@ -326,8 +326,13 @@ export async function loadComponent(
 
 			let extensionModule: any;
 			const pkg = componentConfig.package;
+			const loadComponentOption = componentConfig.loadComponent ?? 'always';
 			try {
 				if (pkg) {
+					if (loadComponentOption === 'dev-only' && !process.env.DEV_MODE) {
+						componentLifecycle.loaded(componentStatusName, `Component '${componentStatusName}' skipped (dev-only)`);
+						continue;
+					}
 					let componentPath: string | null = null;
 					if (isRoot) {
 						componentPath = join(componentDirectory, 'components', componentName);
@@ -344,7 +349,7 @@ export async function loadComponent(
 						}
 					}
 					if (componentPath) {
-						subApplicationScope.verifyPath ??= componentPath;
+						subApplicationScope.allowedPath ??= realpathSync(componentPath);
 						if (!process.env.HARPER_SAFE_MODE) {
 							extensionModule = await loadComponent(componentPath, resources, origin, {
 								isRoot: false,
@@ -354,6 +359,12 @@ export async function loadComponent(
 							});
 							componentFunctionality[componentName] = true;
 						}
+					} else if (loadComponentOption === 'if-installed') {
+						componentLifecycle.loaded(
+							componentStatusName,
+							`Component '${componentStatusName}' skipped (not installed)`
+						);
+						continue;
 					} else {
 						throw new Error(`Unable to find package ${componentName}:${pkg}`);
 					}

package/core/config/harperConfigEnvVars.ts

@@ -15,6 +15,7 @@ import type { Logger } from '../utility/logging/logger.ts';
 import * as fs from 'fs-extra';
 import * as path from 'node:path';
 import * as crypto from 'node:crypto';
+import { cloneDeep } from 'lodash';
 import { getBackupDirPath } from './configHelpers.ts';
 
 const STATE_FILE_NAME = '.harper-config-state.json';
@@ -590,6 +591,39 @@ function cleanupRemovedEnvVar(
 	logger.debug?.(`${envVarName} removed, cleaned up values`);
 }
 
+/**
+ * Compose a merged config from HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG
+ * layered with an optional base. Later layers win:
+ *   HARPER_DEFAULT_CONFIG  <  base  <  HARPER_SET_CONFIG
+ *
+ * HARPER_DEFAULT_CONFIG provides scaffolding defaults, the base (e.g., the
+ * user's existing config file) is layered on top, and HARPER_SET_CONFIG
+ * force-overrides everything. This matches the precedence applied by the
+ * runtime pipeline in applyRuntimeEnvConfig.
+ *
+ * Unlike applyRuntimeEnvConfig, this does NOT read or write the config state
+ * file and does NOT track sources — it returns a fresh object. Use when you
+ * need the effective value of a config key before the state/file wiring is in
+ * place (e.g., during clone / pre-install).
+ */
+export function composeConfigFromEnv(base: ConfigObject = {}): ConfigObject {
+	const result: ConfigObject = {};
+	const layers: (ConfigObject | null)[] = [
+		parseConfigEnvVar(process.env.HARPER_DEFAULT_CONFIG, 'HARPER_DEFAULT_CONFIG'),
+		cloneDeep(base),
+		parseConfigEnvVar(process.env.HARPER_SET_CONFIG, 'HARPER_SET_CONFIG'),
+	];
+
+	for (const layer of layers) {
+		if (!layer) continue;
+		for (const [p, value] of Object.entries(flattenObject(layer))) {
+			setNestedValue(result, p, value);
+		}
+	}
+
+	return result;
+}
+
 /**
  * Apply HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG
  * Can be used for both install-time and runtime

package/core/resources/DatabaseTransaction.ts

@@ -7,6 +7,7 @@ import * as envMngr from '../utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 import { convertToMS } from '../utility/common_utils.js';
 import { when } from '../utility/when.ts';
+import { setTimeout as delay } from 'node:timers/promises';
 import { Transaction as RocksTransaction, type Store as RocksStore } from '@harperfast/rocksdb-js';
 import type { RootDatabaseKind } from './databases.ts';
 import type { Entry } from './RecordEncoder.ts';
@@ -19,6 +20,7 @@ export const TRANSACTION_STATE = {
 	OPEN: 1, // the transaction is open and can be used for reads and writes
 	LINGERING: 2, // the transaction has completed a read, but can be used for immediate writes
 };
+const MAX_RETRIES = 40;
 let outstandingCommit, outstandingCommitStart;
 let confirmReplication;
 export function replicationConfirmation(callback) {
@@ -93,6 +95,7 @@ export class DatabaseTransaction implements Transaction {
 		if (this.open !== TRANSACTION_STATE.OPEN) return; // can not start a new read transaction as there is no future commit that will take place, just have to allow the read to latest database state
 
 		this.transaction = new RocksTransaction(this.db.store);
+
 		if (this.timestamp) {
 			this.transaction.setTimestamp(this.timestamp);
 		}
@@ -158,7 +161,10 @@ export class DatabaseTransaction implements Transaction {
 		transaction ??= this.transaction;
 		let immediateCommit = false;
 		if (!transaction) {
-			transaction = new RocksTransaction(this.db.store as RocksStore);
+			transaction = new RocksTransaction(operation.store.store as RocksStore);
+			if (operation.store.rootStore !== this.db.rootStore) {
+				harperLogger.warn?.('Created new transaction in save, but the store does match existing store', transaction.id);
+			}
 			if (this.open === TRANSACTION_STATE.OPEN) {
 				this.transaction = transaction;
 			} else {
@@ -168,9 +174,10 @@ export class DatabaseTransaction implements Transaction {
 			if (txnTime) {
 				transaction.setTimestamp(txnTime);
 			}
+		} else {
 		}
 		if (this.retries > 0) {
-			// this is marks the rocks transaction as a retry so we don't write the transaction log again
+			// This marks the Rocks transaction as a retry so we don't write the transaction log again
 			transaction.isRetry = true;
 		}
 		if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
@@ -295,6 +302,16 @@ export class DatabaseTransaction implements Transaction {
 							// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
 							// for future transactions
 							this.retries++;
+							harperLogger.debug?.('retrying', transaction.id, this.retries);
+							if (this.retries > 2) {
+								if (this.retries > MAX_RETRIES) {
+									throw new ServerError(
+										`After ${MAX_RETRIES} retries, unable to commit transaction, transaction is in conflict with ongoing writes`
+									);
+								}
+								// start delaying, back off to try to space out transactions and avoid excessive conflicts
+								return delay(this.retries * this.retries).then(() => this.commit({ transaction }));
+							}
 							return this.commit({ transaction }); // try again
 						} else throw error;
 					}

package/core/resources/RecordEncoder.ts

@@ -688,13 +688,13 @@ export function recordUpdater(store, tableId, auditStore) {
 export function setAdditionalAuditRefs(refs: Array<{ version: number; nodeId: number }> | undefined) {
 	additionalAuditRefsNextEncoding = refs;
 }
-export function removeEntry(store: any, entry: any, existingVersion?: number) {
+export function removeEntry(store: any, entry: any, options?: any) {
 	if (!entry) return;
 	if (entry.value && entry.metadataFlags & HAS_BLOBS) {
 		// if it used to have blobs, we need to delete the old blobs
 		deleteBlobsInObject(entry.value);
 	}
-	return store.remove(entry.key, existingVersion);
+	return store.remove(entry.key, options);
 }
 export interface RecordObject {
 	getUpdatedTime(): number;

package/core/resources/ResourceInterface.ts

@@ -69,7 +69,7 @@ export interface Context {
 	/**	 The user making the request */
 	user?: User;
 	/** Check the username and password against the core user table to verify user identity */
-	login: (username: string, password: string) => Promise<string>;
+	login?: (username: string, password: string) => Promise<string>;
 	/** Describes the current cookie-based session if it is present and grants the capacity to delete it. authentication.enableSessions must be turned on in the harperdb-config.yaml  */
 	session?: Session;
 	/**	 The database transaction object */