@harperfast/harper 5.0.1 → 5.0.3

package/resources/Table.ts

@@ -1424,23 +1424,35 @@ export function makeTable(options) {
 		 */
 		static evict(id, existingRecord, existingVersion) {
 			let entry;
-			if (hasSourceGet || audit) {
-				if (!existingRecord) return;
-				entry = primaryStore.getEntry(id);
-				if (!entry || !existingRecord) return;
-				if (entry.version !== existingVersion) return;
-			}
-			if (hasSourceGet) {
-				// if there is a resolution in-progress, abandon the eviction
-				if (primaryStore.hasLock(id, entry.version)) return;
+			let transaction = txnForContext({ transaction: new DatabaseTransaction() }).getReadTxn();
+			let options = { transaction };
+			try {
+				if (hasSourceGet || audit) {
+					if (!existingRecord) return;
+					entry = primaryStore.getEntry(id, options);
+					if (!entry || !existingRecord) return;
+					if (entry.version !== existingVersion) return;
+				}
+				if (hasSourceGet) {
+					// if there is a resolution in-progress, abandon the eviction
+					if (primaryStore.hasLock(id, entry.version)) return;
+				}
+				// evictions never go in the audit log, so we can not record a deletion entry for the eviction
+				// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
+				// removed the entry entirely, but first cleanup indices
+				if (primaryStore.ifVersion) {
+					// lmdb
+					primaryStore.ifVersion?.(id, existingVersion, () => {
+						updateIndices(id, existingRecord, null);
+					});
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
+				} else {
+					updateIndices(id, existingRecord, null, options);
+					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
+				}
+			} finally {
+				return transaction.commit();
 			}
-			primaryStore.ifVersion?.(id, existingVersion, () => {
-				updateIndices(id, existingRecord, null);
-			});
-			// evictions never go in the audit log, so we can not record a deletion entry for the eviction
-			// as there is no corresponding audit entry and it would never get cleaned up. So we must simply
-			// removed the entry entirely
-			return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
 		}
 		/**
 		 * This is intended to acquire a lock on a record from the whole cluster.
@@ -1951,9 +1963,10 @@ export function makeTable(options) {
 							context.lastModified = existingEntry.version;
 						TableResource._updateResource(this, existingEntry);
 					}
-					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return; // a newer record exists locally
-					updateIndices(id, existingRecord);
-					logger.trace?.(`Deleting record with id: ${id}, txn timestamp: ${new Date(txnTime).toISOString()}`);
+					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
+						return;
+					} // a newer record exists locally
+					updateIndices(id, existingRecord, null, transaction && { transaction });
 					if (audit || trackDeletes) {
 						updateRecord(
 							id,
@@ -2262,7 +2275,9 @@ export function makeTable(options) {
 						return (entryA, entryB) => {
 							const a = getAttributeValue(entryA, order.attribute, context);
 							const b = getAttributeValue(entryB, order.attribute, context);
-							const diff = descending ? compareKeys(b, a) : compareKeys(a, b);
+							const diff = descending
+								? compareKeys(convertToComparableKeys(b), convertToComparableKeys(a))
+								: compareKeys(convertToComparableKeys(a), convertToComparableKeys(b));
 							if (diff === 0) return nextComparator?.(entryA, entryB) || 0;
 							return diff;
 						};
@@ -3509,6 +3524,7 @@ export function makeTable(options) {
 			// determine what index values need to be removed and added
 			let valuesToAdd = getIndexedValues(value, indexNulls) as any[];
 			let valuesToRemove = getIndexedValues(existingValue, indexNulls) as any[];
+			let isLMDB = !!index.prefetch;
 			if (valuesToRemove?.length > 0) {
 				// put this in a conditional so we can do a faster version for new records
 				// determine the changes/diff from new values and old values
@@ -3525,18 +3541,18 @@ export function makeTable(options) {
 						})
 					: [];
 				valuesToRemove = Array.from(setToRemove);
-				if ((valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
+				if (isLMDB && (valuesToRemove.length > 0 || valuesToAdd.length > 0) && LMDB_PREFETCH_WRITES) {
 					// prefetch any values that have been removed or added
 					const valuesToPrefetch = valuesToRemove.concat(valuesToAdd).map((v) => ({ key: v, value: id }));
-					index.prefetch?.(valuesToPrefetch, noop);
+					index.prefetch(valuesToPrefetch, noop);
 				}
 				//if the update cleared out the attribute value we need to delete it from the index
 				for (let i = 0, l = valuesToRemove.length; i < l; i++) {
 					index.remove(valuesToRemove[i], id, options);
 				}
-			} else if (valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
+			} else if (isLMDB && valuesToAdd?.length > 0 && LMDB_PREFETCH_WRITES) {
 				// no old values, just new
-				index.prefetch?.(
+				index.prefetch(
 					valuesToAdd.map((v) => ({ key: v, value: id })),
 					noop
 				);
@@ -4122,7 +4138,7 @@ export function makeTable(options) {
 								// don't do anything if the version has changed
 								return;
 							}
-							updateIndices(id, existingRecord, updatedRecord);
+							updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
 							if (updatedRecord) {
 								if (existingEntry) {
 									context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
@@ -4572,3 +4588,12 @@ function hasOtherProcesses(store) {
 			return +line.match(/\d+/)?.[0] != pid;
 		});
 }
+function convertToComparableKeys(a) {
+	if (a instanceof Date) {
+		return a.getTime();
+	}
+	if (Array.isArray(a)) {
+		return a.map(convertToComparableKeys);
+	}
+	return a;
+}

package/resources/analytics/write.ts

@@ -219,15 +219,12 @@ export async function recordHostname() {
 	const nodeId = stableNodeId(hostname);
 	log.trace?.('recordHostname nodeId:', nodeId);
 	const hostnamesTable = getAnalyticsHostnameTable();
-	const record = await hostnamesTable.get(nodeId);
-	if (!record) {
-		const hostnameRecord = {
-			id: nodeId,
-			hostname,
-		};
-		log.trace?.(`recordHostname storing hostname: ${JSON.stringify(hostnameRecord)}`);
-		await hostnamesTable.put(hostnameRecord.id, hostnameRecord);
-	}
+	const hostnameRecord = {
+		id: nodeId,
+		hostname,
+	};
+	log.trace?.(`recordHostname storing hostname: ${JSON.stringify(hostnameRecord)}`);
+	await hostnamesTable.put(hostnameRecord.id, hostnameRecord);
 }
 
 export interface Metric {
@@ -514,7 +511,7 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 		await rest();
 	}
 	for (const entry of threadsToAverage) {
-		// eslint-disable-next-line @typescript-eslint/no-unused-vars,prefer-const
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
 		let { path, method, type, metric, count, total, distribution, threads, ...measures } = entry;
 		threads = threads.filter((thread) => thread);
 		for (const measureName in measures) {

package/resources/databases.ts

@@ -120,9 +120,13 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo
 	}
 	let db: RocksRootDatabase;
 	if (options.dupSort) {
-		db = RocksDatabase.open(new RocksIndexStore(path, options)) as RocksDatabaseEx;
+		db = new RocksIndexStore(path, options).open() as RocksDatabaseEx;
 	} else {
 		db = RocksDatabase.open(path, options) as RocksDatabaseEx;
+		// the RocksDB put and remove return promises, which masks thrown errors in non-awaiting calls to put/remove,
+		// making them unsafe to replace LMDB methods, which will synchronously throw errors if there is a problem
+		db.put = db.putSync;
+		db.remove = db.removeSync;
 		db.encoder.name = options.name;
 	}
 	db.env = {};
@@ -386,18 +390,18 @@ function initStores(
 ) {
 	const envInit = new OpenEnvironmentObject(path, false);
 	const internalDbiInit = createOpenDBIObject(false);
-	let dbisStore = rootStore.dbisDb;
-	if (!dbisStore) {
+	let attributesDbi = rootStore.dbisDb;
+	if (!attributesDbi) {
 		if (rootStore instanceof RocksDatabase) {
-			dbisStore = openRocksDatabase(rootStore.path, {
+			attributesDbi = openRocksDatabase(rootStore.path, {
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
 			}) as RocksDatabaseEx;
 		} else {
-			dbisStore = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
+			attributesDbi = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
 		}
-		rootStore.dbisDb = dbisStore;
+		rootStore.dbisDb = attributesDbi;
 	}
 
 	let auditStore = rootStore.auditStore;
@@ -428,7 +432,7 @@ function initStores(
 	definedTables.rootStore = rootStore;
 	const tablesToLoad = new Map<string, any>();
 
-	for (const result of dbisStore.getRange({ start: false })) {
+	for (const result of attributesDbi.getRange({ start: false })) {
 		const { key, value } = result as { key: string; value: any };
 		let [tableName, attribute_name] = key.toString().split('/');
 		if (attribute_name === '') {
@@ -489,16 +493,16 @@ function initStores(
 		} else {
 			tableId = primaryAttribute.tableId;
 			if (tableId) {
-				if (tableId >= (dbisStore.getSync(NEXT_TABLE_ID) || 0)) {
-					dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
+				if (tableId >= (attributesDbi.getSync(NEXT_TABLE_ID) || 0)) {
+					attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
 					logger.info(`Updating next table id (it was out of sync) to ${tableId + 1} for ${tableName}`);
 				}
 			} else {
-				primaryAttribute.tableId = tableId = dbisStore.getSync(NEXT_TABLE_ID);
+				primaryAttribute.tableId = tableId = attributesDbi.getSync(NEXT_TABLE_ID);
 				if (!tableId) tableId = 1;
 				logger.debug(`Table {tableName} missing an id, assigning {tableId}`);
-				dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
-				dbisStore.putSync(primaryAttribute.key, primaryAttribute);
+				attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
+				attributesDbi.putSync(primaryAttribute.key, primaryAttribute);
 			}
 			const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
 			dbiInit.compression = primaryAttribute.compression;
@@ -544,7 +548,18 @@ function initStores(
 			const attribute = attributes.find((attribute) => attribute.name === existingAttribute.name);
 			if (!attribute) {
 				if (existingAttribute.isPrimaryKey) {
-					logger.error('Unable to remove existing primary key attribute', existingAttribute);
+					logger.error(
+						new Error('Unable to remove existing primary key attribute'),
+						existingAttribute,
+						'from attributes',
+						existingAttributes,
+						'in',
+						tableName,
+						'requesting new attribute list',
+						attributes,
+						'full metadata list',
+						Array.from(attributesDbi.getRange({ start: false }))
+					);
 					continue;
 				}
 				if (existingAttribute.indexed) {
@@ -581,7 +596,7 @@ function initStores(
 					indices,
 					attributes,
 					schemaDefined: primaryAttribute.schemaDefined,
-					dbisDB: dbisStore,
+					dbisDB: attributesDbi,
 				})
 			);
 			table.schemaVersion = 1;

package/resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -130,8 +130,8 @@ export class HierarchicalNavigableSmallWorld {
 			// Generate random level for this new element
 			const level = oldNode.level ?? Math.min(Math.floor(-Math.log(Math.random()) * this.mL), MAX_LEVEL);
 			let currentLevel = entryPoint.level;
-			if (level >= currentLevel) {
-				// if we are at this level or higher, make this the new entry point
+			if (level > currentLevel) {
+				// if we are at a higher level, make this the new entry point
 				if (typeof nodeId !== 'number') {
 					throw new Error('Invalid nodeId: ' + nodeId);
 				}
@@ -142,7 +142,14 @@ export class HierarchicalNavigableSmallWorld {
 			// For each level from top to bottom
 			while (currentLevel > level) {
 				// Search for closest neighbors at current level
-				const neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, currentLevel);
+				const neighbors = this.searchLayer(
+					vector,
+					entryPointId,
+					entryPoint,
+					this.efConstruction,
+					currentLevel,
+					options
+				);
 
 				if (neighbors.length > 0) {
 					entryPointId = neighbors[0].id; // closest neighbor becomes new entry point
@@ -157,7 +164,7 @@ export class HierarchicalNavigableSmallWorld {
 
 			// Connect the new element to neighbors at its level and below
 			for (let l = Math.min(level, currentLevel); l >= 0; l--) {
-				let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l);
+				let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l, options);
 				neighbors = neighbors.slice(0, this.M << 1) as SearchResults;
 
 				if (neighbors.length === 0 && l === 0) {
@@ -232,6 +239,19 @@ export class HierarchicalNavigableSmallWorld {
 							oldNode[l] = oldConnections;
 						}
 						oldConnections.splice(oldPosition, 1);
+						// update the distance in the reverse connection if the vector changed
+						if (oldConnection.distance !== distance) {
+							const neighborNode = updateNode(id, node);
+							if (neighborNode[l]) {
+								if (Object.isFrozen(neighborNode[l])) {
+									neighborNode[l] = neighborNode[l].slice();
+								}
+								const reverseIdx = neighborNode[l].findIndex(({ id: nid }) => nid === nodeId);
+								if (reverseIdx >= 0) {
+									neighborNode[l][reverseIdx] = { id: nodeId, distance };
+								}
+							}
+						}
 					} else {
 						// add new connection since this is truly a new connection now
 						this.addConnection(id, updateNode(id, node), nodeId, l, distance, updateNode, options);
@@ -323,16 +343,16 @@ export class HierarchicalNavigableSmallWorld {
 			this.indexStore.put(id, updatedNode, options);
 		}
 		for (const [key, vector] of needsReindexing) {
-			this.index(key, vector, vector);
+			this.index(key, vector, vector, options);
 		}
 		this.checkSymmetry(nodeId, this.indexStore.getSync(nodeId, options), options);
 	}
 
-	private getEntryPoint() {
+	private getEntryPoint(options: { transaction?: any } = {}) {
 		// Get entry point
-		const entryPointId = this.indexStore.getSync(ENTRY_POINT);
+		const entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
 		if (entryPointId === undefined) return;
-		const node = this.indexStore.getSync(entryPointId);
+		const node = this.indexStore.getSync(entryPointId, options);
 		return { id: entryPointId, ...node };
 	}
 
@@ -346,6 +366,7 @@ export class HierarchicalNavigableSmallWorld {
 	 * @param ef
 	 * @param level
 	 * @param distanceFunction
+	 * @param options
 	 * @private
 	 */
 	private searchLayer(
@@ -354,13 +375,14 @@ export class HierarchicalNavigableSmallWorld {
 		entryPoint: any,
 		ef: number,
 		level: number,
+		options: { transaction?: any } = {},
 		distanceFunction = this.distance
 	): SearchResults {
 		const visited = new Set([entryPointId]);
 		const candidates = [
 			{
 				id: entryPointId,
-				distance: this.distance(queryVector, entryPoint.vector),
+				distance: distanceFunction(queryVector, entryPoint.vector),
 				node: entryPoint,
 			},
 		];
@@ -383,7 +405,7 @@ export class HierarchicalNavigableSmallWorld {
 				if (visited.has(neighborId) || neighborId === undefined) continue;
 				visited.add(neighborId);
 
-				const neighbor = this.indexStore.getSync(neighborId);
+				const neighbor = this.indexStore.getSync(neighborId, options);
 				if (!neighbor) continue;
 				this.nodesVisitedCount++;
 				const distance = distanceFunction(queryVector, neighbor.vector);
@@ -416,19 +438,22 @@ export class HierarchicalNavigableSmallWorld {
 	 * @param comparator
 	 * @param context
 	 */
-	search({
-		target,
-		value,
-		descending,
-		distance,
-		comparator,
-	}: {
-		target: number[];
-		value: number;
-		descending: boolean;
-		distance: string;
-		comparator: string;
-	}) {
+	search(
+		{
+			target,
+			value,
+			descending,
+			distance,
+			comparator,
+		}: {
+			target: number[];
+			value: number;
+			descending: boolean;
+			distance: string;
+			comparator: string;
+		},
+		context: any
+	) {
 		let limit = 0; // zero is ignored, only used if set below
 		switch (comparator) {
 			case 'lt':
@@ -449,14 +474,23 @@ export class HierarchicalNavigableSmallWorld {
 		if (!target) throw new ClientError('A target vector must be provided for an HNSW query');
 		if (!Array.isArray(target)) throw new ClientError('The target vector must be an array');
 
-		let entryPoint = this.getEntryPoint();
+		const options = context.transaction; // should have a nested RocksDB transaction
+		let entryPoint = this.getEntryPoint(options);
 		if (!entryPoint) return [];
 		let entryPointId = entryPoint.id;
 		let results: Candidate[] = [];
 		// For each level from top to bottom
 		for (let l = entryPoint.level; l >= 0; l--) {
 			// Search for closest neighbors at current level
-			results = this.searchLayer(target, entryPointId, entryPoint, this.efConstructionSearch, l, distanceFunction);
+			results = this.searchLayer(
+				target,
+				entryPointId,
+				entryPoint,
+				this.efConstructionSearch,
+				l,
+				options,
+				distanceFunction
+			);
 
 			if (results.length > 0) {
 				const neighbor = results[0]; // closest neighbor becomes new entry point
@@ -487,7 +521,7 @@ export class HierarchicalNavigableSmallWorld {
 				// verify that the connection is symmetrical
 				const symmetrical = neighborNode[l]?.find(({ id: nid }) => nid == id);
 				if (!symmetrical) {
-					logger.info?.('asymmetry detected', neighborNode[l]);
+					logger.info?.('asymmetry detected', neighborNode[l], 'does not have', id);
 				}
 			}
 			l++;
@@ -531,10 +565,13 @@ export class HierarchicalNavigableSmallWorld {
 				if (removedNode) {
 					// Remove the reverse connection if it exists
 					if (removedNode[level]) {
-						removedNode = updateNode(removed.id, removedNode);
-						removedNode[level] = removedNode[level].filter(({ id }) => id !== fromId);
-						if (level === 0 && removedNode[level].length === 0) {
-							logger.info?.('should not remove last connection', fromId, toId);
+						const filtered = removedNode[level].filter(({ id }) => id !== fromId);
+						if (level === 0 && filtered.length === 0) {
+							// don't remove the last connection at level 0 — it would orphan this node
+							logger.info?.('skipping removal of last connection', fromId, toId);
+						} else {
+							removedNode = updateNode(removed.id, removedNode);
+							removedNode[level] = filtered;
 						}
 					}
 				}

package/security/certificateVerification/ocspVerification.ts

@@ -95,7 +95,7 @@ export async function verifyOCSP(
 			method: cached.method || 'ocsp',
 		};
 	} catch (error) {
-		logger.error?.(`OCSP verification error: ${error}`);
+		logger.error?.(`OCSP verification error:`, error);
 
 		// Check failure mode
 		if (config.failureMode === 'fail-closed') {

package/security/jsLoader.ts

@@ -14,7 +14,16 @@ import * as child_process from 'node:child_process';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 import { contentTypes } from '../server/serverHelpers/contentTypes.ts';
 import type { CompartmentOptions } from 'ses';
-import { mkdirSync, readFileSync, writeFileSync, unlinkSync, openSync, closeSync, statSync } from 'node:fs';
+import {
+	mkdirSync,
+	readFileSync,
+	writeFileSync,
+	unlinkSync,
+	openSync,
+	closeSync,
+	statSync,
+	realpathSync,
+} from 'node:fs';
 import { join } from 'node:path';
 import { EventEmitter } from 'node:events';
 import { whenComponentsLoaded } from '../server/threads/threadServer.js';
@@ -495,13 +504,13 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC
 		}
 
 		if (url.startsWith('file://') && usePrivateGlobal) {
-			checkAllowedModulePath(url, scope.verifyPath);
+			checkAllowedModulePath(url, scope.allowedPath);
 			const source = readFileSync(new URL(url), { encoding: 'utf-8' });
 			return createModuleFromSource(url, source, usePrivateGlobal);
 		}
 
 		// For Node.js built-in modules (node:) and npm packages without application loader for dependency
-		const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
+		const replacedModule = checkAllowedModulePath(url, scope.allowedPath);
 		if (replacedModule) {
 			return createSyntheticModule(url, normalizeImportedModule(replacedModule));
 		}
@@ -570,7 +579,7 @@ async function getCompartment(scope: ApplicationScope, globals) {
 					}
 					return new StaticModuleRecord(moduleText, moduleSpecifier);
 				} else {
-					checkAllowedModulePath(moduleSpecifier, scope.verifyPath);
+					checkAllowedModulePath(moduleSpecifier, scope.allowedPath);
 					const moduleExports = await import(moduleSpecifier);
 					return {
 						imports: [],
@@ -761,23 +770,54 @@ function isProcessRunning(pid: number): boolean {
  * Acquires an exclusive lock using the PID file itself (synchronously with busy-wait)
  * Returns 0 if lock was acquired (need to spawn new process), or the existing PID if process is running
  */
-function acquirePidFileLock(pidFilePath: string, maxRetries = 100, retryDelay = 5): number {
+function parsePidFile(content: string): { pid: number; version: number } {
+	const lines = content.trim().split('\n');
+	const pid = Number.parseInt(lines[0], 10);
+	const version = lines.length > 1 ? parseInt(lines[1], 10) : 0;
+	return { pid, version };
+}
+
+function acquirePidFileLock(
+	pidFilePath: string,
+	requestedVersion?: number,
+	maxRetries = 100,
+	retryDelay = 5
+): { pid: number; version: number } {
 	for (let attempt = 0; attempt < maxRetries; attempt++) {
 		try {
 			// Try to open exclusively - 'wx' fails if file exists
 			const fd = openSync(pidFilePath, 'wx');
 			closeSync(fd);
-			return 0; // Successfully acquired lock (file created), caller should spawn process
+			return { pid: 0, version: 0 }; // Successfully acquired lock (file created), caller should spawn process
 		} catch (err) {
 			if (err.code === 'EEXIST') {
 				// File exists - check if it contains a valid running process
 				try {
 					const pidContent = readFileSync(pidFilePath, 'utf-8');
-					const existingPid = parseInt(pidContent.trim(), 10);
+					const { pid: existingPid, version: existingVersion } = parsePidFile(pidContent);
 
 					if (!isNaN(existingPid) && isProcessRunning(existingPid)) {
-						// Valid process is running, return its PID immediately
-						return existingPid;
+						// If the version isn't the one we want, kill the existing process and re-acquire
+						if (requestedVersion != null && requestedVersion !== existingVersion) {
+							try {
+								process.kill(existingPid);
+							} catch {
+								// Process may have already exited
+							}
+							try {
+								unlinkSync(pidFilePath);
+							} catch {
+								// Another thread may have removed it
+							}
+							// Retry to acquire the lock for the new version
+							const start = Date.now();
+							while (Date.now() - start < retryDelay) {
+								// Busy wait for process cleanup
+							}
+							continue;
+						}
+						// Valid process is running at same or higher version, return its PID
+						return { pid: existingPid, version: existingVersion };
 					}
 
 					// Invalid/empty PID - check file age to determine if it's stale or being written
@@ -824,6 +864,7 @@ function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess
 			throw new Error(
 				`Calling ${spawnFunction.name} in Harper must have a process "name" in the options to ensure that a single process is started and reused`
 			);
+		const requestedVersion = options?.version;
 
 		// Ensure PID directory exists
 		const pidDir = join(basePath, 'pids');
@@ -831,20 +872,22 @@ function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess
 
 		const pidFilePath = join(pidDir, `${processName}.pid`);
 
-		// Try to acquire lock - returns 0 if acquired, or existing PID
-		const existingPid = acquirePidFileLock(pidFilePath);
+		// Try to acquire lock - returns pid: 0 if acquired, or existing PID/version
+		const existing = acquirePidFileLock(pidFilePath, requestedVersion);
 
-		if (existingPid !== 0) {
+		if (existing.pid !== 0) {
 			// Existing process is running, return wrapper
-			return new ExistingProcessWrapper(existingPid);
+			return new ExistingProcessWrapper(existing.pid);
 		}
 
 		// We acquired the lock (file was created), spawn new process
 		const childProcess = spawnFunction(command, args, options, callback);
 
-		// Write PID to the file we just created
+		// Write PID (and version if provided) to the file we just created
+		const pidFileContent =
+			requestedVersion != null ? `${childProcess.pid}\n${requestedVersion}` : childProcess.pid.toString();
 		try {
-			writeFileSync(pidFilePath, childProcess.pid.toString(), 'utf-8');
+			writeFileSync(pidFilePath, pidFileContent, 'utf-8');
 		} catch (err) {
 			// Failed to write PID, clean up
 			try {
@@ -869,21 +912,24 @@ function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess
 
 /**
  * Validates whether a module can be loaded based on security restrictions and returns the module path or replacement.
- * For file URLs, ensures the module is within the containing folder.
+ * For file URLs, ensures the module is within the allowed path.
  * For node built-in modules, checks against an allowlist and returns any replacements.
  *
  * @param {string} moduleUrl - The URL or identifier of the module to be loaded, which may be a file: URL, node: URL, or bare module specifier.
- * @param {string} containingFolder - The absolute path of the folder that contains the application, used to validate file: URLs are within bounds.
+ * @param {string} allowedPath - The absolute path that the module is allowed to load from.
  * @return {any} Returns undefined for allowed file paths, or a replacement module identifier for allowed node built-in modules.
- * @throws {Error} Throws an error if the module is outside the application folder or if the module is not in the allowed list.
+ * @throws {Error} Throws an error if the module is outside the allowed path or if the module is not in the allowed list.
  */
-function checkAllowedModulePath(moduleUrl: string, containingFolder?: string): boolean {
+function checkAllowedModulePath(moduleUrl: string, allowedPath?: string): boolean {
 	if (moduleUrl.startsWith('file:')) {
-		const path = moduleUrl.slice(7);
-		if (!containingFolder || path.startsWith(containingFolder)) {
+		let path = moduleUrl.slice(7);
+		try {
+			path = realpathSync(path);
+		} catch {}
+		if (!allowedPath || path.startsWith(allowedPath)) {
 			return;
 		}
-		throw new Error(`Can not load module outside of application folder ${containingFolder}`);
+		throw new Error(`Can not load module outside of allowed path`);
 	}
 	let simpleName = moduleUrl.startsWith('node:') ? moduleUrl.slice(5) : moduleUrl;
 	simpleName = simpleName.split('/')[0];