@harperfast/harper-pro 5.0.0-beta.7 → 5.0.0-beta.8

package/core/bin/cliOperations.js

@@ -10,7 +10,7 @@ const YAML = require('yaml');
 const { packageDirectory } = require('../components/packageComponent.ts');
 const { encode } = require('cbor-x');
 const { getHdbPid } = require('../utility/processManagement/processManagement.js');
-const { initConfig } = require('../config/configUtils.js');
+const { initConfig, getConfigPath } = require('../config/configUtils.js');
 
 const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' };
 
@@ -93,7 +93,7 @@ async function cliOperations(req) {
 			process.exit(1);
 		}
 
-		if (!fs.existsSync(envMgr.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET))) {
+		if (!fs.existsSync(getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET))) {
 			console.error('No domain socket found, unable to perform this operation');
 			process.exit(1);
 		}
@@ -102,7 +102,7 @@ async function cliOperations(req) {
 	try {
 		let options = target ?? {
 			protocol: 'http:',
-			socketPath: envMgr.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET),
+			socketPath: getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET),
 		};
 		options.method = 'POST';
 		options.headers = { 'Content-Type': 'application/json' };

package/core/components/ApplicationScope.ts

@@ -23,6 +23,7 @@ export class ApplicationScope {
 	dependencyContainment?: boolean; // option to set this from the scope
 	verifyPath?: string;
 	config: any;
+	moduleCache: any; // used by the loader to retain a cache of modules, type is an internal detail of the loader
 	constructor(name: string, resources: Resources, server: Server, isInternal = false, verifyPath?: string) {
 		this.logger = forComponent(name, !isInternal);
 

package/core/components/componentLoader.ts

@@ -39,6 +39,7 @@ import { lifecycle as componentLifecycle } from './status/index.ts';
 import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts';
 import { PluginModule } from './PluginModule.ts';
 import { getEnvBuiltInComponents } from './Application.ts';
+import { pathToFileURL } from 'node:url';
 
 const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
 let loadedComponents = new Map<any, any>();
@@ -360,7 +361,9 @@ export async function loadComponent(
 					const plugin = TRUSTED_RESOURCE_PLUGINS[componentName];
 					extensionModule =
 						typeof plugin === 'string'
-							? await import(plugin.startsWith('@/') ? join(PACKAGE_ROOT, plugin.slice(1)) : plugin)
+							? await import(
+									plugin.startsWith('@/') ? pathToFileURL(join(PACKAGE_ROOT, plugin.slice(1))).toString() : plugin
+								)
 							: plugin;
 				}
 

package/core/resources/DatabaseTransaction.ts

@@ -35,6 +35,7 @@ export type CommitOptions = {
 	timestamp?: number;
 	retries?: number;
 	flush?: boolean;
+	transaction?: RocksTransaction;
 };
 
 type ReadTransaction = (LMDBTransaction | RocksTransaction) & {
@@ -116,8 +117,13 @@ export class DatabaseTransaction implements Transaction {
 		if (!this.transaction) return;
 		if (--this.readTxnsUsed === 0) {
 			trackedTxns.delete(this);
-			this.transaction?.abort();
-			this.transaction = null;
+			if (this.open === TRANSACTION_STATE.LINGERING) {
+				// if we have lingering writes, we have to call commit to finish them
+				this.commit();
+			} else {
+				this.transaction?.abort();
+				this.transaction = null;
+			}
 		}
 	}
 
@@ -139,9 +145,6 @@ export class DatabaseTransaction implements Transaction {
 	}
 
 	addWrite(operation: TransactionWrite) {
-		if (this.open === TRANSACTION_STATE.CLOSED) {
-			throw new Error('Can not use a transaction that is no longer open');
-		}
 		this.writes.push(operation);
 		if (!operation.deferSave) {
 			// Setting saved to false means to defer saving
@@ -150,61 +153,94 @@ export class DatabaseTransaction implements Transaction {
 		return operation;
 	}
 
-	save(operation: TransactionWrite, reloadEntry = false) {
+	save(operation: TransactionWrite, transaction?: RocksTransaction, reloadEntry = false) {
 		let txnTime = this.timestamp;
-		if (!this.transaction) {
-			this.transaction = new RocksTransaction(this.db.store as RocksStore);
+		transaction ??= this.transaction;
+		let immediateCommit = false;
+		if (!transaction) {
+			transaction = new RocksTransaction(this.db.store as RocksStore);
+			if (this.open === TRANSACTION_STATE.OPEN) {
+				this.transaction = transaction;
+			} else {
+				// if it is closed, we have to immediately commit, using our immediate transaction
+				immediateCommit = true;
+			}
 			if (txnTime) {
-				this.transaction.setTimestamp(txnTime);
+				transaction.setTimestamp(txnTime);
 			}
 		}
 		if (this.retries > 0) {
 			// this is marks the rocks transaction as a retry so we don't write the transaction log again
-			this.transaction.isRetry = true;
+			transaction.isRetry = true;
 		}
-		if (!txnTime) txnTime = this.timestamp = this.transaction.getTimestamp();
+		if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
 		if (reloadEntry || operation.entry === undefined) {
-			operation.entry = operation.store.getEntry(operation.key, { transaction: this.transaction });
+			operation.entry = operation.store.getEntry(operation.key, { transaction });
+		}
+		if (!operation.saved) {
+			operation.saved = true;
+			// immediately execute in this transaction
+			if (operation.validate?.(txnTime) === false) {
+				operation.commit = () => {}; // noop if we try again
+				return;
+			}
+			let result: Promise<void> = operation.before?.() as Promise<void>;
+			if (result?.then) this.completions.push(result);
+			result = operation.beforeIntermediate?.() as Promise<void>;
+			if (result?.then) this.completions.push(result);
+		}
+		operation.commit(txnTime, operation.entry, this.retries > 0, transaction);
+		if (immediateCommit) {
+			return this.commit({ transaction }); // immediately commit if the harper transaction is closed
 		}
-		operation.saved = true;
-		// immediately execute in this transaction
-		if (operation.validate?.(txnTime) === false) return;
-		let result: Promise<void> = operation.before?.() as Promise<void>;
-		if (result?.then) this.completions.push(result);
-		result = operation.beforeIntermediate?.() as Promise<void>;
-		if (result?.then) this.completions.push(result);
-		operation.commit(txnTime, operation.entry, this.retries > 0, this.transaction);
 	}
 
 	/**
 	 * Resolves with information on the timestamp and success of the commit
 	 */
 	commit(options: CommitOptions = {}): MaybePromise<CommitResolution> {
+		let transaction = options.transaction ?? this.transaction; // we need to preserve this transaction as we might to resurrect it if we have to retry
 		for (let i = 0; i < this.writes.length; i++) {
 			let operation = this.writes[i];
 			if (this.retries === 0 && operation.saved) continue;
-			this.save(operation, i < this.validated);
+			this.save(operation, transaction, i < this.validated);
 		}
 		this.validated = this.writes.length;
-		return when(this.completions.length > 0 ? Promise.all(this.completions) : null, () => {
+		const completions = this.completions;
+		if (completions.length > 0) this.completions = []; // reset
+		return when(completions.length > 0 ? Promise.all(completions) : null, () => {
+			if (this.writes.length > this.validated) {
+				// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
+				return this.commit(options);
+			}
+			this.open = TRANSACTION_STATE.CLOSED;
 			let commitResolution: MaybePromise<void>;
 			if (--this.readTxnsUsed > 0) {
 				// we still have outstanding iterators using the transaction, we can't just commit/abort it, we will still
 				// need to use it
+				if (this.writes.length > 0) {
+					// if there are outstanding writes, we have to call commit later to finish them
+					this.open = TRANSACTION_STATE.LINGERING;
+					/* TODO: This is not really the intended behavior though, we want to immediately commit writes, but continue to use
+					 * the transaction, as there is likely existing references to the transaction in other parts of the codebase,
+					 * particularly in the query iterator */
+				}
+				/*
 				commitResolution =
 					this.writes.length > 0
-						? this.transaction?.commit({ renewAfterCommit: true /* Try to use RocksDB's CommitAndTryCreateSnapshot */ })
-						: // don't abort, we still have outstanding reads to complete
+						? transaction?.commit({ renewAfterCommit: true }) // Try to use RocksDB's CommitAndTryCreateSnapshot
+			: // don't abort, we still have outstanding reads to complete
 							null;
+				*/
 			} else {
 				// no more reads need to be performed, just commit/abort based if there are any writes
 				trackedTxns.delete(this);
-				if (this.transaction) {
+				this.transaction = null; // clear transaction so any further operations operate immediately
+				if (transaction) {
 					if (this.writes.length > 0) {
-						commitResolution = this.transaction.commit();
+						commitResolution = transaction.commit();
 					} else {
-						commitResolution = this.transaction.abort();
-						this.transaction = null; // immediately clear transaction, no need to wait
+						commitResolution = transaction.abort();
 					}
 				}
 			}
@@ -220,8 +256,7 @@ export class DatabaseTransaction implements Transaction {
 				const completions = [];
 				return commitResolution.then(
 					() => {
-						this.transaction.onCommit?.();
-						this.transaction = null; // the native transaction is done (reset if needed)
+						transaction.onCommit?.();
 						if (this.next) {
 							completions.push(this.next.commit(options));
 						}
@@ -260,7 +295,7 @@ export class DatabaseTransaction implements Transaction {
 							// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
 							// for future transactions
 							this.retries++;
-							return this.commit(options); // try again
+							return this.commit({ transaction }); // try again
 						} else throw error;
 					}
 				);
@@ -318,7 +353,7 @@ export class ImmediateTransaction extends DatabaseTransaction {
 	save(transaction: ImmediateTransaction) {
 		if (this.isCommitting) {
 			// if we are in the commit, do the save and force a reload so we get a read within the transaction
-			super.save(transaction, true);
+			super.save(transaction, null, true);
 		} else {
 			this.isCommitting = true;
 			return when(this.commit(), () => {

package/core/resources/Table.ts

@@ -580,10 +580,16 @@ export function makeTable(options) {
 							// dictates not to go to source
 							if (!this.doesExist()) throw new ServerError('Entry is not cached', 504);
 						} else if (resourceOptions?.ensureLoaded) {
-							const loadingFromSource = ensureLoadedFromSource(this.constructor.source, id, entry, request, this);
+							const loadingFromSource = ensureLoadedFromSource(
+								this.constructor.source,
+								id,
+								entry,
+								request,
+								this,
+								target
+							);
 							if (loadingFromSource) {
 								txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
-								target.loadedFromSource = true;
 								return when(loadingFromSource, (entry) => {
 									TableResource._updateResource(this, entry);
 									return this;
@@ -988,10 +994,16 @@ export function makeTable(options) {
 									// dictates not to go to source
 									if (!entry?.value) throw new ServerError('Entry is not cached', 504);
 								} else if (ensureLoaded) {
-									const loadingFromSource = ensureLoadedFromSource(constructor.source, id, entry, context, this);
+									const loadingFromSource = ensureLoadedFromSource(
+										constructor.source,
+										id,
+										entry,
+										context,
+										this,
+										target
+									);
 									if (loadingFromSource) {
 										txn?.disregardReadTxn(); // this could take some time, so don't keep the transaction open if possible
-										target.loadedFromSource = true;
 										return loadingFromSource.then((entry) => entry?.value);
 									}
 								}
@@ -3732,7 +3744,7 @@ export function makeTable(options) {
 		}
 	}
 
-	function ensureLoadedFromSource(source: typeof TableResource, id, entry, context, resource?) {
+	function ensureLoadedFromSource(source: typeof TableResource, id, entry, context, resource?, target?) {
 		if (hasSourceGet) {
 			let needsSourceData = false;
 			if (context.noCache) needsSourceData = true;
@@ -3751,7 +3763,7 @@ export function makeTable(options) {
 				recordActionBinary(!needsSourceData, 'cache-hit', tableName);
 			}
 			if (needsSourceData) {
-				const loadingFromSource = getFromSource(source, id, entry, context).then((entry) => {
+				const loadingFromSource = getFromSource(source, id, entry, context, target).then((entry) => {
 					if (entry?.value && entry?.value.getRecord?.())
 						logger.error?.('Can not assign a record that is already a resource');
 					if (context) {
@@ -3921,7 +3933,8 @@ export function makeTable(options) {
 		source: typeof TableResource,
 		id: Id,
 		existingEntry: Entry,
-		context: Context
+		context: Context,
+		target?
 	): Promise<Entry> {
 		const metadataFlags = existingEntry?.metadataFlags;
 
@@ -3943,9 +3956,13 @@ export function makeTable(options) {
 				entry.metadataFlags & (INVALIDATED | EVICTED) ||
 				(entry.expiresAt != undefined && entry.expiresAt < Date.now())
 			)
-				// try again
-				whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context));
-			else whenResolved(entry);
+				// try again — entry still not valid, need to actually fetch from source
+				whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context, target));
+			else {
+				// served from cache after waiting for another request to resolve
+				if (target) target.loadedFromSource = false;
+				whenResolved(entry);
+			}
 		};
 		const lockAcquired = primaryStore.tryLock(id, callback);
 
@@ -3957,6 +3974,8 @@ export function makeTable(options) {
 				}, LOCK_TIMEOUT);
 			});
 		}
+		// lock acquired — this request will actually load from source
+		if (target) target.loadedFromSource = true;
 
 		const existingRecord = existingEntry?.value;
 		// it is important to remember that this is _NOT_ part of the current transaction; nothing is changing

package/core/security/jsLoader.ts

@@ -56,6 +56,9 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
 			}
 			overridableProperty(Promise.prototype, 'then');
 			overridableProperty(Date, 'now');
+			for (let name of ['get', 'set', 'has', 'delete', 'clear', 'forEach', 'entries', 'keys', 'values']) {
+				overridableProperty(Map.prototype, name);
+			}
 			for (let Intrinsic of [
 				Object,
 				Array,
@@ -121,12 +124,13 @@ export async function scopedImport(filePath: string | URL, scope?: ApplicationSc
 
 let amaro: typeof import('amaro') | undefined;
 /**
- * Strip TypeScript types using the amaro library (what Node.js uses internally)
- * Falls back to regex-based stripping if amaro is not available
+ * Strip TypeScript types synchronously using the amaro library (what Node.js uses internally)
  */
-async function stripTypeScriptTypes(source: string): Promise<string> {
+function stripTypeScriptTypes(source: string): string {
 	// Use amaro - the library that Node.js uses internally for type stripping
-	amaro = await import('amaro');
+	if (!amaro) {
+		amaro = require('amaro');
+	}
 	return amaro.transformSync(source, { mode: 'strip-only' }).code;
 }
 
@@ -146,13 +150,27 @@ function parseJsonModule(source: string, url: string): any {
  * Load a module using Node's vm.Module API with (not really secure) sandboxing
  */
 async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
-	const moduleCache = new Map<string, Promise<SourceTextModule | SyntheticModule>>();
-	const linkingPromises = new Map<string, Promise<void>>();
-	const cjsCache = new Map<string, { exports: any }>();
-
-	// Create a secure context with limited globals
-	const contextObject = getGlobalObject(scope);
-	const context = createContext(contextObject);
+	// we want to retain the same module caches across any loading with the application scope
+	let moduleCaches = scope.moduleCache as {
+		moduleCache: Map<string, SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule>>;
+		linkingPromises: Map<string, Promise<void>>;
+		cjsCache: Map<string, { exports: any }>;
+		contextObject: any;
+		context: any;
+	};
+	if (!moduleCaches) {
+		// if they haven't been initialized, do so now
+		const contextObject = getGlobalObject(scope, true);
+		moduleCaches = scope.moduleCache = {
+			moduleCache: new Map(),
+			linkingPromises: new Map(),
+			cjsCache: new Map(),
+			// Create a secure context with limited globals
+			contextObject,
+			context: createContext(contextObject),
+		};
+	}
+	const { moduleCache, linkingPromises, cjsCache, contextObject, context } = moduleCaches;
 
 	/**
 	 * Resolve module specifier to absolute URL
@@ -298,9 +316,11 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 	}
 
 	/**
-	 * Linker function for module resolution during instantiation
+	 * Linker function for module resolution during instantiation.
+	 * This is synchronous because Node's module.link() requires the linker
+	 * to return modules synchronously.
 	 */
-	async function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
+	function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
 		const resolvedUrl = resolveModule(specifier, referencingModule.identifier);
 
 		// Determine if we should use VM containment for this module
@@ -316,15 +336,15 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 			}
 		}
 
-		// Return the module immediately (even if not yet linked) to support circular dependencies
-		return await getOrCreateModule(resolvedUrl, useContainment);
+		// Return the module
+		return getOrCreateModule(resolvedUrl, useContainment);
 	}
 
-	async function getOrCreateModule(
+	function getOrCreateModule(
 		url: string,
 		usePrivateGlobal: boolean
-	): Promise<SourceTextModule | SyntheticModule> {
-		// Check cache first - return cached module immediately (even if not linked yet)
+	): SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule> {
+		// Check if module is already created
 		if (moduleCache.has(url)) {
 			return moduleCache.get(url)!;
 		}
@@ -345,8 +365,15 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 		// Only link/evaluate once per module
 		if (!linkingPromises.has(url)) {
 			const linkingPromise = (async () => {
-				await module.link(linker);
-				await module.evaluate();
+				// Check module status - only link if it's 'unlinked'
+				// Status can be: 'unlinked', 'linking', 'linked', 'evaluating', 'evaluated'
+				if (module.status === 'unlinked') {
+					await module.link(linker);
+				}
+				// Only evaluate if not already evaluated
+				if (module.status === 'linked') {
+					await module.evaluate();
+				}
 			})();
 			linkingPromises.set(url, linkingPromise);
 		}
@@ -357,94 +384,107 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 		return module;
 	}
 	/**
-	 * Create a module from URL without linking or evaluating
+	 * Create a SyntheticModule from exported object
 	 */
-	async function createModule(url: string, usePrivateGlobal: boolean): Promise<SourceTextModule | SyntheticModule> {
-		let module: SourceTextModule | SyntheticModule;
-
-		// Handle special built-in modules
-		if (url === 'harper') {
-			let harperExports = getHarperExports(scope);
-			module = new SyntheticModule(
-				Object.keys(harperExports),
-				function () {
-					for (let key in harperExports) {
-						this.setExport(key, harperExports[key]);
-					}
-				},
-				{ identifier: url, context }
-			);
-		} else if (url.startsWith('file://') && usePrivateGlobal) {
-			checkAllowedModulePath(url, scope.verifyPath);
-			let source = await readFile(new URL(url), { encoding: 'utf-8' });
-
-			// Handle JSON modules as a SyntheticModule with a default export.
-			// JSON imports only support default exports per the ESM spec.
-			if (url.endsWith('.json')) {
-				const jsonData = parseJsonModule(source, url);
-				module = new SyntheticModule(
-					['default'],
-					function () {
-						this.setExport('default', jsonData);
-					},
-					{ identifier: url, context }
-				);
-			} else {
-				// Strip TypeScript types if this is a .ts file
-				if (url.endsWith('.ts') || url.endsWith('.tsx')) {
-					source = await stripTypeScriptTypes(source);
+	function createSyntheticModule(url: string, exportedObject: any): SyntheticModule {
+		const exportNames = Object.keys(exportedObject);
+		return new SyntheticModule(
+			exportNames,
+			function () {
+				for (const key of exportNames) {
+					this.setExport(key, exportedObject[key]);
 				}
+			},
+			{ identifier: url, context }
+		);
+	}
 
-				// Try CJS first since it will fail fast with clear syntax errors on ESM syntax
-				try {
-					module = loadCJSModule(url, source, usePrivateGlobal);
-				} catch {
-					// If CJS loading fails (likely due to ESM syntax like import/export), try ESM
-					try {
-						module = new SourceTextModule(source, {
-							identifier: url,
-							context,
-							initializeImportMeta(meta) {
-								meta.url = url;
-							},
-							async importModuleDynamically(specifier: string) {
-								const resolvedUrl = resolveModule(specifier, url);
-								const dynamicModule = await loadModuleWithCache(resolvedUrl, true);
-								return dynamicModule;
-							},
-						});
-					} catch (esmErr) {
-						// Both failed - throw the ESM error as it's likely more relevant
-						throw esmErr;
-					}
-				}
-			}
-		} else {
-			const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
-			// For Node.js built-in modules (node:) and npm packages without dependency containment
-			// Always try require first to properly handle CJS modules with named exports
-			// Fall back to dynamic import for ESM packages
-			let importedModule = replacedModule ?? (await import(url));
-			const cjsModule = importedModule['module.exports'];
-			if (cjsModule) {
-				// back-compat import
-				importedModule = importedModule.default ? { default: importedModule.default, ...cjsModule } : cjsModule;
-			}
-			const exportNames = Object.keys(importedModule);
-			module = new SyntheticModule(
-				exportNames,
+	/**
+	 * Normalize imported module to ensure it has proper exports including default
+	 */
+	function normalizeImportedModule(importedModule: any): any {
+		const cjsModule = importedModule['module.exports'];
+		if (cjsModule) {
+			// back-compat import
+			importedModule = importedModule.default ? { default: importedModule.default, ...cjsModule } : cjsModule;
+		}
+		// Ensure there's a default export for ESM imports that expect it
+		if (!importedModule.default) {
+			importedModule = { default: importedModule, ...importedModule };
+		}
+		return importedModule;
+	}
+
+	/**
+	 * Create a SourceTextModule or SyntheticModule from source code
+	 */
+	function createModuleFromSource(
+		url: string,
+		source: string,
+		usePrivateGlobal: boolean
+	): SourceTextModule | SyntheticModule {
+		// Handle JSON modules
+		if (url.endsWith('.json')) {
+			const jsonData = parseJsonModule(source, url);
+			return new SyntheticModule(
+				['default'],
 				function () {
-					for (const key of exportNames) {
-						this.setExport(key, importedModule[key]);
-					}
+					this.setExport('default', jsonData);
 				},
 				{ identifier: url, context }
 			);
 		}
 
-		return module;
+		// Strip TypeScript types if this is a .ts file
+		if (url.endsWith('.ts') || url.endsWith('.tsx')) {
+			source = stripTypeScriptTypes(source);
+		}
+
+		// Try CJS first since it will fail fast with clear syntax errors on ESM syntax
+		try {
+			return loadCJSModule(url, source, usePrivateGlobal);
+		} catch {
+			// If CJS loading fails (likely due to ESM syntax like import/export), try ESM
+			return new SourceTextModule(source, {
+				identifier: url,
+				context,
+				initializeImportMeta(meta) {
+					meta.url = url;
+				},
+				importModuleDynamically(specifier: string) {
+					const resolvedUrl = resolveModule(specifier, url);
+					const useContainment = specifier.startsWith('.') || scope.dependencyContainment !== false;
+					return loadModuleWithCache(resolvedUrl, useContainment);
+				},
+			});
+		}
 	}
 
+	/**
+	 * Create a module from URL without linking or evaluating (async version for initial load)
+	 */
+	function createModule(
+		url: string,
+		usePrivateGlobal: boolean
+	): SourceTextModule | SyntheticModule | Promise<SourceTextModule | SyntheticModule> {
+		// Handle special built-in modules
+		if (url === 'harper') {
+			return createSyntheticModule(url, getHarperExports(scope));
+		}
+
+		if (url.startsWith('file://') && usePrivateGlobal) {
+			checkAllowedModulePath(url, scope.verifyPath);
+			const source = readFileSync(new URL(url), { encoding: 'utf-8' });
+			return createModuleFromSource(url, source, usePrivateGlobal);
+		}
+
+		// For Node.js built-in modules (node:) and npm packages without dependency containment
+		const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
+		if (replacedModule) {
+			return createSyntheticModule(url, normalizeImportedModule(replacedModule));
+		}
+		return import(url).then((importedModule) => createSyntheticModule(url, normalizeImportedModule(importedModule)));
+	}
 	// Load the entry module
 	const entryModule = await loadModuleWithCache(moduleUrl, true);
 
@@ -490,8 +530,8 @@ async function getCompartment(scope: ApplicationScope, globals) {
 						},
 					};
 				} else if (moduleSpecifier.startsWith('file:') && !moduleSpecifier.includes('node_modules')) {
-					const moduleText = await readFile(new URL(moduleSpecifier), { encoding: 'utf-8' });
-					// Handle JSON files in comparttment mode the same way as in VM mode
+					let moduleText = await readFile(new URL(moduleSpecifier), { encoding: 'utf-8' });
+					// Handle JSON files in compartment mode the same way as in VM mode
 					if (moduleSpecifier.endsWith('.json')) {
 						const jsonData = parseJsonModule(moduleText, moduleSpecifier);
 						return {
@@ -502,6 +542,10 @@ async function getCompartment(scope: ApplicationScope, globals) {
 							},
 						};
 					}
+					// Strip TypeScript types if this is a .ts file
+					if (moduleSpecifier.endsWith('.ts') || moduleSpecifier.endsWith('.tsx')) {
+						moduleText = stripTypeScriptTypes(moduleText);
+					}
 					return new StaticModuleRecord(moduleText, moduleSpecifier);
 				} else {
 					checkAllowedModulePath(moduleSpecifier, scope.verifyPath);
@@ -536,6 +580,9 @@ function secureOnlyFetch(resource, options) {
 	return fetch(resource, options);
 }
 
+// These globals need to match the literals produced in the VM context
+const contextualizedJSGlobals = ['Object', 'Array', 'Function', 'globalThis'];
+
 let defaultJSGlobalNames: string[];
 // get the global variable names that are intrinsically present in a VM context (so we don't override them)
 function getDefaultJSGlobalNames() {
@@ -551,12 +598,13 @@ function getDefaultJSGlobalNames() {
 /**
  * Get the set of global variables that should be available to modules that run in scoped compartments/contexts.
  */
-function getGlobalObject(scope: ApplicationScope) {
+function getGlobalObject(scope: ApplicationScope, copyIntrinsics = false) {
 	const appGlobal = {};
 	// create the new global object, assigning all the global variables from this global
 	// except those that will be natural intrinsics of the new VM
+	const globalsToExclude = copyIntrinsics ? contextualizedJSGlobals : getDefaultJSGlobalNames();
 	for (let name of Object.getOwnPropertyNames(global)) {
-		if (getDefaultJSGlobalNames().includes(name)) continue;
+		if (globalsToExclude.includes(name)) continue;
 		appGlobal[name] = global[name];
 	}
 	// now assign Harper scope-specific variables