@harperfast/harper 5.0.0-beta.6 → 5.0.0-beta.7

package/resources/analytics/write.ts

@@ -402,7 +402,12 @@ export async function getDirectorySizeAsync(dirPath: string): Promise<number> {
 	}
 }
 
+const DEFAULT_STORAGE_INTERVAL = 10;
+let nodeStorageInterval = DEFAULT_STORAGE_INTERVAL;
+let nodeStorageCycleCount = 0;
+
 async function storeNodeStorageMetric(analyticsTable: Table) {
+	if (nodeStorageInterval <= 0 || ++nodeStorageCycleCount % nodeStorageInterval !== 1) return;
 	try {
 		const size = await getDirectorySizeAsync(getHdbBasePath());
 		storeMetric(analyticsTable, {
@@ -689,6 +694,7 @@ if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics);
 let scheduledTasksRunning;
 function startScheduledTasks() {
 	scheduledTasksRunning = true;
+	nodeStorageInterval = envGet(CONFIG_PARAMS.ANALYTICS_STORAGEINTERVAL) ?? DEFAULT_STORAGE_INTERVAL;
 	const AGGREGATE_PERIOD = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1000;
 	if (AGGREGATE_PERIOD) {
 		setInterval(

package/resources/databases.ts

@@ -12,6 +12,7 @@ import {
 import { makeTable } from './Table.ts';
 import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
 import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
+import { getConfigPath } from '../config/configUtils.js';
 import { _assignPackageExport } from '../globals.js';
 import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
 import * as signalling from '../utility/signalling.js';
@@ -176,7 +177,7 @@ export function getDatabases(): Databases {
 	if (process.env.SCHEMAS_DATA_PATH) schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
 	databasePath =
 		process.env.STORAGE_PATH ||
-		envGet(CONFIG_PARAMS.STORAGE_PATH) ||
+		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
 		(databasePath && (existsSync(databasePath) ? databasePath : join(getHdbBasePath(), LEGACY_DATABASES_DIR_NAME)));
 	if (!databasePath) return;
 
@@ -694,7 +695,7 @@ export function database({ database: databaseName, table: tableName }) {
 		tablePath ||
 		databaseConfig[databaseName]?.path ||
 		process.env.STORAGE_PATH ||
-		envGet(CONFIG_PARAMS.STORAGE_PATH) ||
+		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
 		(existsSync(join(hdbBasePath, DATABASES_DIR_NAME))
 			? join(hdbBasePath, DATABASES_DIR_NAME)
 			: join(hdbBasePath, LEGACY_DATABASES_DIR_NAME));

package/security/jsLoader.ts

@@ -148,6 +148,7 @@ function parseJsonModule(source: string, url: string): any {
 async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 	const moduleCache = new Map<string, Promise<SourceTextModule | SyntheticModule>>();
 	const linkingPromises = new Map<string, Promise<void>>();
+	const cjsCache = new Map<string, { exports: any }>();
 
 	// Create a secure context with limited globals
 	const contextObject = getGlobalObject(scope);
@@ -165,6 +166,9 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 			// block harper/* for now (reserving for potential future use)
 			throw new Error(`Module ${specifier} is not allowed, may only access the 'harper' module`);
 		}
+		if (parts[0] === 'file:') {
+			return specifier;
+		}
 		const resolved = createRequire(referrer).resolve(specifier);
 		if (isAbsolute(resolved)) {
 			return pathToFileURL(resolved).toString();
@@ -176,7 +180,16 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 	 * Load a CommonJS module in our private context
 	 */
 	function loadCJS(url: string, source: string): { exports: any } {
+		// Check cache first to handle circular dependencies
+		if (cjsCache.has(url)) {
+			return cjsCache.get(url)!;
+		}
+
+		// Create module object and cache it immediately (before execution)
+		// This allows circular dependencies to get a reference to the incomplete module
 		const cjsModule = { exports: {} };
+		cjsCache.set(url, cjsModule);
+
 		if (url.endsWith('.json')) {
 			cjsModule.exports = parseJsonModule(source, url);
 			return cjsModule;
@@ -204,7 +217,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 			filename: url,
 			async importModuleDynamically(specifier: string, script) {
 				const resolvedUrl = resolveModule(specifier, script.sourceURL);
-				const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
+				const useContainment = specifier.startsWith('.') || scope.dependencyContainment !== false;
 				const dynamicModule = await loadModuleWithCache(resolvedUrl, useContainment);
 				return dynamicModule;
 			},
@@ -221,16 +234,18 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 	}
 	function loadCJSModule(url: string, source: string, usePrivateGlobal: boolean): SyntheticModule {
 		const cjsModule = usePrivateGlobal ? loadCJS(url, source) : { exports: require(url) };
-		const exportNames = Object.keys(cjsModule.exports);
+		let exports = cjsModule.exports;
+		if (exports.default === undefined) {
+			// provide the default export for compatibility
+			exports = { default: exports, ...exports };
+		}
+		const exportNames = Object.keys(exports);
+
 		const synModule = new SyntheticModule(
-			exportNames.length > 0 ? exportNames : ['default'],
+			exportNames,
 			function () {
-				if (exportNames.length > 0) {
-					for (const key of exportNames) {
-						this.setExport(key, cjsModule.exports[key]);
-					}
-				} else {
-					this.setExport('default', cjsModule.exports);
+				for (const key of exportNames) {
+					this.setExport(key, exports[key]);
 				}
 			},
 			{ identifier: url, context }
@@ -239,13 +254,68 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 		return synModule;
 	}
 
+	/**
+	 * Check if a package (or any of its dependencies) depends on harper
+	 * Expects a file URL like: file:///path/to/node_modules/package-name/dist/index.js
+	 */
+	function packageDependsOnHarper(fileUrl: string): boolean {
+		try {
+			// Convert file:// URL to path
+			const filePath = fileURLToPath(fileUrl);
+
+			// Find the node_modules directory and package name
+			// Example: /path/to/node_modules/package-name/dist/index.js
+			// or: /path/to/node_modules/@scope/package-name/dist/index.js
+			const nodeModulesMarker = '/node_modules/';
+			const nodeModulesIndex = filePath.lastIndexOf(nodeModulesMarker);
+			if (nodeModulesIndex === -1) return false;
+
+			// Get the part after /node_modules/
+			const afterNodeModules = filePath.substring(nodeModulesIndex + nodeModulesMarker.length);
+			const parts = afterNodeModules.split('/');
+
+			// Handle scoped packages (@scope/package-name) vs regular packages (package-name)
+			const beforeNodeModules = filePath.substring(0, nodeModulesIndex);
+			const packageRoot = parts[0].startsWith('@')
+				? join(beforeNodeModules, 'node_modules', parts[0], parts[1])
+				: join(beforeNodeModules, 'node_modules', parts[0]);
+
+			// Read package.json from the package root
+			const packageJsonPath = join(packageRoot, 'package.json');
+			const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
+
+			const deps = {
+				...packageJson.dependencies,
+				...packageJson.devDependencies,
+				...packageJson.peerDependencies,
+			};
+
+			// Check if harper is a direct dependency
+			return Object.keys(deps).some((dep) => HARPER_MODULE_IDS.has(dep));
+		} catch {
+			return false;
+		}
+	}
+
 	/**
 	 * Linker function for module resolution during instantiation
 	 */
 	async function linker(specifier: string, referencingModule: SourceTextModule | SyntheticModule) {
 		const resolvedUrl = resolveModule(specifier, referencingModule.identifier);
 
-		const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
+		// Determine if we should use VM containment for this module
+		let useContainment = specifier.startsWith('.'); // Always contain relative imports
+
+		if (!useContainment && scope.dependencyContainment !== false) {
+			// For npm packages, check if they depend on harper
+			if (resolvedUrl.startsWith('file://') && resolvedUrl.includes('node_modules')) {
+				useContainment = packageDependsOnHarper(resolvedUrl);
+			} else {
+				// Non-file URLs (bare specifiers) - use default behavior
+				useContainment = scope.dependencyContainment === true;
+			}
+		}
+
 		// Return the module immediately (even if not yet linked) to support circular dependencies
 		return await getOrCreateModule(resolvedUrl, useContainment);
 	}
@@ -351,38 +421,25 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 			}
 		} else {
 			const replacedModule = checkAllowedModulePath(url, scope.verifyPath);
-			// For Node.js built-in modules (node:) and npm packages
+			// For Node.js built-in modules (node:) and npm packages without dependency containment
 			// Always try require first to properly handle CJS modules with named exports
-			try {
-				const cjsExports = replacedModule ?? require(url);
-				// It's a CJS module - expose all properties as named exports
-				const exportNames = Object.keys(cjsExports);
-				module = new SyntheticModule(
-					exportNames.length > 0 ? [...exportNames, 'default'] : ['default'],
-					function () {
-						if (exportNames.length > 0) {
-							for (const key of exportNames) {
-								this.setExport(key, cjsExports[key]);
-							}
-						}
-						this.setExport('default', cjsExports);
-					},
-					{ identifier: url, context }
-				);
-			} catch {
-				// Fall back to dynamic import for ESM packages
-				const importedModule = await import(url);
-				const exportNames = Object.keys(importedModule);
-				module = new SyntheticModule(
-					exportNames,
-					function () {
-						for (const key of exportNames) {
-							this.setExport(key, importedModule[key]);
-						}
-					},
-					{ identifier: url, context }
-				);
+			// Fall back to dynamic import for ESM packages
+			let importedModule = replacedModule ?? (await import(url));
+			const cjsModule = importedModule['module.exports'];
+			if (cjsModule) {
+				// back-compat import
+				importedModule = importedModule.default ? { default: importedModule.default, ...cjsModule } : cjsModule;
 			}
+			const exportNames = Object.keys(importedModule);
+			module = new SyntheticModule(
+				exportNames,
+				function () {
+					for (const key of exportNames) {
+						this.setExport(key, importedModule[key]);
+					}
+				},
+				{ identifier: url, context }
+			);
 		}
 
 		return module;
@@ -533,6 +590,25 @@ function getHarperExports(scope: ApplicationScope) {
 		authenticateUser: server.authenticateUser,
 		operation: server.operation,
 		contentTypes,
+		Attribute: undefined,
+		Config: undefined,
+		ConfigValue: undefined,
+		Context: undefined,
+		FileAndURLPathConfig: undefined,
+		FilesOption: undefined,
+		FilesOptionObject: undefined,
+		IterableEventQueue: undefined,
+		Logger: undefined,
+		Query: undefined,
+		RecordObject: undefined,
+		RequestTargetOrId: undefined,
+		ResourceInterface: undefined,
+		Scope: undefined,
+		Session: undefined,
+		SourceContext: undefined,
+		SubscriptionRequest: undefined,
+		Table: undefined,
+		User: undefined,
 	};
 }
 const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES)
@@ -544,14 +620,19 @@ const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDB
 			},
 		};
 const ALLOWED_COMMANDS = new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? []);
-const REPLACED_BUILTIN_MODULES = {
-	child_process: {
-		exec: createSpawn(child_process.exec),
-		execFile: createSpawn(child_process.execFile),
-		fork: createSpawn(child_process.fork, true), // this is launching node, so deemed safe
-		spawn: createSpawn(child_process.spawn),
+const child_processConstrained = {
+	exec: createSpawn(child_process.exec),
+	execFile: createSpawn(child_process.execFile),
+	fork: createSpawn(child_process.fork, true), // this is launching node, so deemed safe
+	spawn: createSpawn(child_process.spawn),
+	execSync: function () {
+		throw new Error('execSync is not allowed');
 	},
 };
+child_processConstrained.default = child_processConstrained;
+const REPLACED_BUILTIN_MODULES = {
+	child_process: child_processConstrained,
+};
 /**
  * Creates a ChildProcess-like object for an existing process
  */

package/server/REST.ts

@@ -4,7 +4,7 @@ import * as harperLogger from '../utility/logging/harper_logger.js';
 import { ServerOptions } from 'http';
 import { ServerError, ClientError } from '../utility/errors/hdbError.js';
 import { Resources } from '../resources/Resources.ts';
-import { Resource } from '../resources/Resource.ts';
+import { Resource, missingMethod, allowedMethods } from '../resources/Resource.ts';
 import { IterableEventQueue } from '../resources/IterableEventQueue.ts';
 import { transaction } from '../resources/transaction.ts';
 import { Headers, mergeHeaders } from '../server/serverHelpers/Headers.ts';
@@ -114,29 +114,38 @@ async function http(request: Context & Request, nextHandler) {
 			switch (method) {
 				case 'GET':
 				case 'HEAD':
-					return resource.get(target, request);
+					return resource.get ? resource.get(target, request) : missingMethod(resource, 'get');
 				case 'POST':
-					return resource.post(target, request.data, request);
+					return resource.post ? resource.post(target, request.data, request) : missingMethod(resource, 'post');
 				case 'PUT':
-					return resource.put(target, request.data, request);
+					return resource.put ? resource.put(target, request.data, request) : missingMethod(resource, 'put');
 				case 'DELETE':
-					return resource.delete(target, request);
+					return resource.delete ? resource.delete(target, request) : missingMethod(resource, 'delete');
 				case 'PATCH':
-					return resource.patch(target, request.data, request);
+					return resource.patch ? resource.patch(target, request.data, request) : missingMethod(resource, 'patch');
 				case 'OPTIONS': // used primarily for CORS
-					headers.setIfNone('Allow', 'GET, HEAD, POST, PUT, DELETE, PATCH, OPTIONS, TRACE, QUERY, COPY, MOVE');
+					headers.setIfNone(
+						'Allow',
+						allowedMethods(resource)
+							.map((method) => method.toUpperCase())
+							.join(', ')
+					);
 					return;
 				case 'CONNECT':
 					// websockets? and event-stream
-					return resource.connect(target, null, request);
+					return resource.connect ? resource.connect(target, null, request) : missingMethod(resource, 'connect');
 				case 'TRACE':
 					return 'Harper is the terminating server';
 				case 'QUERY':
-					return resource.query(target, request.data, request);
+					return resource.query ? resource.query(target, request.data, request) : missingMethod(resource, 'query');
 				case 'COPY': // methods suggested from webdav RFC 4918
-					return resource.copy(target, headersObject.destination, request);
+					return resource.copy
+						? resource.copy(target, headersObject.destination, request)
+						: missingMethod(resource, 'copy');
 				case 'MOVE':
-					return resource.move(target, headersObject.destination, request);
+					return resource.move
+						? resource.move(target, headersObject.destination, request)
+						: missingMethod(resource, 'move');
 				case 'BREW': // RFC 2324
 					throw new ClientError("Harper is short and stout and can't brew coffee", 418);
 				default:

package/server/http.ts

@@ -8,7 +8,7 @@ import harperLogger from '../utility/logging/harper_logger.js';
 import { parentPort } from 'node:worker_threads';
 import env from '../utility/environment/environmentManager.js';
 import * as terms from '../utility/hdbTerms.ts';
-import { resolvePath } from '../config/configUtils.js';
+import { getConfigPath } from '../config/configUtils.js';
 import { getTicketKeys } from './threads/manageThreads.js';
 import { createTLSSelector } from '../security/keys.js';
 import { createSecureServer } from 'node:http2';
@@ -181,7 +181,7 @@ function getPorts(options) {
 
 	if (options?.usageType === 'operations-api' && env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)) {
 		ports.push({
-			port: resolvePath(env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)),
+			port: getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET),
 			secure: false,
 		});
 	}

package/server/itc/serverHandlers.js

@@ -149,7 +149,7 @@ async function componentStatusRequestHandler(event) {
 			if (originatorThreadId === undefined) {
 				hdbLogger.debug('No originator threadId, falling back to broadcast');
 			} else {
-				hdbLogger.warn(`Failed to send direct response to thread ${originatorThreadId}, falling back to broadcast`);
+				hdbLogger.debug(`Failed to send direct response to thread ${originatorThreadId}, falling back to broadcast`);
 			}
 			await sendItcEvent(responseMessage);
 		}

package/static/defaultConfig.yaml

@@ -26,7 +26,7 @@ analytics:
 applications:
   lockdown: freeze
   containment: vm
-  dependencyContainment: false
+  dependencyContainment: auto
   allowedSpawnCommands:
     - npm
     - node