@harperfast/harper-pro 5.0.0-beta.5 → 5.0.0-beta.7

package/core/bin/harper.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S node --disable-warning=ExperimentalWarning --experimental-vm-modules
+#!/usr/bin/env node
 'use strict';
 
 const fs = require('node:fs');

package/core/bin/run.js

@@ -266,7 +266,7 @@ function startupLog(portResolutions) {
 			? `, TCP: ${env.get(CONFIG_PARAMS.THREADS_DEBUG_PORT)}\n`
 			: '\n';
 	}
-	const logFilePath = path.join(env.get(CONFIG_PARAMS.LOGGING_ROOT), 'hdb.log');
+	const logFilePath = path.join(configUtils.getConfigPath(CONFIG_PARAMS.LOGGING_ROOT), 'hdb.log');
 	logMsg += `${pad('Logging:')}level: ${env.get(CONFIG_PARAMS.LOGGING_LEVEL)}, location: ${
 		logFilePath + (env.get(CONFIG_PARAMS.LOGGING_STDSTREAMS) ? ', stdout/err' : '')
 	}\n`;
@@ -292,7 +292,7 @@ function startupLog(portResolutions) {
 			? `enabled for ${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST)}`
 			: 'disabled'
 	}`;
-	logMsg += `, unix socket: ${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)}\n`;
+	logMsg += `, unix socket: ${configUtils.getConfigPath(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)}\n`;
 	if (env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT)) {
 		logMsg +=
 			pad('') +

package/core/components/Application.ts

@@ -1,5 +1,5 @@
 import { type Logger } from '../utility/logging/logger.ts';
-import { getConfigObj, getConfigValue } from '../config/configUtils.js';
+import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.js';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
 import logger from '../utility/logging/harper_logger.js';
 
@@ -359,7 +359,8 @@ export async function installApplication(application: Application) {
 		application.name,
 		(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + 'npm',
 		['install', '--force'],
-		application.dirPath
+		application.dirPath,
+		application.install?.timeout
 	);
 
 	// if it succeeds, return
@@ -401,7 +402,9 @@ export class Application {
 		this.payload = payload;
 		this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
 		this.install = install;
-		this.dirPath = join(getConfigValue(CONFIG_PARAMS.COMPONENTSROOT), name);
+		const componentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
+		if (!componentsRoot) throw new Error('componentsRoot is not configured');
+		this.dirPath = join(componentsRoot, name);
 		this.logger = logger.loggerWithTag(name);
 		this.packageManagerPrefix = getConfigValue(CONFIG_PARAMS.APPLICATIONS_PACKAGEMANAGERPREFIX);
 	}
@@ -463,7 +466,8 @@ export async function installApplications() {
 
 	const config = getConfigObj();
 
-	const componentsRootDirPath = getConfigValue(CONFIG_PARAMS.COMPONENTSROOT);
+	const componentsRootDirPath = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
+	if (!componentsRootDirPath) throw new Error('componentsRoot is not configured');
 
 	// Ensure component directory exists
 	await mkdir(componentsRootDirPath, { recursive: true });
@@ -554,7 +558,7 @@ export function nonInteractiveSpawn(
 	command: string,
 	args: string[],
 	cwd: string,
-	timeoutMs: number = 5 * 60 * 1000
+	timeoutMs: number = 60 * 60 * 1000
 ): Promise<{ stdout: string; stderr: string; code: number }> {
 	return new Promise((resolve, reject) => {
 		logger

package/core/components/ApplicationScope.ts

@@ -1,6 +1,6 @@
 import type { Resources } from '../resources/Resources.ts';
 import { type Server } from '../server/Server.ts';
-import { loggerWithTag } from '../utility/logging/harper_logger.js';
+import { forComponent } from '../utility/logging/harper_logger.js';
 import { scopedImport } from '../security/jsLoader.ts';
 import * as env from '../utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
@@ -23,8 +23,8 @@ export class ApplicationScope {
 	dependencyContainment?: boolean; // option to set this from the scope
 	verifyPath?: string;
 	config: any;
-	constructor(name: string, resources: Resources, server: Server, verifyPath?: string) {
-		this.logger = loggerWithTag(name);
+	constructor(name: string, resources: Resources, server: Server, isInternal = false, verifyPath?: string) {
+		this.logger = forComponent(name, !isInternal);
 
 		this.resources = resources;
 		this.server = server;

package/core/components/componentLoader.ts

@@ -1,10 +1,11 @@
+import { onMessageByType } from '../server/threads/manageThreads.js';
 import { readdirSync, readFileSync, existsSync, realpathSync, mkdirSync, rmSync, symlinkSync } from 'node:fs';
 import { join, basename, dirname } from 'node:path';
 import { isMainThread } from 'node:worker_threads';
 import { parseDocument } from 'yaml';
 import * as env from '../utility/environment/environmentManager.js';
 import { PACKAGE_ROOT } from '../utility/packageUtils.js';
-import { CONFIG_PARAMS, HDB_ROOT_DIR_NAME } from '../utility/hdbTerms.ts';
+import { CONFIG_PARAMS, HDB_ROOT_DIR_NAME, ITC_EVENT_TYPES } from '../utility/hdbTerms.ts';
 import * as graphqlHandler from '../resources/graphql.ts';
 import * as graphqlQueryHandler from '../server/graphqlQuerying.ts';
 import * as roles from '../resources/roles.ts';
@@ -26,7 +27,7 @@ import { getHdbBasePath } from '../utility/environment/environmentManager.js';
 import * as operationsServer from '../server/operationsServer.ts';
 import * as auth from '../security/auth.ts';
 import * as mqtt from '../server/mqtt.ts';
-import { getConfigObj, resolvePath } from '../config/configUtils.js';
+import { getConfigObj, getConfigPath } from '../config/configUtils.js';
 import { createReuseportFd } from '../server/serverHelpers/Request.ts';
 import { ErrorResource } from '../resources/ErrorResource.ts';
 import { Scope } from './Scope.ts';
@@ -39,7 +40,7 @@ import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts';
 import { PluginModule } from './PluginModule.ts';
 import { getEnvBuiltInComponents } from './Application.ts';
 
-const CF_ROUTES_DIR = resolvePath(env.get(CONFIG_PARAMS.COMPONENTSROOT));
+const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
 let loadedComponents = new Map<any, any>();
 let watchesSetup;
 let resources;
@@ -146,20 +147,28 @@ function symlinkHarperModule(componentDirectory: string) {
 					mkdirSync(nodeModulesDir);
 				}
 
-				// validate harperdb module
+				// validate harper module
 				const harperModule = join(nodeModulesDir, 'harper');
 				if (existsSync(harperModule)) {
-					if (realpathSync(harperModule) === realpathSync(PACKAGE_ROOT)) {
-						// if it exists and correctly linked, resolve
-						return resolve();
+					if (realpathSync(harperModule) !== realpathSync(PACKAGE_ROOT)) {
+						// if it exists but is incorrectly linked, fix it
+						rmSync(harperModule, { recursive: true, force: true });
+						// create link to harper module
+						symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
 					}
-
-					// Otherwise remove it then link
-					rmSync(harperModule, { recursive: true, force: true });
+				} else {
+					// create link to harper module
+					symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
+				}
+				// if there is a harperdb module, fix that too
+				const harperdbModule = join(nodeModulesDir, 'harperdb');
+				if (existsSync(harperdbModule) && realpathSync(harperdbModule) !== realpathSync(PACKAGE_ROOT)) {
+					// if it exists but is incorrectly linked, fix it
+					rmSync(harperdbModule, { recursive: true, force: true });
+					// create link to harper module
+					symlinkSync(PACKAGE_ROOT, harperdbModule, 'dir');
 				}
 
-				// create link to harperdb module
-				symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
 				resolve();
 			} finally {
 				// finally release the lock
@@ -259,7 +268,7 @@ export async function loadComponent(
 
 	const {
 		providedLoadedComponents,
-		applicationScope = new ApplicationScope(basename(componentDirectory), resources, server),
+		applicationScope = new ApplicationScope(basename(componentDirectory), resources, server, options.isRoot),
 		isRoot,
 		autoReload,
 		appName,
@@ -310,7 +319,9 @@ export async function loadComponent(
 			// Initialize loading status for all components (applications and extensions)
 			componentLifecycle.loading(componentStatusName);
 
-			const subApplicationScope = isRoot ? new ApplicationScope(componentName, resources, server) : applicationScope;
+			const subApplicationScope = isRoot
+				? new ApplicationScope(componentName, resources, server, TRUSTED_RESOURCE_PLUGINS.hasOwnProperty(componentName))
+				: applicationScope;
 
 			let extensionModule: any;
 			const pkg = componentConfig.package;
@@ -332,7 +343,7 @@ export async function loadComponent(
 						}
 					}
 					if (componentPath) {
-						subApplicationScope.verifyPath = componentPath;
+						subApplicationScope.verifyPath ??= componentPath;
 						if (!process.env.HARPER_SAFE_MODE) {
 							extensionModule = await loadComponent(componentPath, resources, origin, {
 								isRoot: false,
@@ -392,6 +403,8 @@ export async function loadComponent(
 				if (resources.isWorker && extensionModule.handleApplication) {
 					const scope = new Scope(appName || 'harper', componentName, componentDirectory, configPath, applicationScope);
 
+					onMessageByType(ITC_EVENT_TYPES.SHUTDOWN, () => scope.close());
+
 					await sequentiallyHandleApplication(scope, extensionModule);
 
 					// Mark component as loaded after successful handleApplication call

package/core/components/operations.js

@@ -31,7 +31,7 @@ function customFunctionsStatus() {
 	try {
 		response = {
 			port: env.get(hdbTerms.CONFIG_PARAMS.HTTP_PORT),
-			directory: env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT),
+			directory: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT),
 			is_enabled: true,
 		};
 	} catch (err) {
@@ -54,7 +54,7 @@ function customFunctionsStatus() {
 function getCustomFunctions() {
 	log.trace(`getting custom api endpoints`);
 	let response = {};
-	const dir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const dir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 
 	try {
 		const projectFolders = fg.sync(normalize(`${dir}/*`), { onlyDirectories: true });
@@ -103,7 +103,7 @@ function getCustomFunction(req) {
 	}
 
 	log.trace(`getting custom api endpoint file content`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file } = req;
 	const fileLocation = path.join(cfDir, project, type, file + '.js');
 
@@ -141,7 +141,7 @@ async function setCustomFunction(req) {
 	}
 
 	log.trace(`setting custom function file content`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file, function_content } = req;
 
 	try {
@@ -181,7 +181,7 @@ async function dropCustomFunction(req) {
 	}
 
 	log.trace(`dropping custom function file`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file } = req;
 
 	try {
@@ -216,7 +216,7 @@ async function addComponent(req) {
 	}
 
 	log.trace(`adding component`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, install_command, install_timeout } = req;
 
 	const template = req.template || 'https://github.com/harperdb/application-template';
@@ -264,7 +264,7 @@ async function dropCustomFunctionProject(req) {
 	}
 
 	log.trace(`dropping custom function project`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project } = req;
 
 	let apps = env.get(hdbTerms.CONFIG_PARAMS.APPS);
@@ -318,7 +318,7 @@ async function packageComponent(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project } = req;
 	log.trace(`packaging component`, project);
 
@@ -507,8 +507,8 @@ async function getComponents() {
 		}
 	};
 
-	const results = await walkDir(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), {
-		name: env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
+	const results = await walkDir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), {
+		name: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
 		entries: [],
 	});
 	for (let entry of results.entries) {
@@ -557,7 +557,7 @@ async function getComponentFile(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
-	const compRoot = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const compRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const options = req.encoding ? { encoding: req.encoding } : { encoding: 'utf8' };
 
 	try {
@@ -588,7 +588,7 @@ async function setComponentFile(req) {
 	}
 
 	const options = req.encoding ? { encoding: req.encoding } : { encoding: 'utf8' };
-	const pathToComp = path.join(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project, req.file);
+	const pathToComp = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project, req.file);
 	if (req.payload !== undefined) {
 		await fs.ensureFile(pathToComp);
 		await fs.outputFile(pathToComp, req.payload, options);
@@ -613,7 +613,7 @@ async function dropComponent(req) {
 
 	const { project, file } = req;
 	const projectPath = req.file ? path.join(project, file) : project;
-	const pathToComponent = path.join(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
+	const pathToComponent = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
 
 	const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project);
 	if (await fs.pathExists(componentSymlink)) {

package/core/components/operationsValidation.js

@@ -4,9 +4,9 @@ const Joi = require('joi');
 const fs = require('fs-extra');
 const path = require('path');
 const validator = require('../validation/validationWrapper.js');
-const envMangr = require('../utility/environment/environmentManager.js');
 const hdbTerms = require('../utility/hdbTerms.ts');
 const hdbLogger = require('../utility/logging/harper_logger.js');
+const configUtils = require('../config/configUtils.js');
 const { hdbErrors } = require('../utility/errors/hdbError.js');
 const { HDB_ERROR_MSGS } = hdbErrors;
 
@@ -34,7 +34,7 @@ module.exports = {
  */
 function checkProjectExists(checkExists, project, helpers) {
 	try {
-		const cfDir = envMangr.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+		const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 		const projectDir = path.join(cfDir, project);
 
 		if (!fs.existsSync(projectDir)) {
@@ -71,7 +71,7 @@ function checkFilePath(path, helpers) {
  */
 function checkFileExists(project, type, file, helpers) {
 	try {
-		const cfDir = envMangr.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+		const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 		const filePath = path.join(cfDir, project, type, file + '.js');
 		if (!fs.existsSync(filePath)) {
 			return helpers.message(HDB_ERROR_MSGS.NO_FILE);

package/core/config/configUtils.js

@@ -57,6 +57,7 @@ exports.deleteConfigFromFile = deleteConfigFromFile;
 exports.getConfigObj = getConfigObj;
 exports.resolvePath = resolvePath;
 exports.getFlatConfigObj = getFlatConfigObj;
+exports.getConfigPath = getConfigPath;
 
 function resolvePath(relativePath) {
 	if (relativePath?.startsWith('~/')) {
@@ -70,7 +71,23 @@ function resolvePath(relativePath) {
 		return relativePath;
 	}
 }
-
+/**
+ * Get a config value and resolve it as a path relative to rootPath.
+ * Use this for any config param that represents a file/directory path.
+ * @param param
+ */
+function getConfigPath(param) {
+	const env = require('../utility/environment/environmentManager.js');
+	const value = env.get(param);
+	if (!value || typeof value !== 'string') return value;
+	if (value.startsWith('~/')) {
+		return path.join(hdbUtils.getHomeDir(), value.slice(1));
+	}
+	if (path.isAbsolute(value)) return value;
+	const rootPath = env.getHdbBasePath();
+	if (!rootPath) return value;
+	return path.resolve(rootPath, value);
+}
 /**
  * Builds the Harper config file using user inputs and default values from defaultConfig.yaml
  * @param args - any args that the user provided.
@@ -332,15 +349,14 @@ function initConfig(force = false) {
  * @param configFilePath
  */
 function checkForUpdatedConfig(configDoc, configFilePath) {
-	const rootPath = configDoc.getIn(['rootPath']);
 	let updateFile = false;
 	if (!configDoc.hasIn(['storage', 'path'])) {
-		configDoc.setIn(['storage', 'path'], path.join(rootPath, 'database'));
+		configDoc.setIn(['storage', 'path'], 'database');
 		updateFile = true;
 	}
 
 	if (!configDoc.hasIn(['logging', 'rotation', 'path'])) {
-		configDoc.setIn(['logging', 'rotation', 'path'], path.join(rootPath, 'log'));
+		configDoc.setIn(['logging', 'rotation', 'path'], 'log');
 		updateFile = true;
 	}
 

package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js

@@ -7,6 +7,7 @@ const path = require('path');
 const minimist = require('minimist');
 const fs = require('fs-extra');
 const _ = require('lodash');
+const { getConfigPath } = require('../../../../config/configUtils.js');
 env.initSync();
 
 const { CONFIG_PARAMS, DATABASES_PARAM_CONFIG, SYSTEM_SCHEMA_NAME } = hdbTerms;
@@ -25,7 +26,7 @@ function getBaseSchemaPath() {
 
 	if (env.getHdbBasePath() !== undefined) {
 		BASE_SCHEMA_PATH =
-			env.get(CONFIG_PARAMS.STORAGE_PATH) || path.join(env.getHdbBasePath(), hdbTerms.DATABASES_DIR_NAME);
+			getConfigPath(CONFIG_PARAMS.STORAGE_PATH) || path.join(env.getHdbBasePath(), hdbTerms.DATABASES_DIR_NAME);
 		return BASE_SCHEMA_PATH;
 	}
 }
@@ -52,7 +53,7 @@ function getTransactionAuditStoreBasePath() {
 
 	if (env.getHdbBasePath() !== undefined) {
 		TRANSACTION_STORE_PATH =
-			env.get(hdbTerms.CONFIG_PARAMS.STORAGE_AUDIT_PATH) ||
+			getConfigPath(hdbTerms.CONFIG_PARAMS.STORAGE_AUDIT_PATH) ||
 			path.join(env.getHdbBasePath(), hdbTerms.TRANSACTIONS_DIR_NAME);
 		return TRANSACTION_STORE_PATH;
 	}

package/core/resources/Resource.ts

@@ -683,15 +683,26 @@ function transactional(
 		}
 	}
 }
-function missingMethod(resource, method) {
-	const error = new ClientError(`The ${resource.constructor.name} does not have a ${method} method implemented`, 405);
-	error.allow = [];
+const KNOWN_METHODS = ['get', 'head', 'put', 'post', 'delete', 'patch', 'query', 'move', 'copy'];
+type ClientErrorWithMethods = ClientError & { allow: string[]; method: string };
+export function missingMethod(resource: any, method: string) {
+	const error = new ClientError(
+		`The ${resource.constructor.name} does not have a ${method} method implemented`,
+		405
+	) as ClientErrorWithMethods;
+	error.allow = allowedMethods(resource);
 	error.method = method;
-	for (const method of ['get', 'put', 'post', 'delete', 'query', 'move', 'copy']) {
-		if (typeof resource[method] === 'function') error.allow.push(method);
-	}
 	throw error;
 }
+
+export function allowedMethods(resource: any) {
+	const allow = [];
+	for (const method of KNOWN_METHODS) {
+		if (typeof resource[method] === 'function') allow.push(method);
+	}
+	return allow;
+}
+
 /**
  * This is responsible for handling a select query parameter/call that selects specific
  * properties from the returned record(s).

package/core/resources/RocksTransactionLogStore.ts

@@ -3,6 +3,7 @@ import { ExtendedIterable } from '@harperfast/extended-iterable';
 import { Decoder, readAuditEntry, ENTRY_DATAVIEW, AuditRecord, createAuditEntry } from './auditStore.ts';
 import { isMainThread } from 'node:worker_threads';
 import { EventEmitter } from 'node:events';
+import { asBinary } from 'lmdb';
 
 if (!process.env.HARPER_NO_FLUSH_ON_EXIT && isMainThread) {
 	// we want to be able to test log replay
@@ -81,7 +82,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 
 	putSync(suggestedKey: any, value: any, options: any) {
 		if (typeof suggestedKey === 'symbol') {
-			this.rootStore.putSync(suggestedKey, value, options);
+			this.rootStore.putSync(suggestedKey, asBinary(value), options);
 		} else {
 			this.put(suggestedKey, value, options);
 		}
@@ -107,6 +108,12 @@ export class RocksTransactionLogStore extends EventEmitter {
 			return this.rootStore.getSync(key);
 		}
 	}
+	getBinary(key: any) {
+		if (typeof key === 'number') {
+			throw new Error('Unsupported binary access by number');
+		}
+		return this.rootStore.getBinarySync(key);
+	}
 	getEntry() {
 		throw new Error('Not implemented');
 	}

package/core/resources/analytics/metadata.ts

@@ -5,6 +5,7 @@ export const METRIC = {
 	MAIN_THREAD_UTILIZATION: 'main-thread-utilization',
 	RESOURCE_USAGE: 'resource-usage',
 	UTILIZATION: 'utilization',
+	NODE_STORAGE: 'node-storage',
 } as const;
 
 export type BuiltInMetricName = (typeof METRIC)[keyof typeof METRIC];

package/core/resources/analytics/read.ts

@@ -131,7 +131,7 @@ export async function get(metric: string, opts?: GetAnalyticsOpts): Promise<Metr
 		// remove nodeId from 'id' attr and resolve it to the actual hostname and
 		// add back in as 'node' attr if selected
 		const nodeId = result.id[1];
-		result['id'] = result['id'][0];
+		result = { ...result, id: result.id[0] };
 		if (isSelected(select, 'node')) {
 			log.trace?.(`get_analytics lookup hostname for nodeId: ${nodeId}`);
 			const hostname = await lookupHostname(nodeId);

package/core/resources/analytics/write.ts

@@ -3,12 +3,12 @@ import { onMessageByType } from '../../server/threads/manageThreads.js';
 import { getDatabases, table } from '../databases.ts';
 import type { Databases, Table, Tables } from '../databases.ts';
 import harperLogger from '../../utility/logging/harper_logger.js';
-import { stat } from 'node:fs/promises';
+import { stat, readdir } from 'node:fs/promises';
 const { getLogFilePath, forComponent } = harperLogger;
 import { dirname, join } from 'path';
 import { open } from 'fs/promises';
 import { getNextMonotonicTime } from '../../utility/lmdb/commonUtility.js';
-import { get as envGet, initSync } from '../../utility/environment/environmentManager.js';
+import { get as envGet, getHdbBasePath, initSync } from '../../utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts';
 import { server } from '../../server/Server.ts';
 import * as fs from 'node:fs';
@@ -382,6 +382,43 @@ function storeVolumeMetrics(analyticsTable: Table, databases: Databases) {
 	}
 }
 
+export async function getDirectorySizeAsync(dirPath: string): Promise<number> {
+	try {
+		const entries = await readdir(dirPath, { withFileTypes: true });
+		const sizes = await Promise.all(
+			entries.map((entry) => {
+				const fullPath = join(dirPath, entry.name);
+				if (entry.isDirectory()) return getDirectorySizeAsync(fullPath);
+				if (entry.isFile()) return stat(fullPath).then((s) => s.size);
+				return 0;
+			})
+		);
+		let total = 0;
+		for (const size of sizes) total += size;
+		return total;
+	} catch {
+		// directory may not exist or be inaccessible
+		return 0;
+	}
+}
+
+const DEFAULT_STORAGE_INTERVAL = 10;
+let nodeStorageInterval = DEFAULT_STORAGE_INTERVAL;
+let nodeStorageCycleCount = 0;
+
+async function storeNodeStorageMetric(analyticsTable: Table) {
+	if (nodeStorageInterval <= 0 || ++nodeStorageCycleCount % nodeStorageInterval !== 1) return;
+	try {
+		const size = await getDirectorySizeAsync(getHdbBasePath());
+		storeMetric(analyticsTable, {
+			metric: METRIC.NODE_STORAGE,
+			size,
+		});
+	} catch (error) {
+		log.warn?.('Error getting node storage metric', error);
+	}
+}
+
 async function aggregation(fromPeriod, toPeriod = 60000) {
 	const rawAnalyticsTable = getRawAnalyticsTable();
 	const analyticsTable = getAnalyticsTable();
@@ -574,6 +611,9 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 	// database storage volume metrics
 	storeVolumeMetrics(analyticsTable, databases);
 	storeVolumeMetrics(analyticsTable, { system: databases.system });
+
+	// node storage metric (total HDB directory size)
+	await storeNodeStorageMetric(analyticsTable);
 }
 let lastIdle = 0;
 let lastActive = 0;
@@ -654,6 +694,7 @@ if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics);
 let scheduledTasksRunning;
 function startScheduledTasks() {
 	scheduledTasksRunning = true;
+	nodeStorageInterval = envGet(CONFIG_PARAMS.ANALYTICS_STORAGEINTERVAL) ?? DEFAULT_STORAGE_INTERVAL;
 	const AGGREGATE_PERIOD = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1000;
 	if (AGGREGATE_PERIOD) {
 		setInterval(

package/core/resources/databases.ts

@@ -12,6 +12,7 @@ import {
 import { makeTable } from './Table.ts';
 import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
 import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
+import { getConfigPath } from '../config/configUtils.js';
 import { _assignPackageExport } from '../globals.js';
 import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
 import * as signalling from '../utility/signalling.js';
@@ -176,7 +177,7 @@ export function getDatabases(): Databases {
 	if (process.env.SCHEMAS_DATA_PATH) schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
 	databasePath =
 		process.env.STORAGE_PATH ||
-		envGet(CONFIG_PARAMS.STORAGE_PATH) ||
+		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
 		(databasePath && (existsSync(databasePath) ? databasePath : join(getHdbBasePath(), LEGACY_DATABASES_DIR_NAME)));
 	if (!databasePath) return;
 
@@ -694,7 +695,7 @@ export function database({ database: databaseName, table: tableName }) {
 		tablePath ||
 		databaseConfig[databaseName]?.path ||
 		process.env.STORAGE_PATH ||
-		envGet(CONFIG_PARAMS.STORAGE_PATH) ||
+		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
 		(existsSync(join(hdbBasePath, DATABASES_DIR_NAME))
 			? join(hdbBasePath, DATABASES_DIR_NAME)
 			: join(hdbBasePath, LEGACY_DATABASES_DIR_NAME));
@@ -734,41 +735,49 @@ export async function dropDatabase(databaseName) {
 	if (!databases[databaseName]) throw new Error('Database does not exist');
 	const dbTables = databases[databaseName];
 	let rootStore;
+
 	for (const tableName in dbTables) {
 		const table = dbTables[tableName];
 		rootStore = table.primaryStore.rootStore;
-
 		lmdbDatabaseEnvs.delete(rootStore.path);
 		rocksdbDatabaseEnvs.delete(rootStore.path);
+	}
+
+	for (const tableName in dbTables) {
+		databaseEventsEmitter.emit('dropTable', tableName, databaseName);
+	}
+
+	if (databaseName === 'data') {
+		for (const tableName in tables) {
+			delete tables[tableName];
+		}
+		delete tables[DEFINED_TABLES];
+	}
+	delete databases[databaseName];
 
+	databaseEventsEmitter.emit('dropDatabase', databaseName);
+
+	if (rootStore) {
 		if (rootStore.status === 'open') {
 			if (rootStore instanceof RocksDatabase) {
 				rootStore.close();
 				rootStore.destroy();
-			} else if (rootStore.status === 'open') {
+			} else {
 				await rootStore.close();
 				await unlink(rootStore.path);
 			}
 		}
-		databaseEventsEmitter.emit('dropTable', tableName, databaseName);
-	}
-	if (!rootStore) {
+	} else {
 		rootStore = database({ database: databaseName, table: null });
 		if (rootStore instanceof RocksDatabase) {
+			rootStore.close();
 			rootStore.destroy();
 		} else if (rootStore.status === 'open') {
 			await rootStore.close();
 			await unlink(rootStore.path);
 		}
 	}
-	if (databaseName === 'data') {
-		for (const tableName in tables) {
-			delete tables[tableName];
-		}
-		delete tables[DEFINED_TABLES];
-	}
-	delete databases[databaseName];
-	databaseEventsEmitter.emit('dropDatabase', databaseName);
+
 	await deleteRootBlobPathsForDB(rootStore);
 }
 // opens an index, consulting with custom indexes that may use alternate store configuration