@harperfast/harper-pro 5.0.0-beta.6 → 5.0.0-beta.7

package/core/bin/harper.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S node --disable-warning=ExperimentalWarning --experimental-vm-modules
+#!/usr/bin/env node
 'use strict';
 
 const fs = require('node:fs');

package/core/bin/run.js

@@ -266,7 +266,7 @@ function startupLog(portResolutions) {
 			? `, TCP: ${env.get(CONFIG_PARAMS.THREADS_DEBUG_PORT)}\n`
 			: '\n';
 	}
-	const logFilePath = path.join(env.get(CONFIG_PARAMS.LOGGING_ROOT), 'hdb.log');
+	const logFilePath = path.join(configUtils.getConfigPath(CONFIG_PARAMS.LOGGING_ROOT), 'hdb.log');
 	logMsg += `${pad('Logging:')}level: ${env.get(CONFIG_PARAMS.LOGGING_LEVEL)}, location: ${
 		logFilePath + (env.get(CONFIG_PARAMS.LOGGING_STDSTREAMS) ? ', stdout/err' : '')
 	}\n`;
@@ -292,7 +292,7 @@ function startupLog(portResolutions) {
 			? `enabled for ${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST)}`
 			: 'disabled'
 	}`;
-	logMsg += `, unix socket: ${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)}\n`;
+	logMsg += `, unix socket: ${configUtils.getConfigPath(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)}\n`;
 	if (env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT)) {
 		logMsg +=
 			pad('') +

package/core/components/Application.ts

@@ -1,5 +1,5 @@
 import { type Logger } from '../utility/logging/logger.ts';
-import { getConfigObj, getConfigValue } from '../config/configUtils.js';
+import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.js';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
 import logger from '../utility/logging/harper_logger.js';
 
@@ -402,7 +402,9 @@ export class Application {
 		this.payload = payload;
 		this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
 		this.install = install;
-		this.dirPath = join(getConfigValue(CONFIG_PARAMS.COMPONENTSROOT), name);
+		const componentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
+		if (!componentsRoot) throw new Error('componentsRoot is not configured');
+		this.dirPath = join(componentsRoot, name);
 		this.logger = logger.loggerWithTag(name);
 		this.packageManagerPrefix = getConfigValue(CONFIG_PARAMS.APPLICATIONS_PACKAGEMANAGERPREFIX);
 	}
@@ -464,7 +466,8 @@ export async function installApplications() {
 
 	const config = getConfigObj();
 
-	const componentsRootDirPath = getConfigValue(CONFIG_PARAMS.COMPONENTSROOT);
+	const componentsRootDirPath = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
+	if (!componentsRootDirPath) throw new Error('componentsRoot is not configured');
 
 	// Ensure component directory exists
 	await mkdir(componentsRootDirPath, { recursive: true });

package/core/components/componentLoader.ts

@@ -27,7 +27,7 @@ import { getHdbBasePath } from '../utility/environment/environmentManager.js';
 import * as operationsServer from '../server/operationsServer.ts';
 import * as auth from '../security/auth.ts';
 import * as mqtt from '../server/mqtt.ts';
-import { getConfigObj, resolvePath } from '../config/configUtils.js';
+import { getConfigObj, getConfigPath } from '../config/configUtils.js';
 import { createReuseportFd } from '../server/serverHelpers/Request.ts';
 import { ErrorResource } from '../resources/ErrorResource.ts';
 import { Scope } from './Scope.ts';
@@ -40,7 +40,7 @@ import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts';
 import { PluginModule } from './PluginModule.ts';
 import { getEnvBuiltInComponents } from './Application.ts';
 
-const CF_ROUTES_DIR = resolvePath(env.get(CONFIG_PARAMS.COMPONENTSROOT));
+const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
 let loadedComponents = new Map<any, any>();
 let watchesSetup;
 let resources;
@@ -147,20 +147,28 @@ function symlinkHarperModule(componentDirectory: string) {
 					mkdirSync(nodeModulesDir);
 				}
 
-				// validate harperdb module
+				// validate harper module
 				const harperModule = join(nodeModulesDir, 'harper');
 				if (existsSync(harperModule)) {
-					if (realpathSync(harperModule) === realpathSync(PACKAGE_ROOT)) {
-						// if it exists and correctly linked, resolve
-						return resolve();
+					if (realpathSync(harperModule) !== realpathSync(PACKAGE_ROOT)) {
+						// if it exists but is incorrectly linked, fix it
+						rmSync(harperModule, { recursive: true, force: true });
+						// create link to harper module
+						symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
 					}
-
-					// Otherwise remove it then link
-					rmSync(harperModule, { recursive: true, force: true });
+				} else {
+					// create link to harper module
+					symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
+				}
+				// if there is a harperdb module, fix that too
+				const harperdbModule = join(nodeModulesDir, 'harperdb');
+				if (existsSync(harperdbModule) && realpathSync(harperdbModule) !== realpathSync(PACKAGE_ROOT)) {
+					// if it exists but is incorrectly linked, fix it
+					rmSync(harperdbModule, { recursive: true, force: true });
+					// create link to harper module
+					symlinkSync(PACKAGE_ROOT, harperdbModule, 'dir');
 				}
 
-				// create link to harperdb module
-				symlinkSync(PACKAGE_ROOT, harperModule, 'dir');
 				resolve();
 			} finally {
 				// finally release the lock

package/core/components/operations.js

@@ -31,7 +31,7 @@ function customFunctionsStatus() {
 	try {
 		response = {
 			port: env.get(hdbTerms.CONFIG_PARAMS.HTTP_PORT),
-			directory: env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT),
+			directory: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT),
 			is_enabled: true,
 		};
 	} catch (err) {
@@ -54,7 +54,7 @@ function customFunctionsStatus() {
 function getCustomFunctions() {
 	log.trace(`getting custom api endpoints`);
 	let response = {};
-	const dir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const dir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 
 	try {
 		const projectFolders = fg.sync(normalize(`${dir}/*`), { onlyDirectories: true });
@@ -103,7 +103,7 @@ function getCustomFunction(req) {
 	}
 
 	log.trace(`getting custom api endpoint file content`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file } = req;
 	const fileLocation = path.join(cfDir, project, type, file + '.js');
 
@@ -141,7 +141,7 @@ async function setCustomFunction(req) {
 	}
 
 	log.trace(`setting custom function file content`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file, function_content } = req;
 
 	try {
@@ -181,7 +181,7 @@ async function dropCustomFunction(req) {
 	}
 
 	log.trace(`dropping custom function file`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, type, file } = req;
 
 	try {
@@ -216,7 +216,7 @@ async function addComponent(req) {
 	}
 
 	log.trace(`adding component`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, install_command, install_timeout } = req;
 
 	const template = req.template || 'https://github.com/harperdb/application-template';
@@ -264,7 +264,7 @@ async function dropCustomFunctionProject(req) {
 	}
 
 	log.trace(`dropping custom function project`);
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project } = req;
 
 	let apps = env.get(hdbTerms.CONFIG_PARAMS.APPS);
@@ -318,7 +318,7 @@ async function packageComponent(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
-	const cfDir = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project } = req;
 	log.trace(`packaging component`, project);
 
@@ -507,8 +507,8 @@ async function getComponents() {
 		}
 	};
 
-	const results = await walkDir(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), {
-		name: env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
+	const results = await walkDir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), {
+		name: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
 		entries: [],
 	});
 	for (let entry of results.entries) {
@@ -557,7 +557,7 @@ async function getComponentFile(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
-	const compRoot = env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const compRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const options = req.encoding ? { encoding: req.encoding } : { encoding: 'utf8' };
 
 	try {
@@ -588,7 +588,7 @@ async function setComponentFile(req) {
 	}
 
 	const options = req.encoding ? { encoding: req.encoding } : { encoding: 'utf8' };
-	const pathToComp = path.join(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project, req.file);
+	const pathToComp = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project, req.file);
 	if (req.payload !== undefined) {
 		await fs.ensureFile(pathToComp);
 		await fs.outputFile(pathToComp, req.payload, options);
@@ -613,7 +613,7 @@ async function dropComponent(req) {
 
 	const { project, file } = req;
 	const projectPath = req.file ? path.join(project, file) : project;
-	const pathToComponent = path.join(env.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
+	const pathToComponent = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
 
 	const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project);
 	if (await fs.pathExists(componentSymlink)) {

package/core/components/operationsValidation.js

@@ -4,9 +4,9 @@ const Joi = require('joi');
 const fs = require('fs-extra');
 const path = require('path');
 const validator = require('../validation/validationWrapper.js');
-const envMangr = require('../utility/environment/environmentManager.js');
 const hdbTerms = require('../utility/hdbTerms.ts');
 const hdbLogger = require('../utility/logging/harper_logger.js');
+const configUtils = require('../config/configUtils.js');
 const { hdbErrors } = require('../utility/errors/hdbError.js');
 const { HDB_ERROR_MSGS } = hdbErrors;
 
@@ -34,7 +34,7 @@ module.exports = {
  */
 function checkProjectExists(checkExists, project, helpers) {
 	try {
-		const cfDir = envMangr.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+		const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 		const projectDir = path.join(cfDir, project);
 
 		if (!fs.existsSync(projectDir)) {
@@ -71,7 +71,7 @@ function checkFilePath(path, helpers) {
  */
 function checkFileExists(project, type, file, helpers) {
 	try {
-		const cfDir = envMangr.get(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+		const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 		const filePath = path.join(cfDir, project, type, file + '.js');
 		if (!fs.existsSync(filePath)) {
 			return helpers.message(HDB_ERROR_MSGS.NO_FILE);

package/core/config/configUtils.js

@@ -57,6 +57,7 @@ exports.deleteConfigFromFile = deleteConfigFromFile;
 exports.getConfigObj = getConfigObj;
 exports.resolvePath = resolvePath;
 exports.getFlatConfigObj = getFlatConfigObj;
+exports.getConfigPath = getConfigPath;
 
 function resolvePath(relativePath) {
 	if (relativePath?.startsWith('~/')) {
@@ -70,7 +71,23 @@ function resolvePath(relativePath) {
 		return relativePath;
 	}
 }
-
+/**
+ * Get a config value and resolve it as a path relative to rootPath.
+ * Use this for any config param that represents a file/directory path.
+ * @param param
+ */
+function getConfigPath(param) {
+	const env = require('../utility/environment/environmentManager.js');
+	const value = env.get(param);
+	if (!value || typeof value !== 'string') return value;
+	if (value.startsWith('~/')) {
+		return path.join(hdbUtils.getHomeDir(), value.slice(1));
+	}
+	if (path.isAbsolute(value)) return value;
+	const rootPath = env.getHdbBasePath();
+	if (!rootPath) return value;
+	return path.resolve(rootPath, value);
+}
 /**
  * Builds the Harper config file using user inputs and default values from defaultConfig.yaml
  * @param args - any args that the user provided.
@@ -332,15 +349,14 @@ function initConfig(force = false) {
  * @param configFilePath
  */
 function checkForUpdatedConfig(configDoc, configFilePath) {
-	const rootPath = configDoc.getIn(['rootPath']);
 	let updateFile = false;
 	if (!configDoc.hasIn(['storage', 'path'])) {
-		configDoc.setIn(['storage', 'path'], path.join(rootPath, 'database'));
+		configDoc.setIn(['storage', 'path'], 'database');
 		updateFile = true;
 	}
 
 	if (!configDoc.hasIn(['logging', 'rotation', 'path'])) {
-		configDoc.setIn(['logging', 'rotation', 'path'], path.join(rootPath, 'log'));
+		configDoc.setIn(['logging', 'rotation', 'path'], 'log');
 		updateFile = true;
 	}
 

package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js

@@ -7,6 +7,7 @@ const path = require('path');
 const minimist = require('minimist');
 const fs = require('fs-extra');
 const _ = require('lodash');
+const { getConfigPath } = require('../../../../config/configUtils.js');
 env.initSync();
 
 const { CONFIG_PARAMS, DATABASES_PARAM_CONFIG, SYSTEM_SCHEMA_NAME } = hdbTerms;
@@ -25,7 +26,7 @@ function getBaseSchemaPath() {
 
 	if (env.getHdbBasePath() !== undefined) {
 		BASE_SCHEMA_PATH =
-			env.get(CONFIG_PARAMS.STORAGE_PATH) || path.join(env.getHdbBasePath(), hdbTerms.DATABASES_DIR_NAME);
+			getConfigPath(CONFIG_PARAMS.STORAGE_PATH) || path.join(env.getHdbBasePath(), hdbTerms.DATABASES_DIR_NAME);
 		return BASE_SCHEMA_PATH;
 	}
 }
@@ -52,7 +53,7 @@ function getTransactionAuditStoreBasePath() {
 
 	if (env.getHdbBasePath() !== undefined) {
 		TRANSACTION_STORE_PATH =
-			env.get(hdbTerms.CONFIG_PARAMS.STORAGE_AUDIT_PATH) ||
+			getConfigPath(hdbTerms.CONFIG_PARAMS.STORAGE_AUDIT_PATH) ||
 			path.join(env.getHdbBasePath(), hdbTerms.TRANSACTIONS_DIR_NAME);
 		return TRANSACTION_STORE_PATH;
 	}

package/core/resources/Resource.ts

@@ -683,15 +683,26 @@ function transactional(
 		}
 	}
 }
-function missingMethod(resource, method) {
-	const error = new ClientError(`The ${resource.constructor.name} does not have a ${method} method implemented`, 405);
-	error.allow = [];
+const KNOWN_METHODS = ['get', 'head', 'put', 'post', 'delete', 'patch', 'query', 'move', 'copy'];
+type ClientErrorWithMethods = ClientError & { allow: string[]; method: string };
+export function missingMethod(resource: any, method: string) {
+	const error = new ClientError(
+		`The ${resource.constructor.name} does not have a ${method} method implemented`,
+		405
+	) as ClientErrorWithMethods;
+	error.allow = allowedMethods(resource);
 	error.method = method;
-	for (const method of ['get', 'put', 'post', 'delete', 'query', 'move', 'copy']) {
-		if (typeof resource[method] === 'function') error.allow.push(method);
-	}
 	throw error;
 }
+
+export function allowedMethods(resource: any) {
+	const allow = [];
+	for (const method of KNOWN_METHODS) {
+		if (typeof resource[method] === 'function') allow.push(method);
+	}
+	return allow;
+}
+
 /**
  * This is responsible for handling a select query parameter/call that selects specific
  * properties from the returned record(s).

package/core/resources/RocksTransactionLogStore.ts

@@ -3,6 +3,7 @@ import { ExtendedIterable } from '@harperfast/extended-iterable';
 import { Decoder, readAuditEntry, ENTRY_DATAVIEW, AuditRecord, createAuditEntry } from './auditStore.ts';
 import { isMainThread } from 'node:worker_threads';
 import { EventEmitter } from 'node:events';
+import { asBinary } from 'lmdb';
 
 if (!process.env.HARPER_NO_FLUSH_ON_EXIT && isMainThread) {
 	// we want to be able to test log replay
@@ -81,7 +82,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 
 	putSync(suggestedKey: any, value: any, options: any) {
 		if (typeof suggestedKey === 'symbol') {
-			this.rootStore.putSync(suggestedKey, value, options);
+			this.rootStore.putSync(suggestedKey, asBinary(value), options);
 		} else {
 			this.put(suggestedKey, value, options);
 		}
@@ -107,6 +108,12 @@ export class RocksTransactionLogStore extends EventEmitter {
 			return this.rootStore.getSync(key);
 		}
 	}
+	getBinary(key: any) {
+		if (typeof key === 'number') {
+			throw new Error('Unsupported binary access by number');
+		}
+		return this.rootStore.getBinarySync(key);
+	}
 	getEntry() {
 		throw new Error('Not implemented');
 	}

package/core/resources/analytics/write.ts

@@ -402,7 +402,12 @@ export async function getDirectorySizeAsync(dirPath: string): Promise<number> {
 	}
 }
 
+const DEFAULT_STORAGE_INTERVAL = 10;
+let nodeStorageInterval = DEFAULT_STORAGE_INTERVAL;
+let nodeStorageCycleCount = 0;
+
 async function storeNodeStorageMetric(analyticsTable: Table) {
+	if (nodeStorageInterval <= 0 || ++nodeStorageCycleCount % nodeStorageInterval !== 1) return;
 	try {
 		const size = await getDirectorySizeAsync(getHdbBasePath());
 		storeMetric(analyticsTable, {
@@ -689,6 +694,7 @@ if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics);
 let scheduledTasksRunning;
 function startScheduledTasks() {
 	scheduledTasksRunning = true;
+	nodeStorageInterval = envGet(CONFIG_PARAMS.ANALYTICS_STORAGEINTERVAL) ?? DEFAULT_STORAGE_INTERVAL;
 	const AGGREGATE_PERIOD = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1000;
 	if (AGGREGATE_PERIOD) {
 		setInterval(

package/core/resources/databases.ts

@@ -12,6 +12,7 @@ import {
 import { makeTable } from './Table.ts';
 import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
 import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
+import { getConfigPath } from '../config/configUtils.js';
 import { _assignPackageExport } from '../globals.js';
 import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
 import * as signalling from '../utility/signalling.js';
@@ -176,7 +177,7 @@ export function getDatabases(): Databases {
 	if (process.env.SCHEMAS_DATA_PATH) schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
 	databasePath =
 		process.env.STORAGE_PATH ||
-		envGet(CONFIG_PARAMS.STORAGE_PATH) ||
+		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
 		(databasePath && (existsSync(databasePath) ? databasePath : join(getHdbBasePath(), LEGACY_DATABASES_DIR_NAME)));
 	if (!databasePath) return;
 
@@ -694,7 +695,7 @@ export function database({ database: databaseName, table: tableName }) {
 		tablePath ||
 		databaseConfig[databaseName]?.path ||
 		process.env.STORAGE_PATH ||
-		envGet(CONFIG_PARAMS.STORAGE_PATH) ||
+		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
 		(existsSync(join(hdbBasePath, DATABASES_DIR_NAME))
 			? join(hdbBasePath, DATABASES_DIR_NAME)
 			: join(hdbBasePath, LEGACY_DATABASES_DIR_NAME));