@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

package/bin/BinObjects.js

@@ -0,0 +1,17 @@
+'use strict';
+
+/**
+ * This is meant as a central place to defined POJOs used by functions in the /bin/ directory.
+ */
+
+class HdbInfoInsertObject {
+	constructor(id, dataVersionNum, hdbVersionNum) {
+		this.info_id = id;
+		this.data_version_num = dataVersionNum;
+		this.hdb_version_num = hdbVersionNum;
+	}
+}
+
+module.exports = {
+	HdbInfoInsertObject,
+};

package/bin/cliOperations.js

@@ -0,0 +1,157 @@
+'use strict';
+
+const envMgr = require('../utility/environment/environmentManager.js');
+envMgr.initSync();
+const terms = require('../utility/hdbTerms.ts');
+const { httpRequest } = require('../utility/common_utils.js');
+const path = require('path');
+const fs = require('fs-extra');
+const YAML = require('yaml');
+const { packageDirectory } = require('../components/packageComponent.ts');
+const { encode } = require('cbor-x');
+const { getHdbPid } = require('../utility/processManagement/processManagement.js');
+const { initConfig } = require('../config/configUtils.js');
+
+const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' };
+
+module.exports = { cliOperations, buildRequest };
+const PREPARE_OPERATION = {
+	deploy_component: async (req) => {
+		if (req.package) {
+			return;
+		}
+
+		const projectPath = process.cwd();
+		req.payload = await packageDirectory(projectPath, { skip_node_modules: true, ...req });
+		req.cborEncode = true;
+		if (!req.project) req.project = path.basename(projectPath);
+	},
+};
+
+/**
+ * Builds an Op-API request object from CLI args
+ */
+function buildRequest() {
+	const req = {};
+	for (const arg of process.argv.slice(2)) {
+		if (OP_ALIASES.hasOwnProperty(arg)) {
+			req.operation = OP_ALIASES[arg];
+		} else if (arg.includes('=')) {
+			let [first, ...rest] = arg.split('=');
+			rest = rest.join('=');
+
+			try {
+				rest = JSON.parse(rest);
+			} catch {
+				/* noop */
+			}
+
+			req[first] = rest;
+		} else {
+			// operation should only be in the first arg
+			req.operation ??= arg;
+		}
+	}
+
+	return req;
+}
+
+/**
+ * Using a unix domain socket will send a request to hdb operations API server
+ * @param req
+ * @returns {Promise<void>}
+ */
+async function cliOperations(req) {
+	if (!req.target) {
+		req.target = process.env.CLI_TARGET;
+	}
+	let target;
+	if (req.target) {
+		try {
+			target = new URL(req.target);
+		} catch (error) {
+			try {
+				target = new URL(`https://${req.target}:9925`);
+			} catch {
+				throw error; // throw the original error
+			}
+		}
+		target = {
+			protocol: target.protocol,
+			hostname: target.hostname,
+			port: target.port,
+			username: req.username || target.username || process.env.CLI_TARGET_USERNAME,
+			password: req.password || target.password || process.env.CLI_TARGET_PASSWORD,
+			rejectUnauthorized: req.rejectUnauthorized,
+		};
+	} else {
+		// if we aren't doing a targeted operation (like deploy), we initialize the config and verify that local harper
+		// is running and that we can communicate with it.
+		initConfig();
+		if (!getHdbPid()) {
+			console.error('Harper must be running to perform this operation');
+			process.exit(1);
+		}
+
+		if (!fs.existsSync(envMgr.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET))) {
+			console.error('No domain socket found, unable to perform this operation');
+			process.exit(1);
+		}
+	}
+	await PREPARE_OPERATION[req.operation]?.(req);
+	try {
+		let options = target ?? {
+			protocol: 'http:',
+			socketPath: envMgr.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET),
+		};
+		options.method = 'POST';
+		options.headers = { 'Content-Type': 'application/json' };
+		if (target?.username) {
+			options.headers.Authorization = `Basic ${Buffer.from(`${target.username}:${target.password}`).toString('base64')}`;
+		}
+		if (req.cborEncode) {
+			options.headers['Content-Type'] = 'application/cbor';
+			req = encode(req);
+		}
+		let response = await httpRequest(options, req);
+
+		let responseData;
+		try {
+			responseData = JSON.parse(response.body);
+		} catch {
+			responseData = {
+				status: response.statusCode + ' ' + (response.statusMessage || 'Unknown'),
+				body: response.body,
+			};
+		}
+
+		let responseLog;
+		if (req.json) {
+			responseLog = JSON.stringify(responseData, null, 2);
+		} else {
+			responseLog = YAML.stringify(responseData).trim();
+		}
+
+		const { statusCode } = response;
+		if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
+			const errorPrefix = responseLog.startsWith('error:') ? '' : 'error: ';
+			console.error(`${errorPrefix}${responseLog}`);
+			process.exit(1);
+		}
+
+		console.log(responseLog);
+
+		return responseData;
+	} catch (err) {
+		if (err.code === 'ENOENT' || err.code === 'ECONNREFUSED') {
+			console.error(`error: Failed to connect to Harper (${err.code}): ${err.message}`);
+		} else if (err.code === 'EACCES') {
+			console.error(`error: Permission denied accessing the domain socket: ${err.message}`);
+		} else if (err.code === 'ENOTFOUND') {
+			console.error(`error: Host not found: "${err.hostname}" ${err.message}`);
+		} else {
+			console.error(`error: ${err.message ?? err}`);
+		}
+		process.exit(1);
+	}
+}

package/bin/copyDb.ts

@@ -0,0 +1,280 @@
+import { getDatabases, getDefaultCompression, resetDatabases } from '../resources/databases.ts';
+import { open } from 'lmdb';
+import { join } from 'path';
+import { move, remove } from 'fs-extra';
+import { get } from '../utility/environment/environmentManager.js';
+import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
+import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.js';
+import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../utility/lmdb/terms.js';
+import { CONFIG_PARAMS, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
+import { AUDIT_STORE_OPTIONS } from '../resources/auditStore.ts';
+import { describeSchema } from '../dataLayer/schemaDescribe.js';
+import { updateConfigValue } from '../config/configUtils.js';
+import * as hdbLogger from '../utility/logging/harper_logger.js';
+
+export async function compactOnStart() {
+	hdbLogger.notify('Running compact on start');
+	console.log('Running compact on start');
+
+	// Create compact copy and backup
+	const rootPath = get(CONFIG_PARAMS.ROOTPATH);
+	const compactedDb = new Map();
+	const databases = getDatabases();
+
+	updateConfigValue(CONFIG_PARAMS.STORAGE_COMPACTONSTART, false); // don't run this again, and update it before starting so that it fails we don't just keep retrying over and over
+
+	try {
+		for (const databaseName in databases) {
+			if (databaseName === 'system') continue;
+			if (databaseName.endsWith('-copy')) continue; // don't copy the copy
+			let dbPath;
+			for (const tableName in databases[databaseName]) {
+				dbPath = databases[databaseName][tableName].primaryStore.path;
+				break;
+			}
+			if (!dbPath) {
+				console.log("Couldn't find any tables in database", databaseName);
+				continue;
+			}
+
+			const backupDest = join(rootPath, 'backup', databaseName + '.mdb');
+			const copyDest = join(rootPath, DATABASES_DIR_NAME, databaseName + '-copy.mdb');
+			let recordCount = 0;
+			try {
+				recordCount = await getTotalDBRecordCount(databaseName);
+				console.log('Database', databaseName, 'before compact has a total record count of', recordCount);
+			} catch (error) {
+				hdbLogger.error('Error getting record count for database', databaseName, error);
+				console.error('Error getting record count for database', databaseName, error);
+			}
+			compactedDb.set(databaseName, {
+				dbPath,
+				copyDest,
+				backupDest,
+				recordCount,
+			});
+
+			await copyDb(databaseName, copyDest);
+
+			console.log('Backing up', databaseName, 'to', backupDest);
+			try {
+				await move(dbPath, backupDest, { overwrite: true });
+			} catch (error) {
+				console.log('Error moving database', dbPath, 'to', backupDest, error);
+			}
+			// Move compacted DB to back to original DB path
+			console.log('Moving copy compacted', databaseName, 'to', dbPath);
+			await move(copyDest, dbPath, { overwrite: true });
+			await remove(join(rootPath, DATABASES_DIR_NAME, `${databaseName}-copy.mdb-lock`));
+		}
+		try {
+			resetDatabases();
+		} catch (err) {
+			hdbLogger.error('Error resetting databases after backup', err);
+			console.error('Error resetting databases after backup', err);
+		}
+
+		try {
+			resetDatabases();
+		} catch (err) {
+			hdbLogger.error('Error resetting databases after backup', err);
+			console.error('Error resetting databases after backup', err);
+			process.exit(0); // just let the process restart
+		}
+	} catch (err) {
+		hdbLogger.error('Error compacting database, rolling back operation', err);
+		console.error('Error compacting database, rolling back operation', err);
+
+		updateConfigValue(CONFIG_PARAMS.STORAGE_COMPACTONSTART, false);
+
+		for (const [_db, { dbPath, backupDest }] of compactedDb) {
+			console.error('Moving backup database', backupDest, 'back to', dbPath);
+			try {
+				await move(backupDest, dbPath, { overwrite: true });
+			} catch (err) {
+				console.error(err);
+			}
+		}
+		resetDatabases();
+
+		throw err;
+	}
+
+	// Clean up backups
+	for (const [db, { backupDest, recordCount }] of compactedDb) {
+		const compactRecordCount = await getTotalDBRecordCount(db);
+		console.log('Database', db, 'after compact has a total record count of', compactRecordCount);
+
+		if (recordCount !== compactRecordCount) {
+			const errMsg = `There is a discrepancy between pre and post compact record count for database ${db}.\nTotal record count before compaction: ${recordCount}, total after: ${compactRecordCount}.\nDatabase backup has not been removed and can be found here: ${backupDest}`;
+			hdbLogger.warn(errMsg);
+			console.warn(errMsg);
+		}
+
+		if (get(CONFIG_PARAMS.STORAGE_COMPACTONSTARTKEEPBACKUP) === true) continue;
+		console.log('Removing backup', backupDest);
+		await remove(backupDest);
+	}
+}
+
+async function getTotalDBRecordCount(database: string) {
+	const dbDescribe = await describeSchema({ database });
+	let total = 0;
+	for (const table in dbDescribe) {
+		total += dbDescribe[table].record_count;
+	}
+
+	return total;
+}
+
+// we replace the write functions with a noop during this process, just in case they get called
+function noop() {
+	// if there are any attempts to write to the db, ignore them
+}
+
+export async function copyDb(sourceDatabase: string, targetDatabasePath: string) {
+	console.log(`Copying database ${sourceDatabase} to ${targetDatabasePath}`);
+	const sourceDb = getDatabases()[sourceDatabase];
+	if (!sourceDb) throw new Error(`Source database not found: ${sourceDatabase}`);
+	let rootStore;
+	for (const tableName in sourceDb) {
+		const table = sourceDb[tableName];
+		// ensure that writes aren't occurring
+		table.primaryStore.put = noop;
+		table.primaryStore.remove = noop;
+		for (const attributeName in table.indices) {
+			const index = table.indices[attributeName];
+			index.put = noop;
+			index.remove = noop;
+		}
+		if (table.auditStore) {
+			table.auditStore.put = noop;
+			table.auditStore.remove = noop;
+		}
+		rootStore = table.primaryStore.rootStore;
+	}
+	if (!rootStore) throw new Error(`Source database does not have any tables: ${sourceDatabase}`);
+	// this contains the list of all the dbis
+	const sourceDbisDb = rootStore.dbisDb;
+	const sourceAuditStore = rootStore.auditStore;
+	const targetEnv = open(new OpenEnvironmentObject(targetDatabasePath));
+	const targetDbisDb = targetEnv.openDB(INTERNAL_DBIS_NAME);
+	let written;
+	let outstandingWrites = 0;
+	// we use a single transaction to get a snapshot, also we can't use snapshot: false on dupsort dbs
+	const transaction = sourceDbisDb.useReadTransaction();
+	try {
+		for (const { key, value: attribute } of sourceDbisDb.getRange({ transaction })) {
+			const isPrimary = attribute.isPrimaryKey;
+			let existingCompression, newCompression;
+			if (isPrimary) {
+				existingCompression = attribute.compression;
+				newCompression = getDefaultCompression();
+				if (newCompression) attribute.compression = newCompression;
+				else delete attribute.compression;
+				if (existingCompression?.dictionary?.toString() === newCompression?.dictionary?.toString()) {
+					// no need to change the compression, it's the same, so we can, and should, skip decompressing and recompressing
+					existingCompression = null;
+					newCompression = null;
+				}
+			}
+			targetDbisDb.put(key, attribute);
+			if (!(isPrimary || attribute.indexed)) continue;
+			const dbiInit = new OpenDBIObject(!isPrimary, isPrimary);
+			// we want to directly copy bytes so we don't have the overhead of
+			// encoding and decoding
+			dbiInit.encoding = 'binary';
+			dbiInit.compression = existingCompression;
+			//dbiInit.keyEncoding = 'binary';
+			const sourceDbi = rootStore.openDB(key, dbiInit);
+			sourceDbi.decoder = null;
+			sourceDbi.decoderCopies = false;
+			sourceDbi.encoding = 'binary';
+			dbiInit.compression = newCompression;
+			const targetDbi = targetEnv.openDB(key, dbiInit);
+			targetDbi.encoder = null;
+			console.log('copying', key, 'from', sourceDatabase, 'to', targetDatabasePath);
+			await copyDbi(sourceDbi, targetDbi, isPrimary, transaction);
+		}
+		if (sourceAuditStore) {
+			const targetAuditStore = rootStore.openDB(AUDIT_STORE_NAME, AUDIT_STORE_OPTIONS);
+			console.log('copying audit log for', sourceDatabase, 'to', targetDatabasePath);
+			copyDbi(sourceAuditStore, targetAuditStore, false, transaction);
+		}
+
+		async function copyDbi(sourceDbi, targetDbi, isPrimary, transaction) {
+			let recordsCopied = 0;
+			let bytesCopied = 0;
+			let skippedRecord = 0;
+			let retries = 10000000;
+			let start = null;
+			while (retries-- > 0) {
+				try {
+					for (const key of sourceDbi.getKeys({ start, transaction })) {
+						try {
+							start = key;
+							const { value, version } = sourceDbi.getEntry(key, { transaction });
+							// deleted entries should be 13 bytes long (8 for timestamp, 4 bytes for flags, 1 byte of the encoding of null)
+							if (value?.length < 14 && isPrimary) {
+								skippedRecord++;
+								continue;
+							}
+							written = targetDbi.put(key, value, isPrimary ? version : undefined);
+							recordsCopied++;
+							if (transaction.openTimer) transaction.openTimer = 0; // reset the timer, don't want it to time out
+							bytesCopied += (key?.length || 10) + value.length;
+							if (outstandingWrites++ > 5000) {
+								await written;
+								console.log(
+									'copied',
+									recordsCopied,
+									'entries, skipped',
+									skippedRecord,
+									'delete records,',
+									bytesCopied,
+									'bytes'
+								);
+								outstandingWrites = 0;
+							}
+						} catch (error) {
+							console.error(
+								'Error copying record',
+								typeof key === 'symbol' ? 'symbol' : key,
+								'from',
+								sourceDatabase,
+								'to',
+								targetDatabasePath,
+								error
+							);
+						}
+					}
+					console.log(
+						'finish copying, copied',
+						recordsCopied,
+						'entries, skipped',
+						skippedRecord,
+						'delete records,',
+						bytesCopied,
+						'bytes'
+					);
+					return;
+				} catch {
+					// try to resume with a bigger key
+					if (typeof start === 'string') {
+						if (start === 'z') {
+							return console.error('Reached end of dbi', start, 'for', sourceDatabase, 'to', targetDatabasePath);
+						}
+						start = start.slice(0, -2) + 'z';
+					} else if (typeof start === 'number') start++;
+					else return console.error('Unknown key type', start, 'for', sourceDatabase, 'to', targetDatabasePath);
+				}
+			}
+		}
+
+		await written;
+		console.log('copied database ' + sourceDatabase + ' to ' + targetDatabasePath);
+	} finally {
+		transaction.done();
+		targetEnv.close();
+	}
+}

package/bin/harper.js

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const logger = require('../utility/logging/harper_logger.js');
+const cliOperations = require('./cliOperations.js');
+const { packageJson } = require('../utility/packageUtils.js');
+const checkNode = require('../launchServiceScripts/utility/checkNodeVersion.js');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const { SERVICE_ACTIONS_ENUM } = hdbTerms;
+process.setSourceMapsEnabled(true); // this is necessary for source maps to work, at least on the main thread.
+
+const HELP = `
+Usage: harperdb [command]
+
+With no command, harper will simply run Harper (in the foreground)
+
+By default, the CLI also supports certain Operation APIs. Specify the operation name and any required parameters, and omit the 'operation' command.
+
+Commands:
+copy-db <source> <target>       - Copies a database from source path to target path
+dev <path>                      - Run the application in dev mode with debugging, foreground logging, no auth
+install                         - Install harperdb
+<api-operation> <param>=<value> - Run an API operation and return result to the CLI, not all operations are supported
+register                        - Register harperdb
+renew-certs                     - Generate a new set of self-signed certificates
+restart                         - Restart the harperdb background process
+run <path>                      - Run the application in the specified path
+start                           - Starts a separate background process for harperdb and CLI will exit
+status                          - Print the status of Harper
+stop                            - Stop the harperdb background process
+help                            - Display this output
+upgrade                         - Upgrade harperdb
+version                         - Print the version
+deploy                          - Deploy the application locally or remotely with target=<remote url>
+`;
+
+async function harper() {
+	let nodeResults = checkNode();
+
+	if (nodeResults) {
+		if (nodeResults.error) {
+			console.error(nodeResults.error);
+			logger.error(nodeResults.error);
+			return;
+		} else if (nodeResults.warn) {
+			console.warn(nodeResults.warn);
+			logger.warn(nodeResults.warn);
+		}
+	}
+
+	let service;
+
+	if (process.argv && process.argv[2] && !process.argv[2].startsWith('-')) {
+		service = process.argv[2].toLowerCase();
+	}
+
+	switch (service) {
+		case SERVICE_ACTIONS_ENUM.HELP:
+			return HELP;
+		case SERVICE_ACTIONS_ENUM.START:
+			return require('./run.js').launch();
+		case SERVICE_ACTIONS_ENUM.INSTALL:
+			return require('./install.js')();
+		case SERVICE_ACTIONS_ENUM.STOP:
+			return require('./stop.js')().then(() => {
+				process.exit(0);
+			});
+		case SERVICE_ACTIONS_ENUM.RESTART:
+			return require('./restart.js').restart({});
+		case SERVICE_ACTIONS_ENUM.VERSION:
+			return packageJson.version;
+		case SERVICE_ACTIONS_ENUM.UPGRADE:
+			logger.setLogLevel(hdbTerms.LOG_LEVELS.INFO);
+			// The require is here to better control the flow of imports when this module is called.
+			return require('./upgrade.js')
+				.upgrade(null)
+				.then(() => 'Your instance of Harper is up to date!');
+		case SERVICE_ACTIONS_ENUM.STATUS:
+			return require('./status.js')();
+		case SERVICE_ACTIONS_ENUM.RENEWCERTS:
+			return require('../security/keys.js')
+				.renewSelfSigned()
+				.then(() => 'Successfully renewed self-signed certificates');
+		case SERVICE_ACTIONS_ENUM.COPYDB: {
+			let sourceDb = process.argv[3];
+			let targetDbPath = process.argv[4];
+			return require('./copyDb.ts').copyDb(sourceDb, targetDbPath);
+		}
+		case SERVICE_ACTIONS_ENUM.DEV:
+			process.env.DEV_MODE = true;
+		// fall through
+		case SERVICE_ACTIONS_ENUM.RUN: {
+			// Run a specific application folder
+			let appFolder = process.argv[3];
+			if (appFolder && appFolder[0] !== '-') {
+				if (!fs.existsSync(appFolder)) {
+					throw new Error(`The folder ${appFolder} does not exist`);
+				}
+				if (!fs.statSync(appFolder).isDirectory()) {
+					throw new Error(`The path ${appFolder} is not a folder`);
+				}
+				appFolder = fs.realpathSync(appFolder);
+				if (
+					fs.existsSync(path.join(appFolder, hdbTerms.HARPER_CONFIG_FILE)) ||
+					(fs.existsSync(path.join(appFolder, hdbTerms.HDB_CONFIG_FILE)) &&
+						fs.existsSync(path.join(appFolder, 'database')))
+				) {
+					// This can be used to run HDB without a boot file
+					process.env.ROOTPATH = appFolder;
+				} else {
+					process.env.RUN_HDB_APP = appFolder;
+				}
+			} else if (fs.existsSync(hdbTerms.HDB_COMPONENT_CONFIG_FILE) || fs.existsSync('schema.graphql')) {
+				console.warn(
+					`It appears you are running Harper in an application directory, but did not specify the path. I'll go ahead and run the application for you since that's probably what you meant. But to avoid this warning in the future, run applications in the current directory like this: "harper ${service} ."`
+				);
+				process.env.RUN_HDB_APP = process.cwd();
+			} else if (fs.existsSync(hdbTerms.HARPER_CONFIG_FILE) || fs.existsSync(hdbTerms.HDB_CONFIG_FILE)) {
+				console.warn(
+					`It appears you are running Harper in a root data directory, but did not specify the path. I'll go ahead and run Harper with its root path set to "." for you since that's probably what you meant. But to avoid this warning in the future, run it like this: "harper ${service} ."`
+				);
+				process.env.ROOTPATH = process.cwd();
+			}
+		}
+		// fall through
+		case undefined: // run harperdb in the foreground in standard mode
+			return require('./run.js').main();
+		default:
+			const cliApiOp = cliOperations.buildRequest();
+			logger.trace('calling cli operations with:', cliApiOp);
+			await cliOperations.cliOperations(cliApiOp);
+			return;
+	}
+}
+exports.harper = harper;
+if (require.main === module) {
+	harper()
+		.then((message) => {
+			if (message) {
+				console.log(message);
+				logger.notify(message);
+			}
+			// Intentionally not calling `process.exit(0);` so if a CLI
+			// command resulted in a long running process (aka `run`),
+			// it continues to run.
+		})
+		.catch((error) => {
+			if (error) {
+				console.error(error);
+				logger.error(error);
+			}
+			process.exit(1);
+		});
+}

package/bin/install.js

@@ -0,0 +1,15 @@
+const installer = require('../utility/install/installer.js');
+const hdbLogger = require('../utility/logging/harper_logger.js');
+
+module.exports = install;
+
+async function install() {
+	try {
+		await installer.install();
+	} catch (err) {
+		console.error('There was an error during the install.');
+		console.error(err);
+		hdbLogger.error(err);
+		process.exit(1);
+	}
+}

package/bin/lite.js

@@ -0,0 +1,5 @@
+const { startHTTPThreads, startSocketServer } = require('../server/threads/socketRouter.ts');
+
+startHTTPThreads(0, true);
+startSocketServer(9925);
+startSocketServer(9926);