@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

package/launchServiceScripts/utility/checkNodeVersion.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const semverMajor = require('semver/functions/major');
+const { packageJson } = require('../../utility/packageUtils.js');
+const INSTALLED_NODE_VERSION = process.versions && process.versions.node ? process.versions.node : undefined;
+
+module.exports = checkNodeVersion;
+
+function checkNodeVersion() {
+	const minimumHdbNodeVersion = packageJson.engines['minimum-node'];
+	if (INSTALLED_NODE_VERSION && semverMajor(INSTALLED_NODE_VERSION) < semverMajor(minimumHdbNodeVersion)) {
+		const versionError = `The minimum version of Node.js Harper supports is: ${minimumHdbNodeVersion}, the currently installed Node.js version is: ${INSTALLED_NODE_VERSION}. Please install a version of Node.js that is withing the defined range.`;
+		return { error: versionError };
+	}
+}

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@harperfast/harper",
 	"description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
-	"version": "5.0.0-alpha.10",
+	"version": "5.0.0-beta.3",
 	"license": "Apache-2.0",
 	"homepage": "https://harper.fast",
 	"bugs": {
@@ -23,7 +23,25 @@
 	},
 	"files": [
 		"dist",
+		"bin",
+		"components",
+		"config",
+		"dataLayer",
+		"json",
+		"launchServiceScripts",
+		"resources",
+		"security",
+		"server",
+		"sqlTranslator",
+		"upgrade",
+		"utility",
+		"validation",
+		"studio",
 		"static",
+		"index.*",
+		"v1.*",
+		"v2.*",
+		"README.md",
 		"SECURITY.md",
 		"SUPPORT.md",
 		"CODE_OF_CONDUCT.md"
@@ -31,7 +49,7 @@
 	"scripts": {
 		"build": "tsc --project tsconfig.build.json",
 		"build:watch": "npm run build -- --watch --incremental",
-		"package": "npm run build; npm shrinkwrap && npm pack",
+		"package": "./build-tools/build.sh",
 		"lint": "oxlint --deny-warnings .",
 		"lint:required": "oxlint --quiet .",
 		"lint:fix": "npm run lint -- --fix",
@@ -115,7 +133,7 @@
 		"globals": "^16.5.0",
 		"intercept-stdout": "0.1.2",
 		"mkcert": "^3.2.0",
-		"mocha": "^11.7.4",
+		"mocha": "^11.7.5",
 		"mqtt": "~4.3.8",
 		"oxlint": "^1.31.0",
 		"prettier": "~3.7.3",
@@ -126,20 +144,20 @@
 		"tsx": "^4.20.6",
 		"typescript": "^5.8.2",
 		"typescript-eslint": "^8.45.0",
-		"undici": "^7.16.0",
+		"undici": "^7.24.4",
 		"why-is-node-still-running": "^1.0.0"
 	},
 	"dependencies": {
-		"@aws-sdk/client-s3": "3.964.0",
+		"@aws-sdk/client-s3": "^3.1012.0",
 		"@aws-sdk/lib-storage": "3.964.0",
 		"@datadog/pprof": "^5.11.1",
 		"@endo/static-module-record": "^1.1.2",
-		"@fastify/autoload": "5.10.0",
-		"@fastify/compress": "~6.5.0",
-		"@fastify/cors": "~9.0.1",
-		"@fastify/static": "~7.0.4",
+		"@fastify/autoload": "^6.3.1",
+		"@fastify/compress": "^8.3.1",
+		"@fastify/cors": "^11.2.0",
+		"@fastify/static": "^9.0.0",
 		"@harperfast/extended-iterable": "^1.0.1",
-		"@harperfast/rocksdb-js": "^0.1.11",
+		"@harperfast/rocksdb-js": "^0.1.12",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
 		"@turf/boolean-disjoint": "6.5.0",
@@ -150,6 +168,7 @@
 		"@turf/helpers": "6.5.0",
 		"@turf/length": "6.5.0",
 		"alasql": "4.6.6",
+		"amaro": "^1.1.8",
 		"argon2": "0.44.0",
 		"asn1js": "3.0.7",
 		"cbor-x": "1.6.4",
@@ -160,8 +179,8 @@
 		"dotenv": "^16.4.7",
 		"easy-ocsp": "1.2.2",
 		"fast-glob": "3.3.3",
-		"fastify": "~4.29.0",
-		"fastify-plugin": "~4.5.1",
+		"fastify": "^5.8.2",
+		"fastify-plugin": "^5.1.0",
 		"fs-extra": "11.3.3",
 		"graphql": "^16.10.0",
 		"graphql-http": "^1.22.4",
@@ -174,8 +193,8 @@
 		"json2csv": "5.0.7",
 		"jsonata": "1.8.7",
 		"jsonwebtoken": "9.0.3",
-		"lmdb": "3.5.1",
-		"lodash": "4.17.21",
+		"lmdb": "3.5.2",
+		"lodash": "^4.17.23",
 		"mathjs": "11.12.0",
 		"micromatch": "^4.0.8",
 		"minimist": "1.2.8",
@@ -203,8 +222,8 @@
 		"ses": "^1.14.0",
 		"stream-chain": "2.2.5",
 		"stream-json": "1.9.1",
-		"systeminformation": "5.27.11",
-		"tar-fs": "3.0.9",
+		"systeminformation": "^5.31.4",
+		"tar-fs": "^3.1.2",
 		"ulidx": "0.5.0",
 		"uuid": "11.1.0",
 		"validate.js": "0.13.1",

package/resources/DatabaseTransaction.ts

@@ -0,0 +1,378 @@
+import { Transaction as LMDBTransaction } from 'lmdb';
+import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.js';
+import { ServerError } from '../utility/errors/hdbError.js';
+import * as harperLogger from '../utility/logging/harper_logger.js';
+import type { Context, Id } from './ResourceInterface.ts';
+import * as envMngr from '../utility/environment/environmentManager.js';
+import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
+import { convertToMS } from '../utility/common_utils.js';
+import { when } from '../utility/when.ts';
+import { Transaction as RocksTransaction, type Store as RocksStore } from '@harperfast/rocksdb-js';
+import type { RootDatabaseKind } from './databases.ts';
+import type { Entry } from './RecordEncoder.ts';
+
+const trackedTxns = new Set<DatabaseTransaction>();
+const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 25 seconds before we start rejecting them
+const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS);
+export const TRANSACTION_STATE = {
+	CLOSED: 0, // the transaction has been committed or aborted and can no longer be used for writes (if read txn is active, it can be used for reads)
+	OPEN: 1, // the transaction is open and can be used for reads and writes
+	LINGERING: 2, // the transaction has completed a read, but can be used for immediate writes
+};
+let outstandingCommit, outstandingCommitStart;
+let confirmReplication;
+export function replicationConfirmation(callback) {
+	confirmReplication = callback;
+}
+let txnExpiration = envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONOPENTIME) ?? 30000;
+
+class StartedTransaction extends Error {}
+
+type MaybePromise<T> = T | Promise<T>;
+
+export type CommitOptions = {
+	doneWriting?: boolean;
+	timestamp?: number;
+	retries?: number;
+	flush?: boolean;
+};
+
+type ReadTransaction = (LMDBTransaction | RocksTransaction) & {
+	openTimer?: number;
+	retryRisk?: number;
+};
+
+export type TransactionWrite = {
+	key: Id;
+	store: RootDatabaseKind;
+	invalidated?: boolean;
+	entry?: Partial<Entry>;
+	before?: () => void | Promise<void>;
+	beforeIntermediate?: () => void | Promise<void>;
+	commit?: (txnTime: number, existingEntry: Entry, retry: boolean, transaction: RocksTransaction) => void;
+	validate?: (txnTime: number) => void;
+	fullUpdate?: boolean;
+	saved?: boolean;
+	deferSave?: boolean;
+};
+
+type RocksTransactionWithRetry = RocksTransaction & { isRetry?: boolean };
+
+export class DatabaseTransaction implements Transaction {
+	#context: Context;
+	writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met
+	completions: Promise<void>[] = []; // the set of outstanding async operations to complete
+	db: RootDatabaseKind;
+	transaction: RocksTransactionWithRetry;
+	readTxn: ReadTransaction;
+	readTxnRefCount: number;
+	readTxnsUsed: number;
+	timeout: number;
+	validated = 0;
+	timestamp = 0;
+	retries = 0;
+	declare next: DatabaseTransaction;
+	declare stale: boolean;
+	declare startedFrom?: {
+		resourceName: string;
+		method: string;
+	};
+	declare stackTraces?: StartedTransaction[];
+	overloadChecked: boolean;
+	open = TRANSACTION_STATE.OPEN;
+	replicatedConfirmation: number;
+
+	getReadTxn(): ReadTransaction {
+		this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
+		this.timeout = txnExpiration; // reset the timeout
+		if (this.transaction) {
+			if (this.transaction.openTimer) this.transaction.openTimer = 0;
+			return this.transaction;
+		}
+		if (this.open !== TRANSACTION_STATE.OPEN) return; // can not start a new read transaction as there is no future commit that will take place, just have to allow the read to latest database state
+
+		this.transaction = new RocksTransaction(this.db.store);
+		if (this.timestamp) {
+			this.transaction.setTimestamp(this.timestamp);
+		}
+
+		this.readTxnsUsed = 1;
+		if (DEBUG_LONG_TXNS) {
+			this.stackTraces = [new StartedTransaction()];
+		}
+		if (this.transaction.openTimer) this.transaction.openTimer = 0;
+		trackedTxns.add(this);
+		return this.transaction;
+	}
+
+	useReadTxn() {
+		const readTxn = this.getReadTxn();
+		if (DEBUG_LONG_TXNS) this.stackTraces.push(new StartedTransaction());
+		this.readTxnsUsed++;
+		return readTxn;
+	}
+
+	doneReadTxn() {
+		if (!this.transaction) return;
+		if (--this.readTxnsUsed === 0) {
+			trackedTxns.delete(this);
+			this.transaction?.abort();
+			this.transaction = null;
+		}
+	}
+
+	disregardReadTxn(): void {
+		if (--this.readTxnRefCount === 0 && this.readTxnsUsed === 1) {
+			this.doneReadTxn();
+		}
+	}
+
+	checkOverloaded() {
+		if (
+			outstandingCommit &&
+			!this.overloadChecked &&
+			performance.now() - outstandingCommitStart > MAX_OUTSTANDING_TXN_DURATION
+		) {
+			throw new ServerError('Outstanding write transactions have too long of queue, please try again later', 503);
+		}
+		this.overloadChecked = true; // only check this once, don't interrupt ongoing transactions that have already made writes
+	}
+
+	addWrite(operation: TransactionWrite) {
+		if (this.open === TRANSACTION_STATE.CLOSED) {
+			throw new Error('Can not use a transaction that is no longer open');
+		}
+		this.writes.push(operation);
+		if (!operation.deferSave) {
+			// Setting saved to false means to defer saving
+			this.save(operation);
+		}
+		return operation;
+	}
+
+	save(operation: TransactionWrite, reloadEntry = false) {
+		let txnTime = this.timestamp;
+		if (!this.transaction) {
+			this.transaction = new RocksTransaction(this.db.store as RocksStore);
+			if (txnTime) {
+				this.transaction.setTimestamp(txnTime);
+			}
+		}
+		if (this.retries > 0) {
+			// this is marks the rocks transaction as a retry so we don't write the transaction log again
+			this.transaction.isRetry = true;
+		}
+		if (!txnTime) txnTime = this.timestamp = this.transaction.getTimestamp();
+		if (reloadEntry || operation.entry === undefined) {
+			operation.entry = operation.store.getEntry(operation.key, { transaction: this.transaction });
+		}
+		operation.saved = true;
+		// immediately execute in this transaction
+		if (operation.validate?.(txnTime) === false) return;
+		let result: Promise<void> = operation.before?.() as Promise<void>;
+		if (result?.then) this.completions.push(result);
+		result = operation.beforeIntermediate?.() as Promise<void>;
+		if (result?.then) this.completions.push(result);
+		operation.commit(txnTime, operation.entry, this.retries > 0, this.transaction);
+	}
+
+	/**
+	 * Resolves with information on the timestamp and success of the commit
+	 */
+	commit(options: CommitOptions = {}): MaybePromise<CommitResolution> {
+		for (let i = 0; i < this.writes.length; i++) {
+			let operation = this.writes[i];
+			if (this.retries === 0 && operation.saved) continue;
+			this.save(operation, i < this.validated);
+		}
+		this.validated = this.writes.length;
+		return when(this.completions.length > 0 ? Promise.all(this.completions) : null, () => {
+			let commitResolution: MaybePromise<void>;
+			if (--this.readTxnsUsed > 0) {
+				// we still have outstanding iterators using the transaction, we can't just commit/abort it, we will still
+				// need to use it
+				commitResolution =
+					this.writes.length > 0
+						? this.transaction?.commit({ renewAfterCommit: true /* Try to use RocksDB's CommitAndTryCreateSnapshot */ })
+						: // don't abort, we still have outstanding reads to complete
+							null;
+			} else {
+				// no more reads need to be performed, just commit/abort based if there are any writes
+				trackedTxns.delete(this);
+				if (this.transaction) {
+					if (this.writes.length > 0) {
+						commitResolution = this.transaction.commit();
+					} else {
+						commitResolution = this.transaction.abort();
+						this.transaction = null; // immediately clear transaction, no need to wait
+					}
+				}
+			}
+
+			if (commitResolution) {
+				if (!outstandingCommit) {
+					outstandingCommit = commitResolution;
+					outstandingCommitStart = performance.now();
+					outstandingCommit.finally(() => {
+						outstandingCommit = null;
+					});
+				}
+				const completions = [];
+				return commitResolution.then(
+					() => {
+						this.transaction.onCommit?.();
+						this.transaction = null; // the native transaction is done (reset if needed)
+						if (this.next) {
+							completions.push(this.next.commit(options));
+						}
+						if (options?.flush) {
+							completions.push(this.writes[0].store.flushed);
+						}
+						if (this.replicatedConfirmation) {
+							// if we want to wait for replication confirmation, we need to track the transaction times
+							// and when replication notifications come in, we count the number of confirms until we reach the desired number
+							const databaseName = this.writes[0].store.rootStore.databaseName;
+							const lastWrite = this.writes[this.writes.length - 1];
+							if (confirmReplication && lastWrite) {
+								completions.push(
+									confirmReplication(
+										databaseName,
+										lastWrite.store.getEntry(lastWrite.key).version,
+										this.replicatedConfirmation
+									)
+								);
+							}
+						}
+						// now reset transactions tracking; this transaction be reused and committed again
+						this.writes = [];
+						if (this.#context?.resourceCache) this.#context.resourceCache = null;
+						this.next = null;
+						let txnTime = this.timestamp;
+						this.timestamp = 0; // reset the timestamp as well
+						return Promise.all(completions).then(() => {
+							return {
+								txnTime,
+							};
+						});
+					},
+					(error) => {
+						if (error.code === 'ERR_BUSY') {
+							// if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry
+							// for future transactions
+							this.retries++;
+							return this.commit(options); // try again
+						} else throw error;
+					}
+				);
+			}
+			const txnResolution: CommitResolution = {
+				txnTime: this.timestamp,
+			};
+			if (this.next) {
+				// now run any other transactions
+				options.timestamp = this.timestamp;
+				const nextResolution = this.next?.commit(options);
+				if (nextResolution?.then)
+					return nextResolution?.then((nextResolution) => ({
+						txnTime: this.timestamp,
+						next: nextResolution,
+					}));
+				txnResolution.next = nextResolution;
+			}
+			return txnResolution;
+		});
+	}
+	abort(): void {
+		while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
+		this.open = TRANSACTION_STATE.CLOSED;
+		// reset the transaction
+		this.writes = [];
+		if (this.#context?.resourceCache) this.#context.resourceCache = null;
+	}
+	directCommitSync(): void {
+		trackedTxns.delete(this);
+		this.transaction?.commitSync();
+	}
+	getContext() {
+		return this.#context;
+	}
+	setContext(context) {
+		this.#context = context;
+	}
+}
+export interface CommitResolution {
+	txnTime: number;
+	next?: CommitResolution;
+}
+export interface Transaction {
+	commit(options): MaybePromise<CommitResolution>;
+	abort?(): any;
+}
+
+export class ImmediateTransaction extends DatabaseTransaction {
+	isCommitting = false;
+	constructor(db: RootDatabaseKind) {
+		super();
+		this.db = db;
+	}
+	save(transaction: ImmediateTransaction) {
+		if (this.isCommitting) {
+			// if we are in the commit, do the save and force a reload so we get a read within the transaction
+			super.save(transaction, true);
+		} else {
+			this.isCommitting = true;
+			return when(this.commit(), () => {
+				this.isCommitting = false;
+			});
+		}
+	}
+
+	get timestamp() {
+		return this._timestamp || (this._timestamp = getNextMonotonicTime());
+	}
+	getReadTxn() {
+		return; // no transaction means read latest
+	}
+}
+
+let timer;
+
+function startMonitoringTxns() {
+	timer = setInterval(function () {
+		for (const txn of trackedTxns) {
+			if (txn.timeout <= 0) {
+				const url = txn.getContext()?.url;
+				harperLogger.error(
+					`Transaction was open too long and has been committed, from table: ${
+						txn.db?.name + (url ? ' path: ' + url : '')
+					}`,
+					...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : []),
+					...(DEBUG_LONG_TXNS ? ['starting stack trace', txn.stackTraces] : [])
+				);
+				// reset the transaction
+				try {
+					const result = txn.commit();
+					if (result?.then) {
+						result.catch((error) => {
+							harperLogger.debug?.(`Error committing timed out transaction: ${error.message}`);
+						});
+					}
+				} catch (error) {
+					harperLogger.debug?.(`Error committing timed out transaction: ${error.message}`);
+				}
+				txn.timeout = txnExpiration;
+			} else {
+				txn.timeout -= txnExpiration;
+			}
+		}
+	}, txnExpiration).unref();
+}
+
+startMonitoringTxns();
+
+export function setTxnExpiration(ms) {
+	clearInterval(timer);
+	txnExpiration = ms;
+	startMonitoringTxns();
+	return trackedTxns;
+}

package/resources/ErrorResource.ts

@@ -0,0 +1,57 @@
+import { Resource } from './Resource.ts';
+import type { Context } from './ResourceInterface.ts';
+/**
+ * ErrorResource is a Resource that throws an error on any request, communicating to the client when attempts are made
+ * to access endpoints/resources that had an internal error in their configuration or setup. This helps ensure that
+ * if there is a problem with a resource, it is immediately apparent and can be fixed.
+ */
+export class ErrorResource implements Resource {
+	error: Error;
+	constructor(error: Error) {
+		this.error = error;
+	}
+	isError = true;
+	allowRead(): never {
+		throw this.error;
+	}
+	allowUpdate(): never {
+		throw this.error;
+	}
+	allowCreate(): never {
+		throw this.error;
+	}
+	allowDelete(): never {
+		throw this.error;
+	}
+	getId(): never {
+		throw this.error;
+	}
+	getContext(): Context {
+		throw this.error;
+	}
+	get(): never {
+		throw this.error;
+	}
+	post(): never {
+		throw this.error;
+	}
+	put(): never {
+		throw this.error;
+	}
+	delete(): never {
+		throw this.error;
+	}
+	connect(): never {
+		throw this.error;
+	}
+	getResource() {
+		// all child paths resolve back to reporting this error
+		return this;
+	}
+	publish(): never {
+		throw this.error;
+	}
+	subscribe(): never {
+		throw this.error;
+	}
+}

package/resources/IterableEventQueue.ts

@@ -0,0 +1,94 @@
+import { EventEmitter } from 'events';
+
+export class IterableEventQueue<Event extends object = any> extends EventEmitter {
+	resolveNext: null | ((args: { value: Event }) => void) = null;
+	queue: any[];
+	hasDataListeners: boolean;
+	drainCloseListener: boolean;
+	currentDrainResolver: null | ((draining: boolean) => void) = null;
+	[Symbol.asyncIterator](): AsyncIterator<Event> {
+		const iterator = new EventQueueIterator<Event>();
+		iterator.queue = this;
+		// @ts-expect-error The EventQueueIterator is acceptable as an AsyncIterator
+		return iterator;
+	}
+	push(message: Event) {
+		this.send(message);
+	}
+	send(message: Event) {
+		if (this.resolveNext) {
+			this.resolveNext({ value: message });
+			this.resolveNext = null;
+		} else if (this.hasDataListeners) {
+			this.emit('data', message);
+		} else {
+			if (!this.queue) this.queue = [];
+			this.queue.push(message);
+		}
+	}
+	getNextMessage() {
+		const message = this.queue?.shift();
+		if (!message) this.emit('drained');
+		return message;
+	}
+
+	/**
+	 * Wait for the queue to be drained, resolving to true to continue or false if the queue was closed before draining.
+	 */
+	waitForDrain(): Promise<boolean> {
+		return new Promise((resolve) => {
+			if (!this.queue || this.queue.length === 0) resolve(true);
+			else {
+				this.once('drained', () => resolve(true));
+				this.currentDrainResolver = resolve;
+				if (!this.drainCloseListener) {
+					this.drainCloseListener = true;
+					this.on('close', () => {
+						this.currentDrainResolver?.(false);
+					});
+				}
+			}
+		});
+	}
+	on(eventName: 'data' | string, listener: ((data: Event) => void) | any) {
+		if (eventName === 'data' && !this.hasDataListeners) {
+			this.hasDataListeners = true;
+			while (this.queue?.length > 0) listener(this.queue.shift());
+		}
+		return super.on(eventName, listener);
+	}
+}
+
+class EventQueueIterator<Event extends object = any> implements AsyncIterator<Event> {
+	queue: IterableEventQueue<Event>;
+	push(message: Event) {
+		this.queue.send(message);
+	}
+	// @ts-expect-error TypeScript is wrong, the JS engine accepts MaybePromise<...>
+	next(): { value: Event } | Promise<{ value: Event }> {
+		const message = this.queue.getNextMessage();
+		if (message) {
+			return {
+				value: message,
+			};
+		} else {
+			return new Promise((resolve) => (this.queue.resolveNext = resolve));
+		}
+	}
+	// @ts-expect-error TypeScript is wrong, the JS engine accepts MaybePromise<...>
+	return(value: Event): { value: Event; done: true } {
+		this.queue.emit('close');
+		return {
+			value,
+			done: true,
+		};
+	}
+	// @ts-expect-error TypeScript is wrong, the JS engine accepts MaybePromise<...>
+	throw(error) {
+		this.queue.emit('close', error);
+		return {
+			done: true,
+			value: undefined,
+		};
+	}
+}