@harperfast/harper-pro 5.0.7 → 5.0.8

package/core/bin/status.js

@@ -6,7 +6,7 @@ const YAML = require('yaml');
 
 const hdbTerms = require('../utility/hdbTerms.ts');
 const hdbLog = require('../utility/logging/harper_logger.js');
-const sysInfo = require('../utility/environment/systemInformation.js');
+const systemInformation = require('../utility/environment/systemInformation.ts');
 const envMgr = require('../utility/environment/environmentManager.js');
 const installation = require('../utility/installation.ts');
 envMgr.initSync();
@@ -51,7 +51,7 @@ async function status() {
 	}
 
 	// Check the saved pid against any running hdb processes
-	const hdbSysInfo = await sysInfo.getHDBProcessInfo();
+	const hdbSysInfo = await systemInformation.getHDBProcessInfo();
 	for (const proc of hdbSysInfo.core) {
 		if (proc.pid === hdbPid) {
 			status.harperdb.status = STATUSES.RUNNING;

package/core/bin/stop.js

@@ -4,7 +4,7 @@ const hdbLogger = require('../utility/logging/harper_logger.js');
 const util = require('util');
 const childProcess = require('child_process');
 const exec = util.promisify(childProcess.exec);
-const sysInfo = require('../utility/environment/systemInformation.js');
+const systemInformation = require('../utility/environment/systemInformation.ts');
 
 const STOP_MSG = 'Stopping Harper Pro.';
 
@@ -14,9 +14,8 @@ async function stop() {
 	console.log(STOP_MSG);
 	hdbLogger.notify(STOP_MSG);
 
-	const processes = await sysInfo.getHDBProcessInfo();
-
-	processes.core.forEach((p) => {
-		exec(`kill ${p.pid}`);
-	});
+	const processes = await systemInformation.getHDBProcessInfo();
+	for (const { pid } of processes.core) {
+		exec(`kill ${pid}`);
+	}
 }

package/core/components/OptionsWatcher.ts

@@ -178,7 +178,9 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
 		// First, ensure current and new config values are Config objects (not null, undefined, or a primitive)
 		if (!this.#isConfig(currentConfigValue) || !this.#isConfig(newConfigValue)) {
 			// If either is not a config, then just set as there is no need to diff/merge
-			this.#setValue(prevKeys, newConfigValue);
+			if (!isDeepStrictEqual(currentConfigValue, newConfigValue)) {
+				this.#setValue(prevKeys, newConfigValue);
+			}
 			return;
 		}
 
@@ -256,6 +258,12 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
 			throw new InvalidValueTypeError(keys, value);
 		}
 
+		if (keys.length === 0) {
+			this.#scopedConfig = value;
+			this.emit('change', keys, value, this.#scopedConfig);
+			return;
+		}
+
 		let obj: ConfigValue = this.#scopedConfig;
 
 		for (const key of keys.slice(0, -1)) {

package/core/dataLayer/harperBridge/TableSizeObject.ts

@@ -0,0 +1,35 @@
+/**
+ * Represents the table size entry for a RocksDB or LMDB table.
+ */
+export class TableSizeObject {
+	schema: string;
+	table: string;
+	tableSize: number;
+	recordCount: number;
+	transactionLogSize: number;
+	transactionLogRecordCount?: number;
+
+	/**
+	 * @param schema - The schema of the table
+	 * @param table - The name of the table
+	 * @param tableSize - The data size of the table in bytes
+	 * @param recordCount - The number of entries in the table
+	 * @param transactionLogSize - The number of entries in the transaction log
+	 * @param transactionLogRecordCount - The data size of the transaction log in bytes
+	 */
+	constructor(
+		schema: string,
+		table: string,
+		tableSize: number = 0,
+		recordCount: number = 0,
+		transactionLogSize: number = 0,
+		transactionLogRecordCount?: number
+	) {
+		this.schema = schema;
+		this.table = table;
+		this.tableSize = tableSize;
+		this.recordCount = recordCount;
+		this.transactionLogSize = transactionLogSize;
+		this.transactionLogRecordCount = transactionLogRecordCount;
+	}
+}

package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts

@@ -0,0 +1,24 @@
+import { TableSizeObject } from '../../TableSizeObject.ts';
+import logger from '../../../../utility/logging/harper_logger.js';
+import type { Table } from '../../../../resources/databases.ts';
+
+/**
+ * Calculates the number of entries & data size in bytes for a table & its transaction log
+ * @param table
+ * @returns {TableSizeObject}
+ */
+export function lmdbGetTableSize(table: Table) {
+	const tableStats = new TableSizeObject(table.databaseName, table.tableName);
+	try {
+		const dbiStat = table.primaryStore.getStats();
+
+		//get the txn log record count
+		const txnDbiStat = table.auditStore?.getStats();
+
+		tableStats.recordCount = dbiStat.entryCount;
+		tableStats.transactionLogRecordCount = txnDbiStat.entryCount;
+	} catch (e) {
+		logger.warn(`unable to stat table dbi due to ${e}`);
+	}
+	return tableStats;
+}

package/core/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "harper",
-	"version": "5.0.7",
+	"version": "5.0.8",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "harper",
-			"version": "5.0.7",
+			"version": "5.0.8",
 			"license": "Apache-2.0",
 			"dependencies": {
 				"@aws-sdk/client-s3": "^3.1012.0",
@@ -16,7 +16,7 @@
 				"@fastify/autoload": "^6.3.1",
 				"@fastify/compress": "^8.3.1",
 				"@fastify/cors": "^11.2.0",
-				"@fastify/static": "^9.0.0",
+				"@fastify/static": "^9.1.3",
 				"@harperfast/extended-iterable": "^1.0.1",
 				"@harperfast/rocksdb-js": "^1.0.1",
 				"@turf/area": "6.5.0",
@@ -2160,9 +2160,9 @@
 			}
 		},
 		"node_modules/@fastify/static": {
-			"version": "9.0.0",
-			"resolved": "https://registry.npmjs.org/@fastify/static/-/static-9.0.0.tgz",
-			"integrity": "sha512-r64H8Woe/vfilg5RTy7lwWlE8ZZcTrc3kebYFMEUBrMqlydhQyoiExQXdYAy2REVpST/G35+stAM8WYp1WGmMA==",
+			"version": "9.1.3",
+			"resolved": "https://registry.npmjs.org/@fastify/static/-/static-9.1.3.tgz",
+			"integrity": "sha512-aXrYtsiryLhRxRNaxNqsn7FUISeb7rB9q4eHUPIot5aeQBLNahnz1m6thzm7JWC1poSGXS9XrX8DvuMivp2hkQ==",
 			"funding": [
 				{
 					"type": "github",

package/core/resources/DatabaseTransaction.ts

@@ -256,9 +256,14 @@ export class DatabaseTransaction implements Transaction {
 				if (!outstandingCommit) {
 					outstandingCommit = commitResolution;
 					outstandingCommitStart = performance.now();
-					outstandingCommit.finally(() => {
-						outstandingCommit = null;
-					});
+					outstandingCommit
+						// if `commitResolution` rejects with and `ERR_BUSY` error, the retry logic
+						// will correct course, but the reject will still be propagated on the
+						// `outstandingCommit` promise and needs to be caught and silenced
+						.catch(() => {})
+						.finally(() => {
+							outstandingCommit = null;
+						});
 				}
 				const completions = [];
 				return commitResolution.then(

package/core/resources/RecordEncoder.ts

@@ -470,6 +470,7 @@ export function handleLocalTimeForGets(store, rootStore) {
 			};
 			Txn.prototype.done = function () {
 				done.call(this);
+				this.openTimer = 0; // reset so idle pool time doesn't accumulate toward the stale-open threshold
 				if (this.isDone) {
 					for (let i = 0; i < trackedTxns.length; i++) {
 						const txn = trackedTxns[i].deref();
@@ -498,7 +499,14 @@ setInterval(() => {
 							'Read transaction detected that has been open too long (over 15 minutes), ending transaction',
 							txn
 						);
-						txn.done();
+						trackedTxns.splice(i--, 1);
+						txn.timerTracked = false;
+						txn.openTimer = 0;
+						try {
+							txn.done();
+						} catch (error) {
+							harperLogger.warn('Unexpected error force-closing stale LMDB read transaction', error);
+						}
 					} else
 						harperLogger.error(
 							'Read transaction detected that has been open too long (over one minute), make sure read transactions are quickly closed',

package/core/resources/Table.ts

@@ -363,6 +363,10 @@ export function makeTable(options) {
 						// we listen for events by iterating through the async iterator provided by the subscription
 						for await (const event of subscription) {
 							try {
+								if (!event || typeof event !== 'object') {
+									logger.error?.('Bad subscription event', event);
+									continue;
+								}
 								const firstWrite = event.type === 'transaction' ? event.writes[0] : event;
 								if (!firstWrite) {
 									logger.error?.('Bad subscription event', event);
@@ -1425,7 +1429,8 @@ export function makeTable(options) {
 		 */
 		static evict(id, existingRecord, existingVersion) {
 			let entry;
-			let transaction = txnForContext({ transaction: new DatabaseTransaction() }).getReadTxn();
+			const lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() });
+			let transaction = lmdbTransaction.getReadTxn();
 			let options = { transaction };
 			try {
 				if (hasSourceGet || audit) {
@@ -1452,7 +1457,13 @@ export function makeTable(options) {
 					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
 				}
 			} finally {
-				return transaction.commit();
+				if (primaryStore.ifVersion) {
+					// LMDB: committing the wrapper calls doneReadTxn(), removing it from trackedTxns
+					return lmdbTransaction.commit();
+				}
+				// RocksDB: eviction writes went directly into the raw transaction via options;
+				// commit it directly, as DatabaseTransaction.commit() would abort it (no tracked writes)
+				return transaction?.commit();
 			}
 		}
 		/**

package/core/resources/databases.ts

@@ -368,7 +368,7 @@ function readRocksMetaDb(path: string, defaultTable?: string, databaseName: stri
 		if (rootStore) {
 			initStores(path, rootStore, databaseName, defaultTable);
 		} else {
-			rootStore = openRocksDatabase(path, { disableWAL: false }) as RocksDatabaseEx;
+			rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true }) as RocksDatabaseEx;
 			rocksdbDatabaseEnvs.set(path, rootStore);
 			initStores(path, rootStore, databaseName, defaultTable);
 			replayLogs(rootStore, databases[databaseName]);
@@ -723,6 +723,7 @@ export function database({ database: databaseName, table: tableName }) {
 		if (!rootStore || rootStore.status === 'closed') {
 			rootStore = openRocksDatabase(path, {
 				disableWAL: false,
+				enableStats: true,
 			});
 			rocksdbDatabaseEnvs.set(path, rootStore);
 		}

package/core/resources/graphql.ts

@@ -4,6 +4,7 @@ import { table } from './databases.ts';
 import { getWorkerIndex } from '../server/threads/manageThreads.js';
 import { Resources } from './Resources.ts';
 import type { NamedTypeNode, StringValueNode } from 'graphql';
+import { once } from 'node:events';
 
 const PRIMITIVE_TYPES = ['ID', 'Int', 'Float', 'Long', 'String', 'Boolean', 'Date', 'Bytes', 'Any', 'BigInt', 'Blob'];
 
@@ -35,11 +36,18 @@ server.knownGraphQLDirectives.push(
  * @param filePath
  * @param resources
  */
-export function start({ ensureTable }) {
-	return {
-		handleFile,
-		setupFile: handleFile,
-	};
+export function handleApplication(scope: import('../components/Scope.ts').Scope) {
+	const entryHandler = scope.handleEntry(async (entry) => {
+		if (entry.eventType === 'unlink') return;
+		if (entry.entryType === 'directory') {
+			scope.logger.warn?.('graphqlSchema currently does not handle directories. Specify file patterns only.');
+			return;
+		}
+
+		await processGraphQLSchema(entry.contents, entry.urlPath, entry.absolutePath, scope.resources);
+	});
+	return once(entryHandler, 'initialLoadComplete');
+}
 
 	async function handleFile(gqlContent, urlPath, filePath, resources) {
 		// lazy load the graphql package so we don't load it for users that don't use graphql

package/core/security/jsLoader.ts

@@ -181,7 +181,13 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC
 		if (parts[0] === 'file:') {
 			return specifier;
 		}
-		const resolved = createRequire(referrer).resolve(specifier);
+		let resolveReferrer = referrer;
+		if (referrer.startsWith('file:')) {
+			try {
+				resolveReferrer = pathToFileURL(realpathSync(fileURLToPath(referrer))).toString();
+			} catch {}
+		}
+		const resolved = createRequire(resolveReferrer).resolve(specifier);
 		if (isAbsolute(resolved)) {
 			return pathToFileURL(resolved).toString();
 		}
@@ -206,7 +212,13 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC
 			cjsModule.exports = parseJsonModule(source, url);
 			return cjsModule;
 		}
-		const require = createRequire(url);
+		let requireUrl = url;
+		if (url.startsWith('file://')) {
+			try {
+				requireUrl = pathToFileURL(realpathSync(fileURLToPath(url))).toString();
+			} catch {}
+		}
+		const require = createRequire(requireUrl);
 
 		const cjsRequire = (spec: string) => {
 			const resolvedUrl = resolveModule(spec, url);

package/core/security/keys.js

@@ -857,7 +857,7 @@ function createTLSSelector(type, mtlsOptions) {
 	return SNICallback;
 	function SNICallback(servername, cb) {
 		// find the matching server name, substituting wildcards for each part of the domain to find matches
-		logger.info?.('TLS requested for', servername || '(no SNI)');
+		logger.debug?.('TLS requested for', servername || '(no SNI)');
 		let matchingName = servername;
 		while (true) {
 			let context = secureContexts.get(matchingName);

package/core/server/DurableSubscriptionsSession.ts

@@ -257,6 +257,7 @@ class SubscriptionsSession {
 			const _result = (async () => {
 				for await (const update of subscription) {
 					try {
+						if (!update || typeof update !== 'object') continue;
 						let messageId;
 						if (
 							update.type &&

package/core/server/serverHelpers/serverUtilities.ts

@@ -19,7 +19,7 @@ import restart from '../../bin/restart.js';
 import * as util from 'util';
 import insert from '../../dataLayer/insert.js';
 import globalSchema from '../../utility/globalSchema.js';
-import systemInformation from '../../utility/environment/systemInformation.js';
+import { systemInformation } from '../../utility/environment/systemInformation.ts';
 import jobRunner from '../jobs/jobRunner.js';
 import * as tokenAuthentication from '../../security/tokenAuthentication.ts';
 import * as auth from '../../security/auth.ts';
@@ -363,10 +363,7 @@ function initializeOperationFunctionMap(): Map<OperationFunctionName, OperationF
 	opFuncMap.set(terms.OPERATIONS_ENUM.RESTART, new OperationFunctionObject(restart.restart));
 	opFuncMap.set(terms.OPERATIONS_ENUM.RESTART_SERVICE, new OperationFunctionObject(executeJob, restart.restartService));
 	opFuncMap.set(terms.OPERATIONS_ENUM.CATCHUP, new OperationFunctionObject(catchup));
-	opFuncMap.set(
-		terms.OPERATIONS_ENUM.SYSTEM_INFORMATION,
-		new OperationFunctionObject(systemInformation.systemInformation)
-	);
+	opFuncMap.set(terms.OPERATIONS_ENUM.SYSTEM_INFORMATION, new OperationFunctionObject(systemInformation));
 	opFuncMap.set(
 		terms.OPERATIONS_ENUM.DELETE_AUDIT_LOGS_BEFORE,
 		new OperationFunctionObject(executeJob, delete_.deleteAuditLogsBefore)