@harperfast/harper-pro 5.0.7 → 5.0.9

package/core/bin/status.js

@@ -6,7 +6,7 @@ const YAML = require('yaml');
 
 const hdbTerms = require('../utility/hdbTerms.ts');
 const hdbLog = require('../utility/logging/harper_logger.js');
-const sysInfo = require('../utility/environment/systemInformation.js');
+const systemInformation = require('../utility/environment/systemInformation.ts');
 const envMgr = require('../utility/environment/environmentManager.js');
 const installation = require('../utility/installation.ts');
 envMgr.initSync();
@@ -51,7 +51,7 @@ async function status() {
 	}
 
 	// Check the saved pid against any running hdb processes
-	const hdbSysInfo = await sysInfo.getHDBProcessInfo();
+	const hdbSysInfo = await systemInformation.getHDBProcessInfo();
 	for (const proc of hdbSysInfo.core) {
 		if (proc.pid === hdbPid) {
 			status.harperdb.status = STATUSES.RUNNING;

package/core/bin/stop.js

@@ -4,7 +4,7 @@ const hdbLogger = require('../utility/logging/harper_logger.js');
 const util = require('util');
 const childProcess = require('child_process');
 const exec = util.promisify(childProcess.exec);
-const sysInfo = require('../utility/environment/systemInformation.js');
+const systemInformation = require('../utility/environment/systemInformation.ts');
 
 const STOP_MSG = 'Stopping Harper Pro.';
 
@@ -14,9 +14,8 @@ async function stop() {
 	console.log(STOP_MSG);
 	hdbLogger.notify(STOP_MSG);
 
-	const processes = await sysInfo.getHDBProcessInfo();
-
-	processes.core.forEach((p) => {
-		exec(`kill ${p.pid}`);
-	});
+	const processes = await systemInformation.getHDBProcessInfo();
+	for (const { pid } of processes.core) {
+		exec(`kill ${pid}`);
+	}
 }

package/core/components/OptionsWatcher.ts

@@ -178,7 +178,9 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
 		// First, ensure current and new config values are Config objects (not null, undefined, or a primitive)
 		if (!this.#isConfig(currentConfigValue) || !this.#isConfig(newConfigValue)) {
 			// If either is not a config, then just set as there is no need to diff/merge
-			this.#setValue(prevKeys, newConfigValue);
+			if (!isDeepStrictEqual(currentConfigValue, newConfigValue)) {
+				this.#setValue(prevKeys, newConfigValue);
+			}
 			return;
 		}
 
@@ -256,6 +258,12 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
 			throw new InvalidValueTypeError(keys, value);
 		}
 
+		if (keys.length === 0) {
+			this.#scopedConfig = value;
+			this.emit('change', keys, value, this.#scopedConfig);
+			return;
+		}
+
 		let obj: ConfigValue = this.#scopedConfig;
 
 		for (const key of keys.slice(0, -1)) {

package/core/dataLayer/harperBridge/TableSizeObject.ts

@@ -0,0 +1,35 @@
+/**
+ * Represents the table size entry for a RocksDB or LMDB table.
+ */
+export class TableSizeObject {
+	schema: string;
+	table: string;
+	tableSize: number;
+	recordCount: number;
+	transactionLogSize: number;
+	transactionLogRecordCount?: number;
+
+	/**
+	 * @param schema - The schema of the table
+	 * @param table - The name of the table
+	 * @param tableSize - The data size of the table in bytes
+	 * @param recordCount - The number of entries in the table
+	 * @param transactionLogSize - The number of entries in the transaction log
+	 * @param transactionLogRecordCount - The data size of the transaction log in bytes
+	 */
+	constructor(
+		schema: string,
+		table: string,
+		tableSize: number = 0,
+		recordCount: number = 0,
+		transactionLogSize: number = 0,
+		transactionLogRecordCount?: number
+	) {
+		this.schema = schema;
+		this.table = table;
+		this.tableSize = tableSize;
+		this.recordCount = recordCount;
+		this.transactionLogSize = transactionLogSize;
+		this.transactionLogRecordCount = transactionLogRecordCount;
+	}
+}

package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts

@@ -0,0 +1,24 @@
+import { TableSizeObject } from '../../TableSizeObject.ts';
+import logger from '../../../../utility/logging/harper_logger.js';
+import type { Table } from '../../../../resources/databases.ts';
+
+/**
+ * Calculates the number of entries & data size in bytes for a table & its transaction log
+ * @param table
+ * @returns {TableSizeObject}
+ */
+export function lmdbGetTableSize(table: Table) {
+	const tableStats = new TableSizeObject(table.databaseName, table.tableName);
+	try {
+		const dbiStat = table.primaryStore.getStats();
+
+		//get the txn log record count
+		const txnDbiStat = table.auditStore?.getStats();
+
+		tableStats.recordCount = dbiStat.entryCount;
+		tableStats.transactionLogRecordCount = txnDbiStat.entryCount;
+	} catch (e) {
+		logger.warn(`unable to stat table dbi due to ${e}`);
+	}
+	return tableStats;
+}

package/core/index.ts

@@ -0,0 +1,113 @@
+// Prevents server from starting in worker threads if this was directly imported from a non-server user thread
+import workerThreads from 'node:worker_threads';
+if (!workerThreads.isMainThread) {
+	// @ts-expect-error - Idk this has been here for a while. Types say its readonly, but that must not be true.
+	if (!workerThreads.workerData) workerThreads.workerData = {};
+	workerThreads.workerData.noServerStart = true;
+}
+
+// Regular exports (don't require the same initialization as the globals at the end of this file do)
+export { RequestTarget } from './resources/RequestTarget.ts';
+export { getContext, getResponse, getUser } from './security/jsLoader.ts';
+
+// Type only exports.
+// Anything exported here will only be available as TypeScript types, not as values.
+// For exporting values see below.
+export type {
+	Query,
+	Context,
+	Session,
+	SourceContext,
+	SubscriptionRequest,
+	RequestTargetOrId,
+	ResourceInterface,
+} from './resources/ResourceInterface.ts';
+export type { User } from './security/user.ts';
+export type { RecordObject } from './resources/RecordEncoder.ts';
+export type { IterableEventQueue } from './resources/IterableEventQueue.ts';
+export type { Table } from './resources/databases.ts';
+export type { Attribute } from './resources/Table.ts';
+export type { Scope } from './components/Scope.ts';
+export type { FilesOption, FilesOptionObject } from './components/deriveGlobOptions.ts';
+export type { FileAndURLPathConfig } from './components/Component.ts';
+export type { OptionsWatcher, Config, ConfigValue } from './components/OptionsWatcher.ts';
+export type {
+	EntryHandler,
+	BaseEntry,
+	FileEntry,
+	EntryEvent,
+	AddFileEvent,
+	ChangeFileEvent,
+	UnlinkFileEvent,
+	FileEntryEvent,
+	AddDirectoryEvent,
+	UnlinkDirectoryEvent,
+	DirectoryEntryEvent,
+} from './components/EntryHandler.ts';
+
+// Globals and values
+// This section is responsible for creating the CJS exports map (for static analysis)
+// as well as defining the globals and values exports.
+// The stuff exported here are actually functional pieces of code.
+// Importantly, do not import any values directly.
+// For example, `import { tables } from './resources/databases.ts';` is NOT OKAY!
+// This breaks Harper's dynamic runtime assignment of exports
+// You MUST import as a type and then use `export declare const` instead.
+// This results in the types being written to dist/index.d.ts, but not dist/index.js
+
+// And for my sanity please keep these alphabetically sorted so we can ensure nothing is missing.
+
+import type { contentTypes as ContentTypesImport } from './server/serverHelpers/contentTypes.ts';
+import type { createBlob as CreateBlobImport } from './resources/blob.ts';
+import type { databases as DatabasesImport } from './resources/databases.ts';
+import type { Logger } from './utility/logging/logger.ts';
+import type { operation as OperationImport } from './server/serverHelpers/serverUtilities.ts';
+import type { Resource as ResourceImport } from './resources/Resource.ts';
+import type { server as ServerImport } from './server/Server.ts';
+import type { tables as TablesImport } from './resources/databases.ts';
+type ThreadsImport = unknown[]; // TODO: figure out actual type for this
+import type { transaction as TransactionImport } from './resources/transaction.ts';
+
+declare global {
+	const contentTypes: typeof ContentTypesImport;
+	const createBlob: typeof CreateBlobImport;
+	const databases: typeof DatabasesImport;
+	const logger: Logger;
+	const operation: typeof OperationImport;
+	const Resource: typeof ResourceImport;
+	const server: typeof ServerImport;
+	const tables: typeof TablesImport;
+	const threads: ThreadsImport;
+	const transaction: typeof TransactionImport;
+}
+
+// Declare constant types so these are defined in `index.d.ts`
+export declare const contentTypes: typeof ContentTypesImport;
+export declare const createBlob: typeof CreateBlobImport;
+export declare const databases: typeof DatabasesImport;
+export declare const logger: Logger;
+export declare const operation: typeof OperationImport;
+export declare const Resource: typeof ResourceImport;
+export declare const server: typeof ServerImport;
+export declare const tables: typeof TablesImport;
+export declare const threads: ThreadsImport;
+export declare const transaction: typeof TransactionImport;
+
+// Actual define the values on the `exports` for CJS static analysis
+exports.contentTypes = null;
+exports.createBlob = undefined;
+exports.databases = {};
+exports.logger = {};
+exports.operation = undefined;
+exports.Resource = undefined;
+exports.server = {};
+exports.tables = {};
+exports.threads = [];
+exports.transaction = undefined;
+
+// And finally assign globals to exports.
+// These values are populated at runtime by `_assignPackageExport()` in their respective modules
+// (e.g. Resource.ts, databases.ts, Server.ts, etc.)
+import { globals } from './server/threads/threadServer.js';
+
+Object.assign(exports, globals);

package/core/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "harper",
-	"version": "5.0.7",
+	"version": "5.0.9",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "harper",
-			"version": "5.0.7",
+			"version": "5.0.9",
 			"license": "Apache-2.0",
 			"dependencies": {
 				"@aws-sdk/client-s3": "^3.1012.0",
@@ -16,7 +16,7 @@
 				"@fastify/autoload": "^6.3.1",
 				"@fastify/compress": "^8.3.1",
 				"@fastify/cors": "^11.2.0",
-				"@fastify/static": "^9.0.0",
+				"@fastify/static": "^9.1.3",
 				"@harperfast/extended-iterable": "^1.0.1",
 				"@harperfast/rocksdb-js": "^1.0.1",
 				"@turf/area": "6.5.0",
@@ -2160,9 +2160,9 @@
 			}
 		},
 		"node_modules/@fastify/static": {
-			"version": "9.0.0",
-			"resolved": "https://registry.npmjs.org/@fastify/static/-/static-9.0.0.tgz",
-			"integrity": "sha512-r64H8Woe/vfilg5RTy7lwWlE8ZZcTrc3kebYFMEUBrMqlydhQyoiExQXdYAy2REVpST/G35+stAM8WYp1WGmMA==",
+			"version": "9.1.3",
+			"resolved": "https://registry.npmjs.org/@fastify/static/-/static-9.1.3.tgz",
+			"integrity": "sha512-aXrYtsiryLhRxRNaxNqsn7FUISeb7rB9q4eHUPIot5aeQBLNahnz1m6thzm7JWC1poSGXS9XrX8DvuMivp2hkQ==",
 			"funding": [
 				{
 					"type": "github",

package/core/resources/DatabaseTransaction.ts

@@ -256,9 +256,14 @@ export class DatabaseTransaction implements Transaction {
 				if (!outstandingCommit) {
 					outstandingCommit = commitResolution;
 					outstandingCommitStart = performance.now();
-					outstandingCommit.finally(() => {
-						outstandingCommit = null;
-					});
+					outstandingCommit
+						// if `commitResolution` rejects with and `ERR_BUSY` error, the retry logic
+						// will correct course, but the reject will still be propagated on the
+						// `outstandingCommit` promise and needs to be caught and silenced
+						.catch(() => {})
+						.finally(() => {
+							outstandingCommit = null;
+						});
 				}
 				const completions = [];
 				return commitResolution.then(

package/core/resources/RecordEncoder.ts

@@ -64,8 +64,8 @@ const TRACKED_WRITE_TYPES = new Set(['put', 'patch', 'delete', 'message', 'publi
 // WeakMaps are definitely not the fastest form of private properties, but they are the only
 // way to do this with how the objects are frozen for now.
 export const entryMap = new WeakMap<any, Entry>();
-let lastValueEncoding,
-	timestampNextEncoding = 0,
+export let lastValueEncoding: Buffer | undefined;
+let timestampNextEncoding = 0,
 	metadataInNextEncoding = -1,
 	expiresAtNextEncoding = -1,
 	residencyIdAtNextEncoding = 0,
@@ -470,6 +470,7 @@ export function handleLocalTimeForGets(store, rootStore) {
 			};
 			Txn.prototype.done = function () {
 				done.call(this);
+				this.openTimer = 0; // reset so idle pool time doesn't accumulate toward the stale-open threshold
 				if (this.isDone) {
 					for (let i = 0; i < trackedTxns.length; i++) {
 						const txn = trackedTxns[i].deref();
@@ -498,7 +499,14 @@ setInterval(() => {
 							'Read transaction detected that has been open too long (over 15 minutes), ending transaction',
 							txn
 						);
-						txn.done();
+						trackedTxns.splice(i--, 1);
+						txn.timerTracked = false;
+						txn.openTimer = 0;
+						try {
+							txn.done();
+						} catch (error) {
+							harperLogger.warn('Unexpected error force-closing stale LMDB read transaction', error);
+						}
 					} else
 						harperLogger.error(
 							'Read transaction detected that has been open too long (over one minute), make sure read transactions are quickly closed',

package/core/resources/Table.ts

@@ -363,6 +363,10 @@ export function makeTable(options) {
 						// we listen for events by iterating through the async iterator provided by the subscription
 						for await (const event of subscription) {
 							try {
+								if (!event || typeof event !== 'object') {
+									logger.error?.('Bad subscription event', event);
+									continue;
+								}
 								const firstWrite = event.type === 'transaction' ? event.writes[0] : event;
 								if (!firstWrite) {
 									logger.error?.('Bad subscription event', event);
@@ -1425,7 +1429,8 @@ export function makeTable(options) {
 		 */
 		static evict(id, existingRecord, existingVersion) {
 			let entry;
-			let transaction = txnForContext({ transaction: new DatabaseTransaction() }).getReadTxn();
+			const lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() });
+			let transaction = lmdbTransaction.getReadTxn();
 			let options = { transaction };
 			try {
 				if (hasSourceGet || audit) {
@@ -1452,7 +1457,13 @@ export function makeTable(options) {
 					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
 				}
 			} finally {
-				return transaction.commit();
+				if (primaryStore.ifVersion) {
+					// LMDB: committing the wrapper calls doneReadTxn(), removing it from trackedTxns
+					return lmdbTransaction.commit();
+				}
+				// RocksDB: eviction writes went directly into the raw transaction via options;
+				// commit it directly, as DatabaseTransaction.commit() would abort it (no tracked writes)
+				return transaction?.commit();
 			}
 		}
 		/**

package/core/resources/blob.ts

@@ -44,6 +44,7 @@ import { HAS_BLOBS } from './auditStore.ts';
 import { getHeapStatistics } from 'node:v8';
 import { setTimeout as delay, setImmediate as rest } from 'node:timers/promises';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
+import { _assignPackageExport } from '../globals.js';
 
 type StorageInfo = {
 	storageIndex: number;
@@ -503,7 +504,10 @@ export type BlobCreationOptions = {
  * can be saved/stored.
  * @param source
  */
-global.createBlob = function (source: NodeJS.ReadableStream | NodeJS.Buffer, options?: BlobCreationOptions): Blob {
+export function createBlob(
+	source: Uint8Array | NodeJS.ReadableStream | string | Iterable<Uint8Array> | AsyncIterator<Uint8Array>,
+	options?: BlobCreationOptions
+): Blob {
 	const blob = new FileBackedBlob(options);
 	const storageInfo: StorageInfo = {
 		storageIndex: 0,
@@ -521,7 +525,8 @@ global.createBlob = function (source: NodeJS.ReadableStream | NodeJS.Buffer, opt
 	else if (source?.[Symbol.asyncIterator] || source?.[Symbol.iterator]) storageInfo.source = Readable.from(source);
 	else throw new Error('Invalid source type');
 	return blob;
-};
+}
+_assignPackageExport('createBlob', createBlob);
 
 export function saveBlob(blob: FileBackedBlob, deleteOnFailure = false) {
 	let storageInfo = storageInfoForBlob.get(blob);

package/core/resources/databases.ts

@@ -368,7 +368,7 @@ function readRocksMetaDb(path: string, defaultTable?: string, databaseName: stri
 		if (rootStore) {
 			initStores(path, rootStore, databaseName, defaultTable);
 		} else {
-			rootStore = openRocksDatabase(path, { disableWAL: false }) as RocksDatabaseEx;
+			rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true }) as RocksDatabaseEx;
 			rocksdbDatabaseEnvs.set(path, rootStore);
 			initStores(path, rootStore, databaseName, defaultTable);
 			replayLogs(rootStore, databases[databaseName]);
@@ -723,6 +723,7 @@ export function database({ database: databaseName, table: tableName }) {
 		if (!rootStore || rootStore.status === 'closed') {
 			rootStore = openRocksDatabase(path, {
 				disableWAL: false,
+				enableStats: true,
 			});
 			rocksdbDatabaseEnvs.set(path, rootStore);
 		}

package/core/security/jsLoader.ts

@@ -181,7 +181,13 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC
 		if (parts[0] === 'file:') {
 			return specifier;
 		}
-		const resolved = createRequire(referrer).resolve(specifier);
+		let resolveReferrer = referrer;
+		if (referrer.startsWith('file:')) {
+			try {
+				resolveReferrer = pathToFileURL(realpathSync(fileURLToPath(referrer))).toString();
+			} catch {}
+		}
+		const resolved = createRequire(resolveReferrer).resolve(specifier);
 		if (isAbsolute(resolved)) {
 			return pathToFileURL(resolved).toString();
 		}
@@ -206,7 +212,13 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC
 			cjsModule.exports = parseJsonModule(source, url);
 			return cjsModule;
 		}
-		const require = createRequire(url);
+		let requireUrl = url;
+		if (url.startsWith('file://')) {
+			try {
+				requireUrl = pathToFileURL(realpathSync(fileURLToPath(url))).toString();
+			} catch {}
+		}
+		const require = createRequire(requireUrl);
 
 		const cjsRequire = (spec: string) => {
 			const resolvedUrl = resolveModule(spec, url);
@@ -943,13 +955,13 @@ function checkAllowedModulePath(moduleUrl: string, allowedPath?: string): boolea
 	throw new Error(`Module ${moduleUrl} is not allowed to be imported`);
 }
 
-function getContext() {
+export function getContext() {
 	return contextStorage.getStore() ?? {};
 }
-function getUser() {
+export function getUser() {
 	return contextStorage.getStore()?.user;
 }
-function getResponse() {
+export function getResponse() {
 	return contextStorage.getStore()?.response;
 }
 

package/core/security/keys.js

@@ -857,7 +857,7 @@ function createTLSSelector(type, mtlsOptions) {
 	return SNICallback;
 	function SNICallback(servername, cb) {
 		// find the matching server name, substituting wildcards for each part of the domain to find matches
-		logger.info?.('TLS requested for', servername || '(no SNI)');
+		logger.debug?.('TLS requested for', servername || '(no SNI)');
 		let matchingName = servername;
 		while (true) {
 			let context = secureContexts.get(matchingName);

package/core/server/DurableSubscriptionsSession.ts

@@ -257,6 +257,7 @@ class SubscriptionsSession {
 			const _result = (async () => {
 				for await (const update of subscription) {
 					try {
+						if (!update || typeof update !== 'object') continue;
 						let messageId;
 						if (
 							update.type &&

package/core/server/serverHelpers/serverUtilities.ts

@@ -19,7 +19,7 @@ import restart from '../../bin/restart.js';
 import * as util from 'util';
 import insert from '../../dataLayer/insert.js';
 import globalSchema from '../../utility/globalSchema.js';
-import systemInformation from '../../utility/environment/systemInformation.js';
+import { systemInformation } from '../../utility/environment/systemInformation.ts';
 import jobRunner from '../jobs/jobRunner.js';
 import * as tokenAuthentication from '../../security/tokenAuthentication.ts';
 import * as auth from '../../security/auth.ts';
@@ -363,10 +363,7 @@ function initializeOperationFunctionMap(): Map<OperationFunctionName, OperationF
 	opFuncMap.set(terms.OPERATIONS_ENUM.RESTART, new OperationFunctionObject(restart.restart));
 	opFuncMap.set(terms.OPERATIONS_ENUM.RESTART_SERVICE, new OperationFunctionObject(executeJob, restart.restartService));
 	opFuncMap.set(terms.OPERATIONS_ENUM.CATCHUP, new OperationFunctionObject(catchup));
-	opFuncMap.set(
-		terms.OPERATIONS_ENUM.SYSTEM_INFORMATION,
-		new OperationFunctionObject(systemInformation.systemInformation)
-	);
+	opFuncMap.set(terms.OPERATIONS_ENUM.SYSTEM_INFORMATION, new OperationFunctionObject(systemInformation));
 	opFuncMap.set(
 		terms.OPERATIONS_ENUM.DELETE_AUDIT_LOGS_BEFORE,
 		new OperationFunctionObject(executeJob, delete_.deleteAuditLogsBefore)