@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/utility/hdbTerms.ts

@@ -177,6 +177,8 @@ export const SYSTEM_TABLE_NAMES = {
 	TABLE_TABLE_NAME: 'hdb_table',
 	USER_TABLE_NAME: 'hdb_user',
 	INFO_TABLE_NAME: 'hdb_info',
+	DEPLOYMENT_TABLE_NAME: 'hdb_deployment',
+	AGENT_SESSION_TABLE_NAME: 'hdb_agent_session',
 } as const;
 
 /** Hash attribute for the system info table */
@@ -297,6 +299,16 @@ export const OPERATIONS_ENUM = {
 	GET_STATUS: 'get_status',
 	SET_STATUS: 'set_status',
 	CLEAR_STATUS: 'clear_status',
+	LIST_DEPLOYMENTS: 'list_deployments',
+	GET_DEPLOYMENT: 'get_deployment',
+	GET_DEPLOYMENT_PAYLOAD: 'get_deployment_payload',
+	DELETE_DEPLOYMENT_PAYLOAD: 'delete_deployment_payload',
+	AGENT_PROMPT: 'agent_prompt',
+	GET_AGENT_SESSION: 'get_agent_session',
+	LIST_AGENT_SESSIONS: 'list_agent_sessions',
+	CANCEL_AGENT_RUN: 'cancel_agent_run',
+	APPROVE_AGENT_ACTION: 'approve_agent_action',
+	SET_AGENT_CONFIG: 'set_agent_config',
 } as const;
 
 /** Defines valid file types that we are able to handle in 'import_from_s3' ops */
@@ -337,9 +349,12 @@ export const SERVICE_ACTIONS_ENUM = {
 	UPGRADE: 'upgrade',
 	HELP: 'help',
 	STATUS: 'status',
+	LOGIN: 'login',
+	LOGOUT: 'logout',
 	OPERATION: 'operation',
 	RENEWCERTS: 'renew-certs',
 	COPYDB: 'copy-db',
+	MCP: 'mcp',
 } as const;
 
 /** describes the Geo Conversion types */
@@ -424,6 +439,8 @@ export const LEGACY_CONFIG_PARAMS = {
  */
 export const CONFIG_PARAMS = {
 	ANALYTICS_AGGREGATEPERIOD: 'analytics_aggregatePeriod',
+	ANALYTICS_AGGREGATERETENTIONMS: 'analytics_aggregateRetentionMs',
+	ANALYTICS_RAWRETENTIONMS: 'analytics_rawRetentionMs',
 	ANALYTICS_REPLICATE: 'analytics_replicate',
 	ANALYTICS_STORAGEINTERVAL: 'analytics_storageInterval',
 	AUTHENTICATION_AUTHORIZELOCAL: 'authentication_authorizeLocal',
@@ -514,6 +531,7 @@ export const CONFIG_PARAMS = {
 	OPERATIONSAPI_NETWORK_SECUREPORT: 'operationsApi_network_securePort',
 	OPERATIONSAPI_NETWORK_HTTP2: 'operationsApi_network_http2',
 	OPERATIONSAPI_NETWORK_MAXREQUESTBODYSIZE: 'operationsApi_network_maxRequestBodySize',
+	OPERATIONSAPI_COMPONENTFILE_MAXSIZE: 'operationsApi_componentFile_maxSize',
 	OPERATIONSAPI_TLS: 'operationsApi_tls',
 	OPERATIONSAPI_TLS_CERTIFICATE: 'operationsApi_tls_certificate',
 	OPERATIONSAPI_TLS_PRIVATEKEY: 'operationsApi_tls_privateKey',
@@ -521,6 +539,34 @@ export const CONFIG_PARAMS = {
 	OPERATIONSAPI_NETWORK_TIMEOUT: 'operationsApi_network_timeout',
 	OPERATIONSAPI_SYSINFO_NETWORK: 'operationsApi_sysInfo_network',
 	OPERATIONSAPI_SYSINFO_DISK: 'operationsApi_sysInfo_disk',
+	MCP_OPERATIONS_MOUNTPATH: 'mcp_operations_mountPath',
+	MCP_OPERATIONS_ALLOW: 'mcp_operations_allow',
+	MCP_OPERATIONS_DENY: 'mcp_operations_deny',
+	MCP_OPERATIONS_MAXTOOLS: 'mcp_operations_maxTools',
+	MCP_OPERATIONS_RATELIMIT_PERTOOLPERSECOND: 'mcp_operations_rateLimit_perToolPerSecond',
+	MCP_OPERATIONS_RATELIMIT_PERTOOLBURST: 'mcp_operations_rateLimit_perToolBurst',
+	MCP_OPERATIONS_RATELIMIT_SESSIONCONCURRENCY: 'mcp_operations_rateLimit_sessionConcurrency',
+	MCP_OPERATIONS_RATELIMIT_SESSIONPERSECOND: 'mcp_operations_rateLimit_sessionPerSecond',
+	MCP_APPLICATION_MOUNTPATH: 'mcp_application_mountPath',
+	MCP_APPLICATION_ALLOW: 'mcp_application_allow',
+	MCP_APPLICATION_DENY: 'mcp_application_deny',
+	MCP_APPLICATION_MAXTOOLS: 'mcp_application_maxTools',
+	MCP_APPLICATION_SEARCHMAXRESULTS: 'mcp_application_searchMaxResults',
+	MCP_APPLICATION_RATELIMIT_PERTOOLPERSECOND: 'mcp_application_rateLimit_perToolPerSecond',
+	MCP_APPLICATION_RATELIMIT_PERTOOLBURST: 'mcp_application_rateLimit_perToolBurst',
+	MCP_APPLICATION_RATELIMIT_SESSIONCONCURRENCY: 'mcp_application_rateLimit_sessionConcurrency',
+	MCP_APPLICATION_RATELIMIT_SESSIONPERSECOND: 'mcp_application_rateLimit_sessionPerSecond',
+	MCP_SESSION_IDLETIMEOUTSECONDS: 'mcp_session_idleTimeoutSeconds',
+	MCP_SESSION_ALLOWCLIENTDELETE: 'mcp_session_allowClientDelete',
+	AGENT_ENABLED: 'agent_enabled',
+	AGENT_PROVIDER: 'agent_provider',
+	AGENT_MODEL: 'agent_model',
+	AGENT_MAXTURNS: 'agent_maxTurns',
+	AGENT_MAXCOSTUSD: 'agent_maxCostUsd',
+	AGENT_AUTOAPPROVE: 'agent_autoApprove',
+	AGENT_ALLOWDESTRUCTIVE: 'agent_allowDestructive',
+	AGENT_USER: 'agent_user',
+	AGENT_COMPONENTSSCOPE: 'agent_componentsScope',
 	REPLICATION: 'replication',
 	REPLICATION_HOSTNAME: 'replication_hostname',
 	REPLICATION_URL: 'replication_url',
@@ -549,18 +595,21 @@ export const CONFIG_PARAMS = {
 	REPLICATION_BLOBCONCURRENCY: 'replication_blobConcurrency',
 	REPLICATION_MAXPAYLOAD: 'replication_maxPayload',
 	REPLICATION_RECORDCONCURRENCY: 'replication_recordConcurrency',
+	REPLICATION_PINGINTERVAL: 'replication_pingInterval',
+	REPLICATION_PINGTIMEOUT: 'replication_pingTimeout',
 	ROOTPATH: 'rootPath',
 	SERIALIZATION_BIGINT: 'serialization_bigInt',
 	STORAGE_WRITEASYNC: 'storage_writeAsync',
 	STORAGE_OVERLAPPINGSYNC: 'storage_overlappingSync',
 	STORAGE_CACHING: 'storage_caching',
-	STORAGE_RANDOMACCESSFIELDS: 'storage_randomAccessFields',
 	STORAGE_COMPRESSION: 'storage_compression',
 	STORAGE_NOREADAHEAD: 'storage_noReadAhead',
+	STORAGE_RANDOMACCESSFIELDS: 'storage_randomAccessFields',
 	STORAGE_PREFETCHWRITES: 'storage_prefetchWrites',
 	STORAGE_ENCRYPTION: 'storage_encryption',
 	STORAGE_MAXTRANSACTIONQUEUETIME: 'storage_maxTransactionQueueTime',
 	STORAGE_MAXTRANSACTIONOPENTIME: 'storage_maxTransactionOpenTime',
+	STORAGE_MAX_READ_TRANSACTION_OPEN_TIME: 'storage_maxReadTransactionOpenTime',
 	STORAGE_DEBUGLONGTRANSACTIONS: 'storage_debugLongTransactions',
 	STORAGE_PATH: 'storage_path',
 	STORAGE_BLOBPATHS: 'storage_blobPaths',
@@ -578,6 +627,7 @@ export const CONFIG_PARAMS = {
 	STORAGE_RECLAMATION_INTERVAL: 'storage_reclamation_interval',
 	STORAGE_RECLAMATION_EVICTIONFACTOR: 'storage_reclamation_evictionFactor',
 	STORAGE_ENGINE: 'storage_engine',
+	STORAGE_READONLY: 'storage_readOnly',
 	STORAGE_ROCKS_BLOCKCACHESIZE: 'storage_rocks_blockCacheSize',
 	STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE: 'storage_rocks_writeBufferManagerSize',
 	STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE: 'storage_rocks_writeBufferManagerCostToCache',
@@ -597,6 +647,7 @@ export const CONFIG_PARAMS = {
 	TLS_PRIVATEKEY: 'tls_privateKey',
 	TLS_CERTIFICATEAUTHORITY: 'tls_certificateAuthority',
 	TLS_CIPHERS: 'tls_ciphers',
+	TLS_UNIXDOMAINSOCKETS: 'tls_unixDomainSockets',
 	TLS: 'tls',
 	CLONED: 'cloned',
 	NODE_HOSTNAME: 'node_hostname',
@@ -803,6 +854,8 @@ export const ITC_EVENT_TYPES = {
 	START_JOB: 'start_job',
 	COMPONENT_STATUS_REQUEST: 'component_status_request',
 	COMPONENT_STATUS_RESPONSE: 'component_status_response',
+	RESOURCE_OPENAPI_REQUEST: 'resource_openapi_request',
+	RESOURCE_OPENAPI_RESPONSE: 'resource_openapi_response',
 } as const;
 
 /** Supported thread types */

package/utility/install/checkJWTTokensExist.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const env = require('../../utility/environment/environmentManager.js');
+const env = require('../../utility/environment/environmentManager.ts');
 env.initSync();
 const fs = require('fs-extra');
 const path = require('path');

package/utility/install/{installer.js → installer.ts}

@@ -1,32 +1,32 @@
 'use strict';
 
-const os = require('os');
-const inquirer = require('inquirer');
-const fs = require('fs-extra');
-const PropertiesReader = require('properties-reader');
-const chalk = require('chalk');
-const path = require('path');
+import * as os from 'os';
+import inquirer from 'inquirer';
+import * as fs from 'fs-extra';
+import PropertiesReader from 'properties-reader';
+import chalk from 'chalk';
+import * as path from 'path';
 let ora; // Will be loaded dynamically as it's an ES module
-const YAML = require('yaml');
-
-const hdbLogger = require('../logging/harper_logger.js');
-const envManager = require('../environment/environmentManager.js');
-const hdbUtils = require('../common_utils.js');
-const assignCMDENVVariables = require('../../utility/assignCmdEnvVariables.js');
-const hdbInfoController = require('../../dataLayer/hdbInfoController.js');
-const { packageJson } = require('../packageUtils.js');
-const hdbTerms = require('../hdbTerms.ts');
+import * as YAML from 'yaml';
+
+import * as hdbLogger from '../logging/harper_logger.ts';
+import * as envManager from '../environment/environmentManager.ts';
+import * as hdbUtils from '../common_utils.ts';
+
+import * as hdbInfoController from '../../dataLayer/hdbInfoController.ts';
+import { packageJson } from '../packageUtils.js';
+import * as hdbTerms from '../hdbTerms.ts';
 const { CONFIG_PARAMS } = hdbTerms;
-const installValidator = require('../../validation/installValidator.js');
-const mountHdb = require('../mount_hdb.js');
-const configUtils = require('../../config/configUtils.js');
-const userOps = require('../../security/user.ts');
-const roleOps = require('../../security/role.js');
-const checkJwtTokens = require('./checkJWTTokensExist.js');
-const globalSchema = require('../globalSchema.js');
-const promisify = require('util').promisify;
+import installValidator from '../../validation/installValidator.ts';
+import mountHdb from '../mount_hdb.ts';
+import * as configUtils from '../../config/configUtils.js';
+import * as userOps from '../../security/user.ts';
+import * as roleOps from '../../security/role.ts';
+import checkJwtTokens from './checkJWTTokensExist.js';
+import * as globalSchema from '../globalSchema.ts';
+import { promisify } from 'util';
 const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal);
-const keys = require('../../security/keys.js');
+import * as keys from '../../security/keys.ts';
 
 // Removes the color formatting that was being applied to the prompt answer.
 const PROMPT_ANSWER_TRANSFORMER = (answer) => answer;
@@ -68,6 +68,7 @@ const INSTALL_PROMPTS = {
 	DEFAULTS_MODE: 'Default Config - dev (easy access/debugging) or prod (security/performance): (dev/prod)',
 };
 
+import assignCMDENVVariables from '../assignCmdEnvVariables.ts';
 const cfgEnv = assignCMDENVVariables([hdbTerms.INSTALL_PROMPTS.HDB_CONFIG]);
 let hdbRoot = undefined;
 let conditionalRollback = false;
@@ -77,7 +78,7 @@ let ignoreExisting = false;
  * This module orchestrates the installation of Harper.
  */
 
-module.exports = { install, updateConfigEnv, setIgnoreExisting };
+export { install, updateConfigEnv, setIgnoreExisting };
 install.createSuperUser = createSuperUser;
 
 /**
@@ -85,7 +86,7 @@ install.createSuperUser = createSuperUser;
  * @returns {Promise<void>}
  */
 async function install() {
-	console.log(HDB_PROMPT_MSG(LINE_BREAK + INSTALL_START_MSG + LINE_BREAK));
+	console.error(HDB_PROMPT_MSG(LINE_BREAK + INSTALL_START_MSG + LINE_BREAK));
 	hdbLogger.notify(INSTALL_START_MSG);
 
 	let configFromFile;
@@ -146,10 +147,8 @@ async function install() {
 	if (
 		!ignoreExisting &&
 		!cfgEnv[hdbTerms.INSTALL_PROMPTS.HDB_CONFIG] &&
-		(await fs.pathExists(
-			path.join(hdbRoot, hdbTerms.HARPER_CONFIG_FILE) ||
-				(await fs.pathExists(path.join(hdbRoot, hdbTerms.HDB_CONFIG_FILE)))
-		))
+		((await fs.pathExists(path.join(hdbRoot, hdbTerms.HARPER_CONFIG_FILE))) ||
+			(await fs.pathExists(path.join(hdbRoot, hdbTerms.HDB_CONFIG_FILE))))
 	) {
 		console.error(HDB_EXISTS_MSG);
 		process.exit();
@@ -198,15 +197,15 @@ async function install() {
 
 	spinner.stop();
 
-	console.log(HDB_PROMPT_MSG(LINE_BREAK + INSTALL_COMPLETE_MSG + LINE_BREAK));
+	console.error(HDB_PROMPT_MSG(LINE_BREAK + INSTALL_COMPLETE_MSG + LINE_BREAK));
 	hdbLogger.notify(INSTALL_COMPLETE_MSG);
 }
 
 function getConfigFromFile() {
 	let doc = YAML.parseDocument(fs.readFileSync(cfgEnv[hdbTerms.INSTALL_PROMPTS.HDB_CONFIG], 'utf8'), {
 		simpleKeys: true,
-	});
-	const flatCfg = configUtils.flattenConfig(doc.toJSON());
+	} as any);
+	const flatCfg: any = configUtils.flattenConfig(doc.toJSON()) ?? {};
 
 	// This ensures that if config file has rootpath, rootpath install prompt uses this value
 	if (flatCfg[hdbTerms.CONFIG_PARAMS.ROOTPATH.toLowerCase()])
@@ -321,10 +320,10 @@ async function installPrompts(promptOverride) {
 function displayCmdEnvVar(value, msg) {
 	if (value !== undefined) {
 		if (msg.includes('password')) {
-			console.log(`${HDB_PROMPT_MSG(msg)} ${chalk.gray('[hidden]')}`);
+			console.error(`${HDB_PROMPT_MSG(msg)} ${chalk.gray('[hidden]')}`);
 			hdbLogger.trace(`${HDB_PROMPT_MSG(msg)} [hidden]`);
 		} else {
-			console.log(`${HDB_PROMPT_MSG(msg)} ${value}`);
+			console.error(`${HDB_PROMPT_MSG(msg)} ${value}`);
 			hdbLogger.trace(`${HDB_PROMPT_MSG(msg)} ${value}`);
 		}
 		return false;
@@ -411,11 +410,11 @@ async function checkForExistingInstall() {
 		const upgradeObj = await hdbInfoController.getVersionUpdateInfo();
 		if (upgradeObj) {
 			const upgradeToVerMsg = `Please use \`harperdb upgrade\` to update to ${packageJson.version}. Exiting install...`;
-			console.log(LINE_BREAK + chalk.magenta.bold(UPGRADE_MSG));
-			console.log(chalk.magenta.bold(upgradeToVerMsg));
+			console.error(LINE_BREAK + chalk.magenta.bold(UPGRADE_MSG));
+			console.error(chalk.magenta.bold(upgradeToVerMsg));
 			hdbLogger.error(upgradeToVerMsg);
 		} else {
-			console.log(LINE_BREAK + chalk.magenta.bold(HDB_EXISTS_MSG));
+			console.error(LINE_BREAK + chalk.magenta.bold(HDB_EXISTS_MSG));
 			hdbLogger.error(HDB_EXISTS_MSG);
 		}
 		process.exit(0);
@@ -442,9 +441,28 @@ async function createBootPropertiesFile() {
 		const homeDirPath = path.join(homeDir, hdbTerms.HDB_HOME_DIR_NAME);
 		const homeDirKeysDirPath = path.join(homeDirPath, hdbTerms.LICENSE_KEY_DIR_NAME);
 		const propsFilePath = path.join(homeDirPath, hdbTerms.BOOT_PROPS_FILE_NAME);
-		// if the properties file already exists, and we have an explicit ROOTPATH, we don't overwrite the existing
-		// properties file
-		if (!fs.existsSync(propsFilePath) || !hdbUtils.getEnvCliRootPath()) {
+		// Write boot props when: file doesn't exist, OR no explicit ROOTPATH was given (caller
+		// accepts whatever path this install produces), OR the existing file points to a
+		// settings_path that no longer exists (stale from a deleted previous install dir).
+		let shouldWriteBootProps = !fs.existsSync(propsFilePath) || !hdbUtils.getEnvCliRootPath();
+		if (!shouldWriteBootProps) {
+			try {
+				const content = fs.readFileSync(propsFilePath, 'utf8');
+				// Anchor to line start (with /m) to avoid matching commented-out lines.
+				const match = content.match(/^[ \t]*settings_path\s*=\s*(.+)/m);
+				if (match) {
+					const settingsPath = match[1].trim().replace(/^["']|["']$/g, '');
+					if (!fs.existsSync(settingsPath)) {
+						shouldWriteBootProps = true;
+					}
+				} else {
+					shouldWriteBootProps = true;
+				}
+			} catch {
+				shouldWriteBootProps = true;
+			}
+		}
+		if (shouldWriteBootProps) {
 			try {
 				fs.mkdirpSync(homeDirPath, { mode: hdbTerms.HDB_FILE_PERMISSIONS });
 				fs.mkdirpSync(homeDirKeysDirPath, { mode: hdbTerms.HDB_FILE_PERMISSIONS });
@@ -509,6 +527,12 @@ async function createConfigFile(installParams) {
 			if (args[cfg.toLowerCase()] === undefined) args[cfg] = DEV_MODE_CONFIG[cfg];
 		}
 	} else {
+		const defaultsMode = installParams[hdbTerms.INSTALL_PROMPTS.DEFAULTS_MODE];
+		if (defaultsMode !== undefined && defaultsMode !== 'prod') {
+			const warn = `DEFAULTS_MODE value "${defaultsMode}" is not recognized (expected "dev" or "prod"). Using prod defaults.`;
+			console.error(chalk.yellow.bold(`Warning: ${warn}`));
+			hdbLogger.warn(warn);
+		}
 		if (args[CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT.toLowerCase()])
 			args[CONFIG_PARAMS.OPERATIONSAPI_NETWORK_SECUREPORT] = null;
 		if (args[CONFIG_PARAMS.HTTP_PORT.toLowerCase()]) args[CONFIG_PARAMS.HTTP_SECUREPORT] = null;
@@ -547,7 +571,7 @@ function rollbackInstall(errMsg) {
 		if (conditionalRollback) {
 			const dir = fs.readdirSync(hdbRoot, { withFileTypes: true });
 			dir.forEach((d) => {
-				const fullPath = path.join(d.path, d.name);
+				const fullPath = path.join((d as any).parentPath ?? (d as any).path, d.name);
 				if (fullPath !== cfgEnv[hdbTerms.INSTALL_PROMPTS.HDB_CONFIG]) {
 					fs.removeSync(fullPath);
 				}

package/utility/installation.ts

@@ -1,7 +1,7 @@
 import type { Logger } from '../components/Logger.ts';
-import * as terms from './hdbTerms.js';
+import * as terms from './hdbTerms.ts';
 import fs from 'node:fs';
-import { noBootFile, getPropsFilePath } from './common_utils.js';
+import { noBootFile, getPropsFilePath } from './common_utils.ts';
 
 interface Env {
 	get(key: string): string;

package/utility/lmdb/{DBIDefinition.js → DBIDefinition.ts}

@@ -6,6 +6,9 @@
  * intKey defines if the key entries are integers or not
  */
 class DBIDefinition {
+	dup_sort: boolean;
+	isPrimaryKey: boolean;
+	useVersions: boolean;
 	/**
 	 * @param {Boolean} dupSort - allow duplicate keys, or not
 	 * @param {Boolean} isPrimaryKey - defines if this is the primary key
@@ -17,4 +20,4 @@ class DBIDefinition {
 	}
 }
 
-module.exports = DBIDefinition;
+export default DBIDefinition;

package/utility/lmdb/{DeleteRecordsResponseObject.js → DeleteRecordsResponseObject.ts}

@@ -8,6 +8,7 @@
  * @param {Array.<Object>} originalRecords
  */
 class DeleteRecordsResponseObject {
+	[key: string]: any;
 	/**
 	 * @param {Array.<string|number>} deleted
 	 * @param {Array.<string|number>} skipped
@@ -22,4 +23,4 @@ class DeleteRecordsResponseObject {
 	}
 }
 
-module.exports = DeleteRecordsResponseObject;
+export default DeleteRecordsResponseObject;

package/utility/lmdb/{InsertRecordsResponseObject.js → InsertRecordsResponseObject.ts}

@@ -7,6 +7,7 @@
  * @param {number} txnTime
  */
 class InsertRecordsResponseObject {
+	[key: string]: any;
 	/**
 	 * @param {Array.<string|number>} written_hashes
 	 * @param {Array.<string|number>} skipped_hashes
@@ -19,4 +20,4 @@ class InsertRecordsResponseObject {
 	}
 }
 
-module.exports = InsertRecordsResponseObject;
+export default InsertRecordsResponseObject;

package/utility/lmdb/OpenDBIObject.ts

@@ -0,0 +1,57 @@
+'use strict';
+import * as envMngr from '../environment/environmentManager.ts';
+import * as terms from '../../utility/hdbTerms.ts';
+import { RecordEncoder } from '../../resources/RecordEncoder.ts';
+envMngr.initSync();
+
+const LMDB_CACHING = envMngr.get(terms.CONFIG_PARAMS.STORAGE_CACHING) !== false;
+
+/**
+ * Defines how a DBI will be created/opened
+ */
+export class OpenDBIObject {
+	[key: string]: any;
+	dupSort: boolean;
+	encoding: 'string' | 'json' | 'binary' | 'msgpack' | 'ordered-binary';
+	useVersions: boolean;
+	sharedStructuresKey: symbol;
+	compression: any;
+	cache: any;
+	freezeData: boolean;
+	encoder: any;
+	randomAccessStructure: boolean;
+	/**
+	 * @param {Boolean} dupSort - if the dbi allows duplicate keys
+	 * @param {Boolean} [isPrimary] - if the dbi is the primary dbi
+	 */
+	constructor(dupSort, isPrimary = false) {
+		/** @type {boolean} */
+		this.dupSort = dupSort === true;
+		/** @type {"string" | "json" | "binary" | "msgpack" | "ordered-binary"} */
+		this.encoding = dupSort ? 'ordered-binary' : 'msgpack';
+		/** @type {boolean} */
+		this.useVersions = isPrimary;
+		/** @type {Symbol} */
+		this.sharedStructuresKey = Symbol.for('structures');
+		/** @type {any} */
+		this.compression = undefined;
+		this.encoder = { Encoder: RecordEncoder };
+		// Only enable struct (random-access) encoding on primary DBIs. Struct headers occupy
+		// the 0x20-0x3f range, which readers without struct support (msgpackr v1 / Harper v4)
+		// decode as positive fixints. v4 likewise gated struct mode behind is_primary, so
+		// non-primary stores (e.g. the __dbis__ metadata DBI) stay in records mode and remain
+		// decodable after a downgrade. RecordEncoder still reads struct data so existing v5
+		// struct entries decode.
+		// As of 5.1 the primary default is itself opt-in via storage.randomAccessFields (default
+		// off; overridable per-table via databases.ts before the store opens). Typed structures key
+		// on per-field value WIDTH, so wide/variably-typed schemas can mint an unbounded dictionary
+		// (OOM) and diverge across replicas. Read the config here at construction — not at module
+		// import — so an env/CLI override applied during startup is honored (DBIs open after config
+		// is finalized).
+		this.randomAccessStructure = isPrimary && envMngr.get(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS) === true;
+		if (isPrimary) {
+			this.cache = LMDB_CACHING && { validated: true };
+			this.freezeData = true;
+		}
+	}
+}

package/utility/lmdb/{OpenEnvironmentObject.js → OpenEnvironmentObject.ts}

@@ -5,11 +5,27 @@ const MAP_SIZE = 1024 * 1024 * 1024;
 //allow up to 1,000 named data bases in an environment
 const MAX_DBS = 10000;
 const MAX_READERS = 2048;
-const envMngr = require('../environment/environmentManager.js');
-const terms = require('../../utility/hdbTerms.ts');
+import * as envMngr from '../environment/environmentManager.ts';
+import * as terms from '../../utility/hdbTerms.ts';
 envMngr.initSync();
 
-class OpenEnvironmentObject {
+export default class OpenEnvironmentObject {
+	[key: string]: any;
+	static MAX_DBS = MAX_DBS;
+	path: string;
+	mapSize: number;
+	maxDbs: number;
+	maxReaders: number;
+	sharedStructuresKey: symbol;
+	readOnly: boolean;
+	trackMetrics: boolean;
+	eventTurnBatching: boolean;
+	noSync: boolean;
+	overlappingSync: any;
+	maxFreeSpaceToLoad: any;
+	maxFreeSpaceToRetain: any;
+	pageSize: any;
+	noReadAhead: any;
 	constructor(path, readOnly = false) {
 		this.path = path;
 		this.mapSize = MAP_SIZE;
@@ -36,6 +52,3 @@ class OpenEnvironmentObject {
 		this.noReadAhead = envMngr.get(terms.CONFIG_PARAMS.STORAGE_NOREADAHEAD);
 	}
 }
-
-module.exports = OpenEnvironmentObject;
-OpenEnvironmentObject.MAX_DBS = MAX_DBS;

package/utility/lmdb/{UpdateRecordsResponseObject.js → UpdateRecordsResponseObject.ts}

@@ -8,6 +8,7 @@
  * @param {Array.<Object>} originalRecords
  */
 class UpdateRecordsResponseObject {
+	[key: string]: any;
 	/**
 	 * @param {Array.<string|number>} written_hashes
 	 * @param {Array.<string|number>} skipped_hashes
@@ -22,4 +23,4 @@ class UpdateRecordsResponseObject {
 	}
 }
 
-module.exports = UpdateRecordsResponseObject;
+export default UpdateRecordsResponseObject;

package/utility/lmdb/{UpsertRecordsResponseObject.js → UpsertRecordsResponseObject.ts}

@@ -7,6 +7,7 @@
  * @param {Array.<Object>} originalRecords
  */
 class UpsertRecordsResponseObject {
+	[key: string]: any;
 	/**
 	 * @param {Array.<string|number>} written_hashes
 	 * @param {number} txnTime
@@ -19,4 +20,4 @@ class UpsertRecordsResponseObject {
 	}
 }
 
-module.exports = UpsertRecordsResponseObject;
+export default UpsertRecordsResponseObject;

package/utility/lmdb/{cleanLMDBMap.js → cleanLMDBMap.ts}

@@ -1,16 +1,16 @@
 'use strict';
 
-const environmentUtility = require('./environmentUtility.js');
-const harperLogger = require('../logging/harper_logger.js');
-const LMDB_ERRORS = require('../errors/commonErrors.js').LMDB_ERRORS_ENUM;
+import * as environmentUtility from './environmentUtility.ts';
+import harperLogger from '../logging/harper_logger.ts';
+import { LMDB_ERRORS_ENUM as LMDB_ERRORS } from '../errors/commonErrors.ts';
 
-module.exports = cleanLMDBMap;
+export default cleanLMDBMap;
 
 /**
  * this function strips away the cached environments from global when a schema item is removed
  * @param msg
  */
-async function cleanLMDBMap(msg) {
+async function cleanLMDBMap(msg: any) {
 	try {
 		if (global.lmdb_map !== undefined && msg.operation !== undefined) {
 			let keys = Object.keys(global.lmdb_map);

package/utility/lmdb/{commonUtility.js → commonUtility.ts}

@@ -1,18 +1,17 @@
 'use strict';
 
-const LMDB_ERRORS = require('../errors/commonErrors.js').LMDB_ERRORS_ENUM;
+import { LMDB_ERRORS_ENUM as LMDB_ERRORS } from '../errors/commonErrors.ts';
 // eslint-disable-next-line no-unused-vars
-const lmdb = require('lmdb');
-const lmdbTerms = require('./terms.js');
+import * as lmdb from 'lmdb';
+import * as lmdbTerms from './terms.ts';
 
-const { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH } = lmdbTerms;
 const PRIMITIVES = ['number', 'string', 'symbol', 'boolean', 'bigint'];
 /**
  * validates the env argument
  * @param {lmdb.Transaction|lmdb.RootDatabase} env - environment object used thigh level to interact with all data in an
  * environment
  */
-function validateEnv(env) {
+export function validateEnv(this: any, env) {
 	env = env?.primaryStore || env;
 	if (!env) {
 		throw new Error(LMDB_ERRORS.ENV_REQUIRED);
@@ -24,7 +23,7 @@ function validateEnv(env) {
  * @param rawValue
  * @returns {Number|String|null}
  */
-function stringifyData(rawValue) {
+export function stringifyData(this: any, rawValue) {
 	if (rawValue === null || rawValue === undefined) {
 		return null;
 	}
@@ -46,7 +45,7 @@ function stringifyData(rawValue) {
  * @param {*} key - raw value which needs to be converted
  * @returns {*}
  */
-function convertKeyValueToWrite(key) {
+export function convertKeyValueToWrite(this: any, key) {
 	//if this is a primitive return the value
 	if (key instanceof Date) {
 		return key.valueOf();
@@ -57,7 +56,7 @@ function convertKeyValueToWrite(key) {
 /**
  * Return all the indexable values from an attribute, ready to be indexed
  */
-function getIndexedValues(value, indexNulls) {
+export function getIndexedValues(this: any, value: any, indexNulls?: any) {
 	if (value === null) {
 		return indexNulls ? [null] : undefined;
 	}
@@ -65,8 +64,8 @@ function getIndexedValues(value, indexNulls) {
 		return undefined;
 	}
 	if (PRIMITIVES.includes(typeof value)) {
-		if (value.length > MAX_SEARCH_KEY_LENGTH) {
-			return [value.slice(0, MAX_SEARCH_KEY_LENGTH) + OVERFLOW_MARKER];
+		if (value.length > lmdbTerms.MAX_SEARCH_KEY_LENGTH) {
+			return [value.slice(0, lmdbTerms.MAX_SEARCH_KEY_LENGTH) + lmdbTerms.OVERFLOW_MARKER];
 		}
 		return [value];
 	}
@@ -75,8 +74,8 @@ function getIndexedValues(value, indexNulls) {
 		for (let i = 0, l = value.length; i < l; i++) {
 			let element = value[i];
 			if (PRIMITIVES.includes(typeof element)) {
-				if (element.length > MAX_SEARCH_KEY_LENGTH) {
-					values.push(element.slice(0, MAX_SEARCH_KEY_LENGTH) + OVERFLOW_MARKER);
+				if (element.length > lmdbTerms.MAX_SEARCH_KEY_LENGTH) {
+					values.push(element.slice(0, lmdbTerms.MAX_SEARCH_KEY_LENGTH) + lmdbTerms.OVERFLOW_MARKER);
 				} else {
 					values.push(element);
 				}
@@ -95,7 +94,7 @@ function getIndexedValues(value, indexNulls) {
 
 let lastTime = 0; // reported time used to ensure monotonic time.
 let startTime = 0; // the start time of the (current time relative to performance time counter)
-function adjustStartTime() {
+function adjustStartTime(this: any) {
 	// calculate the start time
 	// TODO: We may actually want to implement a gradual time shift if the clock time really changes substantially
 	// and for sub-millisecond updates, may want to average them so we can progressively narrow in on true time
@@ -109,7 +108,7 @@ setInterval(adjustStartTime, TIME_ADJUSTMENT_INTERVAL).unref();
  * A monotonic timestamp that is guaranteed to be higher than the last call to this function.
  * Will use decimal microseconds as necessary to differentiate from previous calls without too much drift.
  */
-function getNextMonotonicTime() {
+export function getNextMonotonicTime(this: any) {
 	let now = performance.now() + startTime;
 	if (now > lastTime) {
 		// current time is higher than last time, can safely return it
@@ -121,10 +120,3 @@ function getNextMonotonicTime() {
 	lastTime += 0.000488;
 	return lastTime;
 }
-module.exports = {
-	validateEnv,
-	stringifyData,
-	convertKeyValueToWrite,
-	getNextMonotonicTime,
-	getIndexedValues,
-};