@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/utility/environment/{environmentManager.js → environmentManager.ts}

@@ -1,14 +1,14 @@
 'use strict';
 
-const fs = require('fs-extra');
-const path = require('path');
-const os = require('os');
-const PropertiesReader = require('properties-reader');
-const log = require('../logging/harper_logger.js');
-const commonUtils = require('../common_utils.js');
-const hdbTerms = require('../hdbTerms.ts');
-const configUtils = require('../../config/configUtils.js');
-const { mkdirSync } = require('node:fs');
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import * as os from 'os';
+import PropertiesReader from 'properties-reader';
+import log from '../logging/harper_logger.ts';
+import * as commonUtils from '../common_utils.ts';
+import * as hdbTerms from '../hdbTerms.ts';
+import * as configUtils from '../../config/configUtils.js';
+import { mkdirSync } from 'node:fs';
 
 const INIT_ERR = 'Error initializing environment manager';
 const BOOT_PROPS_FILE_PATH = 'BOOT_PROPS_FILE_PATH';
@@ -21,26 +21,15 @@ const installPropsToSave = {
 	[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]: true,
 	BOOT_PROPS_FILE_PATH: true,
 };
-let installProps = {};
-Object.assign(
-	exports,
-	(module.exports = {
-		BOOT_PROPS_FILE_PATH,
-		getHdbBasePath,
-		setHdbBasePath,
-		get,
-		initSync,
-		setProperty,
-		initTestEnvironment,
-	})
-);
+let installProps: any = {};
+export { BOOT_PROPS_FILE_PATH };
 
 /**
  * The base path of the HDB install is often referenced, but is referenced as a const variable at the top of many
  * modules.  This is a problem during install, as the path may not yet be defined.  We offer a function to get the
  * currently known base path here to help with this case.
  */
-function getHdbBasePath() {
+export function getHdbBasePath() {
 	return installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY];
 }
 
@@ -49,7 +38,7 @@ function getHdbBasePath() {
  * This is mainly used by install during a stage where the config file doesn't exist.
  * @param hdbPath
  */
-function setHdbBasePath(hdbPath) {
+export function setHdbBasePath(hdbPath: string) {
 	installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY] = hdbPath;
 }
 
@@ -58,7 +47,7 @@ function setHdbBasePath(hdbPath) {
  * @param propName
  * @returns {*}
  */
-function get(propName) {
+export function get(propName: string): any {
 	const value = configUtils.getConfigValue(propName);
 	if (value === undefined) {
 		return installProps[propName];
@@ -77,7 +66,7 @@ function get(propName) {
  * @param propName
  * @param value
  */
-function setProperty(propName, value) {
+export function setProperty(propName: string, value: any) {
 	if (installPropsToSave[propName]) {
 		installProps[propName] = value;
 	}
@@ -115,7 +104,7 @@ function doesPropFileExist() {
  * Synchronously initializes our config environment.
  * @param force
  */
-function initSync(force = false) {
+export function initSync(force: boolean = false) {
 	try {
 		if (propFileExists || doesPropFileExist() || commonUtils.noBootFile() || force) {
 			configUtils.initConfig(force);
@@ -129,7 +118,11 @@ function initSync(force = false) {
 		log.error(INIT_ERR);
 		log.error(err);
 		console.error(err);
-		process.exit(1);
+		// Use _realExit so this fatal startup error still terminates the worker
+		// even with the worker process guard installed. Inline fallback (rather
+		// than importing the helper) avoids a utility -> server layer
+		// dependency; safe because both alternatives are real exit primitives.
+		(process._realExit ?? process.exit)(1);
 	}
 }
 
@@ -138,7 +131,7 @@ function initSync(force = false) {
  * Most of this is legacy code from before the yaml config refactor.
  * @param testConfigObj
  */
-function initTestEnvironment(testConfigObj = {}) {
+export function initTestEnvironment(testConfigObj: any = {}) {
 	try {
 		const {
 			keep_alive_timeout,
@@ -152,16 +145,24 @@ function initTestEnvironment(testConfigObj = {}) {
 		// __dirname is dist/utility/environment when running tests, so go up 3 levels to reach project root
 		const propsPath = path.join(__dirname, '../../../', 'unitTests');
 		installProps[BOOT_PROPS_FILE_PATH] = path.join(propsPath, 'hdb_boot_properties.file');
+		const TEST_HDB_PATH = path.join(propsPath, 'envDir', process.pid.toString());
 		try {
-			mkdirSync(path.join(propsPath, 'envDir'));
+			mkdirSync(TEST_HDB_PATH, { recursive: true });
 		} catch {}
 		setProperty(hdbTerms.HDB_SETTINGS_NAMES.SETTINGS_PATH_KEY, path.join(propsPath, 'settings.test'));
 		setProperty(hdbTerms.HDB_SETTINGS_NAMES.INSTALL_USER, os.userInfo() ? os.userInfo().username : undefined);
 		setProperty(hdbTerms.HDB_SETTINGS_NAMES.LOG_LEVEL_KEY, `debug`);
-		setProperty(hdbTerms.HDB_SETTINGS_NAMES.LOG_PATH_KEY, path.join(propsPath, 'envDir', 'log'));
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.LOG_PATH_KEY, path.join(TEST_HDB_PATH, 'log'));
 		setProperty(hdbTerms.HDB_SETTINGS_NAMES.LOG_DAILY_ROTATE_KEY, false);
-		setProperty(hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY, path.join(propsPath, 'envDir'));
-		setProperty(hdbTerms.CONFIG_PARAMS.STORAGE_PATH, path.join(propsPath, 'envDir'));
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY, TEST_HDB_PATH);
+		setProperty(hdbTerms.CONFIG_PARAMS.STORAGE_PATH, TEST_HDB_PATH);
+		const systemPath =
+			typeof (globalThis as any).databases !== 'undefined' &&
+			(globalThis as any).databases.system?.hdb_user?.primaryStore?.path;
+		if (systemPath) {
+			setProperty(hdbTerms.CONFIG_PARAMS.DATABASES, { system: { path: path.dirname(systemPath) } });
+		}
+
 		if (https_enabled) {
 			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_SECUREPORT, get(hdbTerms.CONFIG_PARAMS.HTTP_PORT));
 			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_PORT, null);

package/utility/environment/systemInformation.ts

@@ -1,11 +1,12 @@
 import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import si from 'systeminformation';
-import logger from '../logging/harper_logger.js';
+import logger from '../logging/harper_logger.ts';
 import * as hdbTerms from '../hdbTerms.ts';
+import { getQuotaStatus } from '../../server/storageReclamation.ts';
 import { lmdbGetTableSize } from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts';
 import { getThreadInfo } from '../../server/threads/manageThreads.js';
-import env from './environmentManager.js';
+import * as env from './environmentManager.ts';
 import { getDatabases, type Table } from '../../resources/databases.ts';
 import { TableSizeObject } from '../../dataLayer/harperBridge/TableSizeObject.ts';
 import { RocksDatabase, StatsHistogramData } from '@harperfast/rocksdb-js';
@@ -264,6 +265,10 @@ type DiskInfo = {
 	io?: Pick<si.Systeminformation.DisksIoData, 'rIO' | 'wIO' | 'tIO'>;
 	read_write?: Pick<si.Systeminformation.FsStatsData, 'rx' | 'tx' | 'wx'>;
 	size?: si.Systeminformation.FsSizeData[];
+	free_space_basis?: 'quota' | 'filesystem';
+	quota_size_bytes?: number;
+	quota_used_bytes?: number;
+	quota_status_age_seconds?: number;
 };
 
 /**
@@ -272,6 +277,15 @@ type DiskInfo = {
  */
 export async function getDiskInfo(): Promise<DiskInfo> {
 	const disk: DiskInfo = {};
+	const quotaStatus = await getQuotaStatus();
+	if (quotaStatus?.quotaBytes) {
+		disk.free_space_basis = 'quota';
+		disk.quota_size_bytes = quotaStatus.quotaBytes;
+		disk.quota_used_bytes = quotaStatus.usedBytes;
+		disk.quota_status_age_seconds = Math.floor((Date.now() - quotaStatus.updatedAt) / 1000);
+	} else {
+		disk.free_space_basis = 'filesystem';
+	}
 	try {
 		if (!env.get(hdbTerms.CONFIG_PARAMS.OPERATIONSAPI_SYSINFO_DISK)) return disk;
 
@@ -613,8 +627,8 @@ function getLMDBStats(table: Table, dbStats: DBStats): void {
 		const { root: _root, ...stats } = table.primaryStore.rootStore.getStats();
 		Object.assign(dbStats, stats);
 		dbStats.readers = table.primaryStore.rootStore
-			.readerList()
-			.split(/\n\s+/)
+			.readerList?.()
+			?.split(/\n\s+/)
 			.slice(1)
 			.map((line) => {
 				const [pid, thread, txnid] = line.trim().split(' ');

package/utility/errors/{commonErrors.js → commonErrors.ts}

@@ -1,10 +1,10 @@
 'use strict';
 
-const hdb_terms = require('../hdbTerms.ts');
-const lmdb_terms = require('../lmdb/terms.js');
+import * as hdb_terms from '../hdbTerms.ts';
+import * as lmdb_terms from '../lmdb/terms.ts';
 
 // A subset of HTTP error codes that we may use in code.
-const HTTP_STATUS_CODES = {
+export const HTTP_STATUS_CODES = {
 	CONTINUE: 100,
 	OK: 200,
 	CREATED: 201,
@@ -15,6 +15,7 @@ const HTTP_STATUS_CODES = {
 	METHOD_NOT_ALLOWED: 405,
 	REQUEST_TIMEOUT: 408,
 	CONFLICT: 409,
+	CONTENT_TOO_LARGE: 413,
 	TOO_MANY_REQUESTS: 429,
 	INTERNAL_SERVER_ERROR: 500,
 	NOT_IMPLEMENTED: 501,
@@ -30,7 +31,7 @@ const HTTP_STATUS_CODES = {
 // one error message to send to the API (with this wrapper) and log without having to define log message separately
 const CHECK_LOGS_WRAPPER = (err) => `${err} Check logs and try again.`;
 
-const DEFAULT_ERROR_MSGS = {
+export const DEFAULT_ERROR_MSGS = {
 	500: CHECK_LOGS_WRAPPER('There was an error processing your request.'),
 	400: 'Invalid request',
 };
@@ -216,6 +217,8 @@ const ITC_ERRORS = {
 };
 
 const CUSTOM_FUNCTIONS_ERROR_MSGS = {
+	COMPONENT_FILE_TOO_LARGE: (size: number, limit: number) =>
+		`Component file size (${size} bytes) exceeds the configured limit of ${limit} bytes`,
 	FUNCTION_STATUS: 'Error getting custom function status, check the log for more details',
 	GET_FUNCTIONS: 'Error getting custom functions, check the log for more details',
 	GET_FUNCTION: 'Error getting custom function, check the log for more details',
@@ -237,7 +240,7 @@ const CUSTOM_FUNCTIONS_ERROR_MSGS = {
 };
 
 //into a single export while still allowing us to group them here in a more readable/searchable way
-const HDB_ERROR_MSGS = {
+export const HDB_ERROR_MSGS = {
 	...AUTHENTICATION_ERROR_MSGS,
 	...BULK_LOAD_ERROR_MSGS,
 	...COMMON_ERROR_MSGS,
@@ -254,12 +257,9 @@ const HDB_ERROR_MSGS = {
 };
 
 // All error messages should be added to the HDB_ERROR_MSGS ENUM for export - this helps to organize all error messages
-module.exports = {
+export {
 	CHECK_LOGS_WRAPPER,
-	HDB_ERROR_MSGS,
-	DEFAULT_ERROR_MSGS,
 	DEFAULT_ERROR_RESP,
-	HTTP_STATUS_CODES,
 	LMDB_ERRORS_ENUM,
 	AUTHENTICATION_ERROR_MSGS,
 	VALIDATION_ERROR_MSGS,

package/utility/errors/{hdbError.js → hdbError.ts}

@@ -1,13 +1,18 @@
 'use strict';
-const hdbErrors = require('./commonErrors.js');
-const hdbTerms = require('../hdbTerms.ts');
+import logger from '../logging/harper_logger.ts';
+import * as hdbErrors from './commonErrors.ts';
+import * as hdbTerms from '../hdbTerms.ts';
 
 /**
  * Custom error class used for better error and log handling.  Caught errors that evaluate to an instanceof HdbError can
  * be handled differently - e.g. in most cases caught HdbError likely would not need to be logged since that should have
  * already been handled when the custom error was constructed.
  */
-class HdbError extends Error {
+export class HdbError extends Error {
+	statusCode: number;
+	http_resp_msg: string;
+	type: string;
+	logLevel: string;
 	/**
 	 * @param {Error} errOrig -  Error to be translated into HdbError. If manually throwing an error, pass `new Error()` to ensure stack trace is maintained
 	 * @param {String} [httpMsg] - optional -  response message that will be returned via the API
@@ -15,7 +20,7 @@ class HdbError extends Error {
 	 * @param {String} [logLevel] - optional -  log level that will be used for logging of this error
 	 * @param {String} [logMsg] - optional - log message that, if provided, will be logged at the `logLevel` above
 	 */
-	constructor(errOrig, httpMsg, httpCode, logLevel, logMsg) {
+	constructor(errOrig: any, httpMsg?: any, httpCode?: number, logLevel?: string, logMsg?: string) {
 		super();
 
 		//This line ensures the original stack trace is captured and does not include the 'handle' or 'constructor' methods
@@ -37,25 +42,26 @@ class HdbError extends Error {
 		}
 
 		if (logMsg) {
-			const logger = require('../logging/harper_logger.js');
 			logger[logLevel](logMsg);
 		}
 	}
 }
-class ClientError extends Error {
-	constructor(message, statusCode) {
+export class ClientError extends Error {
+	statusCode: number;
+	constructor(message: string | Error, statusCode?: number) {
 		if (message instanceof Error) {
-			message.statusCode = statusCode || 400;
-			return message;
+			(message as any).statusCode = statusCode || 400;
+			return message as any;
 		}
-		super(message);
+		super(message as any);
 		this.statusCode = statusCode || 400;
 	}
 }
 
-class ServerError extends Error {
-	constructor(message, statusCode) {
-		super(message);
+export class ServerError extends Error {
+	statusCode: number;
+	constructor(message: string | Error, statusCode?: number) {
+		super(message as any);
 		this.statusCode = statusCode || 500;
 	}
 }
@@ -73,13 +79,13 @@ class ServerError extends Error {
  * @param deleteStack
  * @returns {HdbError|*}
  */
-function handleHDBError(
-	e,
-	httpMsg,
-	httpCode,
-	logLevel = hdbTerms.LOG_LEVELS.ERROR,
-	logMsg = null,
-	deleteStack = false
+export function handleHDBError(
+	e: any,
+	httpMsg?: any,
+	httpCode?: number,
+	logLevel: string = (hdbTerms as any).LOG_LEVELS.ERROR,
+	logMsg: any = null,
+	deleteStack: boolean = false
 ) {
 	if (isHDBError(e)) {
 		return e;
@@ -96,29 +102,26 @@ function handleHDBError(
 }
 
 /**
- * Represents a general violation of validation/authorization. This should be used in situations where we are performing
- * expected verification, and we do not need to record a stack trace. This extends Error's prototype, but doesn't
- * use the native constructor to avoid stack trace capture which is several times faster.
+ * Represents a general violation of validation/authorization. This is used in situations where we are performing
+ * expected verification. Extends Error for TypeScript class compatibility.
  * @param {Object} user - user object that caused the access violation
  * @constructor
  */
-function Violation(message) {
-	this.message = message;
+export class Violation extends Error {
+	constructor(message: string) {
+		super(message);
+		this.name = this.constructor.name;
+	}
 }
-Violation.prototype = Object.create(Error.prototype);
-Violation.prototype.constructor = Violation;
-Violation.prototype.toString = function () {
-	return `${this.constructor.name}: ${this.message}`;
-};
 
 /**
- * Represents an access violation. This is used to return a 403 or 401 response to the client. Uses fast Violation class
- * to avoid stack trace capture.
+ * Represents an access violation. This is used to return a 403 or 401 response to the client.
  * @param {Object} user - user object that caused the access violation
  * @constructor
  */
-class AccessViolation extends Violation {
-	constructor(user) {
+export class AccessViolation extends Violation {
+	statusCode: number;
+	constructor(user?: any) {
 		if (user) {
 			super('Unauthorized access to resource');
 			this.statusCode = 403;
@@ -130,17 +133,8 @@ class AccessViolation extends Violation {
 	}
 }
 
-function isHDBError(e) {
+export function isHDBError(e: any) {
 	return e.__proto__.constructor.name === HdbError.name;
 }
 
-module.exports = {
-	isHDBError,
-	handleHDBError,
-	ClientError,
-	ServerError,
-	AccessViolation,
-	Violation,
-	//Including common hdbErrors here so that they can be brought into modules on the same line where the handler method is brought in
-	hdbErrors,
-};
+export { hdbErrors };

package/utility/expandEnvVar.ts

@@ -0,0 +1,110 @@
+/**
+ * Environment-variable expansion for config-loaded values.
+ *
+ * Harper's config parser does not expand `${VAR}` placeholders itself — the
+ * `harperdb-config.yaml` value lands in memory verbatim. Component code that
+ * needs to support env-var indirection (secrets, host overrides, etc.)
+ * applies these helpers to its own config entries.
+ *
+ * The pattern matches the convention established by the OAuth component
+ * (`@harperfast/oauth`'s `src/lib/config.ts`); kept identical so operator
+ * documentation and config snippets carry over verbatim.
+ *
+ * Matching rules:
+ * - Only **whole-string** `${VAR_NAME}` placeholders match. Partial expansion
+ *   (e.g. `'http://${HOST}:9926'`) is NOT supported by design — keeps the
+ *   substitution boundary unambiguous and avoids accidental injection into
+ *   structured values.
+ * - When the env var is undefined, the **original placeholder string is
+ *   returned unchanged**. Callers detect missing values via downstream
+ *   required-field checks (a `${VAR}` literal won't satisfy "non-empty string"
+ *   contracts the same way a real key would).
+ * - Non-string values pass through unchanged.
+ */
+
+/**
+ * Single shared predicate. Both `expandEnvVar` and `isUnresolvedEnvVarPlaceholder`
+ * must agree on what counts as a placeholder — otherwise an entry can sail past
+ * one check and trip on the other, producing a confusing downstream error.
+ *
+ * A value qualifies when it's a whole-string `${VAR_NAME}` with no internal
+ * whitespace (real shell env-var names disallow whitespace; rejecting it here
+ * means `'${MY KEY}'` is treated as a literal value rather than a botched
+ * lookup that always returns undefined).
+ */
+function looksLikePlaceholder(value: unknown): value is string {
+	if (typeof value !== 'string' || !value.startsWith('${') || !value.endsWith('}')) return false;
+	return !value.slice(2, -1).includes(' ');
+}
+
+/**
+ * Expand a single value: returns `process.env[VAR_NAME]` if `value` is exactly
+ * `${VAR_NAME}` and the env var is defined; returns `value` unchanged otherwise.
+ *
+ * @example
+ * expandEnvVar('${OPENAI_API_KEY}') // process.env.OPENAI_API_KEY or '${OPENAI_API_KEY}' if unset
+ * expandEnvVar('literal')           // 'literal'
+ * expandEnvVar(123)                 // 123
+ * expandEnvVar(undefined)           // undefined
+ */
+export function expandEnvVar<T>(value: T): T {
+	if (looksLikePlaceholder(value)) {
+		const envVar = value.slice(2, -1);
+		const envValue = process.env[envVar];
+		// Use the env value when defined (even if empty string — operators may
+		// legitimately want to set an empty default).
+		return (envValue !== undefined ? envValue : value) as T;
+	}
+	return value;
+}
+
+/**
+ * Recursively expand `${VAR_NAME}` placeholders on every string leaf of a
+ * value. Walks arrays and plain objects; bails on non-plain values.
+ *
+ * Used when a structured config block (e.g. `models.generative.<name>`) has
+ * multiple string fields that may carry env-var placeholders, so callers
+ * don't have to enumerate them.
+ *
+ * @example
+ * expandEnvVarsDeep({ backend: 'openai', apiKey: '${OPENAI_API_KEY}' })
+ * // → { backend: 'openai', apiKey: 'sk-...' }   (if env var set)
+ */
+export function expandEnvVarsDeep<T>(value: T): T {
+	if (typeof value === 'string') {
+		return expandEnvVar(value);
+	}
+	if (Array.isArray(value)) {
+		return value.map(expandEnvVarsDeep) as unknown as T;
+	}
+	if (value !== null && typeof value === 'object') {
+		// `Object.create(null)` avoids prototype-pollution gadgets via a
+		// `__proto__` / `constructor` / `prototype` key. The YAML parser this
+		// helper is initially used against produces plain objects without
+		// these keys, but the helper is documented as reusable; defending
+		// here makes "feed a JSON-from-HTTP payload through this" safe.
+		const expanded = Object.create(null) as Record<string, unknown>;
+		for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
+			expanded[key] = expandEnvVarsDeep(item);
+		}
+		return expanded as T;
+	}
+	return value;
+}
+
+/**
+ * Returns `true` when `value` is a literal `${VAR_NAME}` placeholder string —
+ * i.e., a value that `expandEnvVar` would have substituted if the env var
+ * were defined but didn't, leaving the original unchanged.
+ *
+ * Useful for distinguishing "operator wrote a literal placeholder by mistake"
+ * (loud, surface to logs) from "operator set the value directly" (proceed).
+ *
+ * @example
+ * isUnresolvedEnvVarPlaceholder('${OPENAI_API_KEY}') // true
+ * isUnresolvedEnvVarPlaceholder('sk-real-key')        // false
+ * isUnresolvedEnvVarPlaceholder('${FOO}-suffix')      // false (partial; not matched by expandEnvVar)
+ */
+export function isUnresolvedEnvVarPlaceholder(value: unknown): boolean {
+	return looksLikePlaceholder(value);
+}

package/utility/functions/geo.js

@@ -17,8 +17,8 @@ const turfBooleanEqual = require('@turf/boolean-equal');
 const turfBooleanDisjoint = require('@turf/boolean-disjoint');
 const turfHelpers = require('@turf/helpers');
 const hdbTerms = require('../hdbTerms.ts');
-const commonUtils = require('../common_utils.js');
-const hdbLog = require('../logging/harper_logger.js');
+const commonUtils = require('../common_utils.ts');
+const hdbLog = require('../logging/harper_logger.ts');
 
 module.exports = {
 	geoArea,

package/utility/functions/sql/alaSQLExtension.js

@@ -8,7 +8,7 @@
 const _ = require('lodash');
 const mathjs = require('mathjs');
 const jsonata = require('jsonata');
-const hdbUtils = require('../../common_utils.js');
+const hdbUtils = require('../../common_utils.ts');
 
 module.exports = {
 	/***

package/utility/globalSchema.ts

@@ -0,0 +1,30 @@
+import systemSchema from '../json/systemSchema.json';
+import { promisify } from 'util';
+import { getDatabases } from '../resources/databases.ts';
+
+export const setSchemaDataToGlobalAsync = promisify(setSchemaDataToGlobal);
+
+export function setSchemaDataToGlobal(callback?: any) {
+	(global as any).hdb_schema = getDatabases();
+	if (callback) callback();
+}
+
+export function getTableSchema(schemaName: string, tableName: string, callback: any) {
+	const database = getDatabases()[schemaName];
+	if (!database) {
+		return callback(`schema ${schemaName} does not exist`);
+	}
+	const table = database[tableName];
+	if (!table) {
+		return callback(`table ${schemaName}.${tableName} does not exist`);
+	}
+	return callback(null, {
+		schema: schemaName,
+		name: tableName,
+		hash_attribute: table.primaryKey,
+	});
+}
+
+export function getSystemSchema() {
+	return systemSchema;
+}