@harperfast/harper-pro 5.0.4 → 5.0.6

package/core/AGENTS.md

@@ -0,0 +1,107 @@
+# AGENTS.md
+
+This file provides guidance when working with code in this repository.
+
+---
+
+## What This Is
+
+Harper is a Node.js unified development platform that fuses a document database (RocksDB-backed), in-memory cache, application runtime, and messaging broker (WebSockets, MQTT, NATS) into a single in-process runtime. This directory is the open-source core (`harper` npm package, Apache-2.0), which is the base for the enterprise `harper-pro` wrapper above it.
+
+---
+
+## Commands
+
+```bash
+# Build
+npm run build              # TypeScript → dist/ via tsconfig.build.json
+npm run build:watch        # Incremental watch build
+
+# Lint / Format
+npm run lint               # oxlint (warnings = errors)
+npm run lint:fix           # Auto-fix
+npm run format:write       # Prettier
+
+# Test — run specific suites
+npm run test:unit                  # All unit tests (mocha)
+npm run test:unit:main             # Core unit tests (excludes apiTests, lmdb, resources)
+npm run test:unit:resources        # Resource layer tests
+npm run test:unit:server           # Server layer tests
+npm run test:unit:dataLayer        # Data layer tests
+npm run test:unit:components       # Component/plugin system tests
+npm run test:unit:security         # Security tests
+npm run test:unit:apitests         # API tests (stops running server first)
+npm run test:unit:lmdb             # LMDB storage engine tests
+npm run test:integration           # Full integration test suite
+```
+
+Run a single test file directly:
+
+```bash
+npx mocha unitTests/resources/mytest.js
+```
+
+TypeScript is stripped at runtime via `--conditions=typestrip` (Node.js native type stripping) — no compilation required for development. Use `npm run test:unit:typestrip` to run tests with this mode.
+
+---
+
+## Architecture
+
+### Layers (top to bottom)
+
+**Components** (`components/`)  
+The plugin/application loader. Applications export a `handleApplication(scope)` function. `Scope` is the primary object passed to apps; it exposes:
+
+- `scope.options` — `OptionsWatcher` for live-reloaded YAML config
+- `scope.resources` — access to database tables and registered resources
+- `scope.server` — the HTTP server handle
+
+Files within a component are discovered via micromatch glob patterns and automatically mapped to URL paths.
+
+**Server** (`server/`)  
+Two HTTP stacks coexist:
+
+- **Native layer** (`server/http.ts`) — direct socket handling for HTTP/1.1, HTTPS, HTTP/2, and WebSockets in one path; highest performance
+- **Fastify layer** (`server/fastifyRoutes.ts`) — used for legacy custom functions; wraps Fastify with autoload
+
+All inbound protocols (REST, GraphQL, MQTT, NATS, WebSockets) eventually resolve to the same **Resource interface**.
+
+**Resources** (`resources/`)  
+The universal abstraction. Everything that can be queried or mutated — database tables, caches, message topics, custom endpoints — extends `Resource` (`resources/Resource.ts`).
+
+Static methods (`Resource.get`, `Resource.put`, `Resource.post`, `Resource.delete`, `Resource.patch`, `Resource.subscribe`) are the entry points and are automatically wrapped with `transactional()` for transaction management. Override instance methods (`get`, `put`, etc.) for custom behavior.
+
+`Table.ts` is the database table implementation (~177KB) — the most complex file in the codebase.
+
+**Data Layer** (`dataLayer/`)  
+Legacy translation modules plus SQL translation (`sqlTranslator/`) via AlaSQL; these should be avoided. The storage engine is selectable via `HARPER_STORAGE_ENGINE=lmdb`.
+
+**Configuration** (`config/`)  
+YAML-based. `configUtils.js` parses config; `RootConfigWatcher.ts` enables hot reload. Environment variables override YAML values.
+
+**Utility** (`utility/`)  
+Logging, error types, helpers, async utilities.
+
+---
+
+## Key Patterns
+
+**`transactional()` wrapper** — All static Resource methods go through this. It ensures async operations run inside a database transaction. Use `contextStorage` (AsyncLocalStorage) to access the current transaction context without passing it explicitly.
+
+**Resource discovery** — A component's config file maps glob patterns to URL paths. Files matching a pattern become routable resources automatically; no explicit route registration is needed.
+
+**Lazy loading** — GraphQL, secure sandboxing, and tarball extraction are imported on demand. Do not add top-level imports for these modules.
+
+**TypeScript + type stripping** — Source files are `.ts` but Node.js runs them directly via type stripping in development. The `dist/` directory is the compiled production artifact. Both `.ts` and legacy `.js` files coexist; new code should be `.ts`.
+
+**Minimal dependencies** — `dependencies.md` documents the rationale for every dependency. Adding a new dependency requires justification; implementing something ourselves is often preferred.
+
+---
+
+## Non-Obvious Constraints
+
+- `Resource` static methods must stay wrapped with `transactional()` — removing this breaks transaction isolation.
+- Worker threads (`server/threads/`) receive `workerData.noServerStart = true` to prevent recursive server startup; never start the server inside a worker.
+- `contextStorage` (AsyncLocalStorage) carries per-request context (user, transaction) across async boundaries — this is how authorization and transactions work without explicit parameter threading.
+- Tests under `unitTests/apiTests/` require the server to be stopped first (`node ./dist/bin/harper.js stop`) — `test:unit:apitests` does this automatically.
+- `@export` annotation on a schema class auto-generates a REST API for that table — this is the primary developer-facing API.

package/core/CLAUDE.md

@@ -0,0 +1 @@
+Please see AGENTS.md for guidance on this project.

package/core/bin/status.js

@@ -6,7 +6,7 @@ const YAML = require('yaml');
 
 const hdbTerms = require('../utility/hdbTerms.ts');
 const hdbLog = require('../utility/logging/harper_logger.js');
-const sysInfo = require('../utility/environment/systemInformation.js');
+const systemInformation = require('../utility/environment/systemInformation.ts');
 const envMgr = require('../utility/environment/environmentManager.js');
 const installation = require('../utility/installation.ts');
 envMgr.initSync();
@@ -51,7 +51,7 @@ async function status() {
 	}
 
 	// Check the saved pid against any running hdb processes
-	const hdbSysInfo = await sysInfo.getHDBProcessInfo();
+	const hdbSysInfo = await systemInformation.getHDBProcessInfo();
 	for (const proc of hdbSysInfo.core) {
 		if (proc.pid === hdbPid) {
 			status.harperdb.status = STATUSES.RUNNING;

package/core/bin/stop.js

@@ -4,7 +4,7 @@ const hdbLogger = require('../utility/logging/harper_logger.js');
 const util = require('util');
 const childProcess = require('child_process');
 const exec = util.promisify(childProcess.exec);
-const sysInfo = require('../utility/environment/systemInformation.js');
+const systemInformation = require('../utility/environment/systemInformation.ts');
 
 const STOP_MSG = 'Stopping Harper Pro.';
 
@@ -14,9 +14,8 @@ async function stop() {
 	console.log(STOP_MSG);
 	hdbLogger.notify(STOP_MSG);
 
-	const processes = await sysInfo.getHDBProcessInfo();
-
-	processes.core.forEach((p) => {
-		exec(`kill ${p.pid}`);
-	});
+	const processes = await systemInformation.getHDBProcessInfo();
+	for (const { pid } of processes.core) {
+		exec(`kill ${pid}`);
+	}
 }

package/core/components/EntryHandler.ts

@@ -189,12 +189,14 @@ export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
 			.watch(this.#component.commonPatternBase, {
 				cwd: this.#component.directory,
 				persistent: false,
+				followSymlinks: false,
 				ignored: (path) => {
 					const normalizedPath = path.replace(/\\/g, '/');
 					const normalizedBases = allowedBases.map((base) => base.replace(/\\/g, '/'));
 					return (
-						normalizedPath !== this.#component.directory.replace(/\\/g, '/') &&
-						normalizedBases.every((base) => !normalizedPath.startsWith(base))
+						normalizedPath.includes('/node_modules') ||
+						(normalizedPath !== this.#component.directory.replace(/\\/g, '/') &&
+							normalizedBases.every((base) => !normalizedPath.startsWith(base)))
 					);
 				},
 			})

package/core/components/Scope.ts

@@ -63,7 +63,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		this.#pluginName = pluginName;
 		this.#directory = directory;
 		this.#configFilePath = configFilePath;
-		this.#logger = logger || loggerWithTag(this.#appName);
+		this.#logger = loggerWithTag(this.#appName);
 
 		this.databaseEvents = databaseEventsEmitter;
 		this.applicationScope = applicationScope;

package/core/components/componentLoader.ts

@@ -17,7 +17,8 @@ import * as staticFiles from '../server/static.ts';
 import * as loadEnv from '../resources/loadEnv.ts';
 import harperLogger from '../utility/logging/harper_logger.js';
 import * as dataLoader from '../resources/dataLoader.ts';
-import { watchDir, getWorkerIndex } from '../server/threads/manageThreads.js';
+import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.js';
+import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts';
 import { scopedImport } from '../security/jsLoader.ts';
 import { server } from '../server/Server.ts';
 import { Resources } from '../resources/Resources.ts';
@@ -515,10 +516,16 @@ export async function loadComponent(
 		}
 
 		compName = parentCompName;
-		// Auto restart threads on changes to any app folder. TODO: Make this configurable
 		if (isMainThread && !watchesSetup && autoReload) {
-			watchDir(componentDirectory, async () => {
-				return loadComponentDirectories(); // return the promise
+			let debounceTimer: ReturnType<typeof setTimeout> | null = null;
+			subscribeToRestartRequests(() => {
+				if (debounceTimer) clearTimeout(debounceTimer);
+				debounceTimer = setTimeout(async () => {
+					debounceTimer = null;
+					resetRestartNeeded();
+					await loadComponentDirectories();
+					restartWorkers();
+				}, 500);
 			});
 		}
 		if ((config.extensionModule || config.pluginModule) && (!isMainThread || config.runOnMainThread)) {

package/core/components/requestRestart.ts

@@ -1,11 +1,19 @@
 import { Status } from '../server/status/index.ts';
 
-let restartArrayBuffer: ArrayBuffer;
+interface NotifyingArrayBuffer extends ArrayBuffer {
+	notify(): void;
+	cancel(): void;
+}
+
+let restartArrayBuffer: NotifyingArrayBuffer;
 let restartNeededArray: Uint8Array;
+let onRestartRequestedCallback: (() => void) | null = null;
 
 function ensureInitialized() {
 	if (!restartArrayBuffer) {
-		restartArrayBuffer = Status.primaryStore.getUserSharedBuffer('restart-needed', new ArrayBuffer(1));
+		restartArrayBuffer = Status.primaryStore.getUserSharedBuffer('restart-needed', new ArrayBuffer(1), {
+			callback: () => onRestartRequestedCallback?.(),
+		}) as NotifyingArrayBuffer;
 		restartNeededArray = new Uint8Array(restartArrayBuffer);
 	}
 }
@@ -13,6 +21,7 @@ function ensureInitialized() {
 export function requestRestart() {
 	ensureInitialized();
 	restartNeededArray[0] = 1;
+	restartArrayBuffer.notify();
 }
 
 export function restartNeeded() {
@@ -24,3 +33,9 @@ export function resetRestartNeeded() {
 	ensureInitialized();
 	restartNeededArray[0] = 0;
 }
+
+export function subscribeToRestartRequests(callback: () => void) {
+	ensureInitialized();
+	if (onRestartRequestedCallback) throw new Error('A restart-request subscriber is already registered');
+	onRestartRequestedCallback = callback;
+}

package/core/dataLayer/harperBridge/TableSizeObject.ts

@@ -0,0 +1,35 @@
+/**
+ * Represents the table size entry for a RocksDB or LMDB table.
+ */
+export class TableSizeObject {
+	schema: string;
+	table: string;
+	tableSize: number;
+	recordCount: number;
+	transactionLogSize: number;
+	transactionLogRecordCount?: number;
+
+	/**
+	 * @param schema - The schema of the table
+	 * @param table - The name of the table
+	 * @param tableSize - The data size of the table in bytes
+	 * @param recordCount - The number of entries in the table
+	 * @param transactionLogSize - The number of entries in the transaction log
+	 * @param transactionLogRecordCount - The data size of the transaction log in bytes
+	 */
+	constructor(
+		schema: string,
+		table: string,
+		tableSize: number = 0,
+		recordCount: number = 0,
+		transactionLogSize: number = 0,
+		transactionLogRecordCount?: number
+	) {
+		this.schema = schema;
+		this.table = table;
+		this.tableSize = tableSize;
+		this.recordCount = recordCount;
+		this.transactionLogSize = transactionLogSize;
+		this.transactionLogRecordCount = transactionLogRecordCount;
+	}
+}

package/core/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts

@@ -0,0 +1,24 @@
+import { TableSizeObject } from '../../TableSizeObject.ts';
+import logger from '../../../../utility/logging/harper_logger.js';
+import type { Table } from '../../../../resources/databases.ts';
+
+/**
+ * Calculates the number of entries & data size in bytes for a table & its transaction log
+ * @param table
+ * @returns {TableSizeObject}
+ */
+export function lmdbGetTableSize(table: Table) {
+	const tableStats = new TableSizeObject(table.databaseName, table.tableName);
+	try {
+		const dbiStat = table.primaryStore.getStats();
+
+		//get the txn log record count
+		const txnDbiStat = table.auditStore?.getStats();
+
+		tableStats.recordCount = dbiStat.entryCount;
+		tableStats.transactionLogRecordCount = txnDbiStat.entryCount;
+	} catch (e) {
+		logger.warn(`unable to stat table dbi due to ${e}`);
+	}
+	return tableStats;
+}