@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/security/{role.js → role.ts}

@@ -1,30 +1,23 @@
 'use strict';
 
-const insert = require('../dataLayer/insert.js');
-const search = require('../dataLayer/search.js');
-const delete_ = require('../dataLayer/delete.js');
-const validation = require('../validation/role_validation.js');
-const signalling = require('../utility/signalling.js');
-const util = require('util');
-const terms = require('../utility/hdbTerms.ts');
-const hdbUtils = require('../utility/common_utils.js');
-const { databases } = require('../resources/databases.ts');
+import * as insert from '../dataLayer/insert.ts';
+import * as search from '../dataLayer/search.ts';
+import * as delete_ from '../dataLayer/delete.ts';
+import * as validation from '../validation/role_validation.ts';
+import * as signalling from '../utility/signalling.ts';
+import * as util from 'util';
+import * as terms from '../utility/hdbTerms.ts';
+import * as hdbUtils from '../utility/common_utils.ts';
+import { databases } from '../resources/databases.ts';
 const pSearchSearchByValue = search.searchByValue;
 const pSearchSearchByHash = search.searchByHash;
-const pDeleteDelete = util.promisify(delete_.delete);
-const SearchObject = require('../dataLayer/SearchObject.js');
-const SearchByHashObject = require('../dataLayer/SearchByHashObject.js');
-const { hdbErrors, handleHDBError } = require('../utility/errors/hdbError.js');
-const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
-const { UserEventMsg } = require('../server/threads/itc.js');
-
-module.exports = {
-	addRole,
-	alterRole,
-	dropRole,
-	listRoles,
-	getRoleByName,
-};
+const pDeleteDelete = util.promisify(delete_.delete_);
+import SearchObject from '../dataLayer/SearchObject.ts';
+import SearchByHashObject from '../dataLayer/SearchByHashObject.ts';
+import { handleHDBError } from '../utility/errors/hdbError.ts';
+import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
+
+import { UserEventMsg } from '../server/threads/itc.js';
 
 function scrubRoleDetails(role) {
 	try {
@@ -46,7 +39,7 @@ function scrubRoleDetails(role) {
 	return role;
 }
 
-async function addRole(role) {
+export async function addRole(role: any) {
 	let validationResp = validation.addRoleValidation(role);
 	if (validationResp) {
 		throw validationResp;
@@ -68,7 +61,7 @@ async function addRole(role) {
 		// here, and for other interactions, need convert to real array
 		searchRole = Array.from((await pSearchSearchByValue(searchObj)) || []);
 	} catch (err) {
-		throw handleHDBError(err);
+		throw handleHDBError(err as any, undefined, undefined, undefined, undefined, undefined);
 	}
 
 	if (searchRole && searchRole.length > 0) {
@@ -100,7 +93,7 @@ async function addRole(role) {
 	return role;
 }
 
-async function alterRole(role) {
+export async function alterRole(role: any) {
 	let validationResp = validation.alterRoleValidation(role);
 	if (validationResp) {
 		throw validationResp;
@@ -119,7 +112,7 @@ async function alterRole(role) {
 	try {
 		updateResponse = await insert.update(updateObject);
 	} catch (err) {
-		throw handleHDBError(err);
+		throw handleHDBError(err as any, undefined, undefined, undefined, undefined, undefined);
 	}
 
 	if (updateResponse && updateResponse?.message === 'updated 0 of 1 records') {
@@ -130,13 +123,13 @@ async function alterRole(role) {
 	return role;
 }
 
-async function dropRole(role) {
+export async function dropRole(role: any) {
 	let validationResp = validation.dropRoleValidation(role);
 	if (validationResp) {
 		throw handleHDBError(new Error(), validationResp, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
 	}
 
-	let roleIdSearch = new SearchByHashObject(
+	let roleIdSearch = new (SearchByHashObject as any)(
 		terms.SYSTEM_SCHEMA_NAME,
 		terms.SYSTEM_TABLE_NAMES.ROLE_TABLE_NAME,
 		[role.id],
@@ -155,7 +148,7 @@ async function dropRole(role) {
 		);
 	}
 
-	let searchUserByRoleid = new SearchObject(
+	let searchUserByRoleid = new (SearchObject as any)(
 		terms.SYSTEM_SCHEMA_NAME,
 		terms.SYSTEM_TABLE_NAMES.USER_TABLE_NAME,
 		'role',
@@ -197,14 +190,14 @@ async function dropRole(role) {
 	return `${roleName[0].role} successfully deleted`;
 }
 
-async function getRoleByName(roleName) {
-	for await (const role of databases.system.hdb_role.search([{ attribute: 'role', value: roleName }])) {
+export async function getRoleByName(roleName: string) {
+	for await (const role of databases.system.hdb_role.search([{ attribute: 'role', value: roleName } as any])) {
 		return role;
 	}
 	return null;
 }
 
-async function listRoles() {
+export async function listRoles() {
 	let searchObj = {
 		table: 'hdb_role',
 		schema: 'system',

package/security/tokenAuthentication.ts

@@ -2,7 +2,7 @@ import jwt, { type Algorithm, type JwtPayload, type Secret, type SignOptions } f
 import fs from 'fs-extra';
 import path from 'node:path';
 import Joi from 'joi';
-import { validateBySchema } from '../validation/validationWrapper.js';
+import { validateBySchema } from '../validation/validationWrapper.ts';
 import {
 	CONFIG_PARAMS,
 	JWT_ENUM,
@@ -10,16 +10,16 @@ import {
 	SYSTEM_SCHEMA_NAME,
 	SYSTEM_TABLE_NAMES,
 } from '../utility/hdbTerms.ts';
-import { ClientError, hdbErrors } from '../utility/errors/hdbError.js';
+import { ClientError, hdbErrors } from '../utility/errors/hdbError.ts';
 const { HTTP_STATUS_CODES, AUTHENTICATION_ERROR_MSGS } = hdbErrors;
-import logger from '../utility/logging/harper_logger.js';
+import logger from '../utility/logging/harper_logger.ts';
 import * as password from '../utility/password.ts';
 import { findAndValidateUser, type User } from './user.ts';
-import { update } from '../dataLayer/insert.js';
-import UpdateObject from '../dataLayer/UpdateObject.js';
-import signalling from '../utility/signalling.js';
+import { update } from '../dataLayer/insert.ts';
+import UpdateObject from '../dataLayer/UpdateObject.ts';
+import * as signalling from '../utility/signalling.ts';
 import { UserEventMsg } from '../server/threads/itc.js';
-import env from '../utility/environment/environmentManager.js';
+import * as env from '../utility/environment/environmentManager.ts';
 env.initSync();
 
 type StringValue = SignOptions['expiresIn'];
@@ -226,3 +226,30 @@ async function validateToken(token: string, tokenType: string): Promise<any> {
 		throw new ClientError(AUTHENTICATION_ERROR_MSGS.INVALID_TOKEN, HTTP_STATUS_CODES.UNAUTHORIZED);
 	}
 }
+
+/**
+ * Decodes a JWT and returns its payload.
+ * @param {string} token The JWT token to decode.
+ * @returns {Object|null} The decoded payload or null if invalid.
+ */
+export function decodeJWT(token: string): null | { exp: number; iat: number } {
+	try {
+		const parts = token.split('.');
+		if (parts.length !== 3) return null;
+		const payload = parts[1];
+		const decoded = Buffer.from(payload, 'base64').toString('utf8');
+		return JSON.parse(decoded);
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Decodes a JWT and checks if it has expired or is going to expire soon (based on the buffer seconds).
+ */
+export function isJWTExpired(token: string, bufferSeconds = 300): boolean {
+	const payload = decodeJWT(token);
+	if (!payload || !payload.exp) return true;
+	const now = Math.floor(Date.now() / 1000);
+	return payload.exp < now + bufferSeconds;
+}

package/security/user.ts

@@ -82,29 +82,29 @@ export interface CRUDPermissions {
 }
 
 //requires must be declared after module.exports to avoid cyclical dependency
-const insert = require('../dataLayer/insert.js');
-const delete_ = require('../dataLayer/delete.js');
-const validation = require('../validation/user_validation.js');
-const search = require('../dataLayer/search.js');
-const signalling = require('../utility/signalling.js');
-const hdbUtility = require('../utility/common_utils.js');
-const validate = require('validate.js');
-const logger = require('../utility/logging/harper_logger.js');
-const { promisify } = require('util');
-const env = require('../utility/environment/environmentManager.js');
-const systemSchema = require('../json/systemSchema.json');
-const { hdbErrors, ClientError } = require('../utility/errors/hdbError.js');
+import * as insert from '../dataLayer/insert.ts';
+import * as delete_ from '../dataLayer/delete.ts';
+import * as validation from '../validation/user_validation.ts';
+import * as search from '../dataLayer/search.ts';
+import * as signalling from '../utility/signalling.ts';
+import * as hdbUtility from '../utility/common_utils.ts';
+import * as validate from 'validate.js';
+import * as logger from '../utility/logging/harper_logger.ts';
+import { promisify } from 'util';
+import * as env from '../utility/environment/environmentManager.ts';
+import systemSchema from '../json/systemSchema.json';
+import { hdbErrors, ClientError } from '../utility/errors/hdbError.ts';
 const { HTTP_STATUS_CODES, AUTHENTICATION_ERROR_MSGS, HDB_ERROR_MSGS } = hdbErrors;
 const { UserEventMsg } = require('../server/threads/itc.js');
-const _ = require('lodash');
-const harperLogger = require('../utility/logging/harper_logger.js');
+import * as _ from 'lodash';
+import * as harperLogger from '../utility/logging/harper_logger.ts';
 
 // Need to use `.js` even for other TS files since TS compiler won't replace requires.
 // Whenever we can fix the cyclical dependency issue in this file (and switch to imports) we can use the correct file extensions.
-const password = require('../utility/password.js');
-const { server } = require('../server/Server.js');
-const terms = require('../utility/hdbTerms.js');
-const { expandOperationsPerms } = require('../utility/operationPermissions.js');
+import * as password from '../utility/password.ts';
+import { server } from '../server/Server.ts';
+import * as terms from '../utility/hdbTerms.ts';
+import { expandOperationsPerms } from '../utility/operationPermissions.ts';
 
 server.getUser = (username: string, password?: string | null): Promise<User> => {
 	return findAndValidateUser(username, password, password != null);
@@ -121,7 +121,7 @@ const USER_ATTRIBUTE_ALLOWLIST = {
 	password: true,
 };
 const passwordHashCache = new Map();
-const promiseDelete = promisify(delete_.delete);
+const promiseDelete = promisify(delete_.delete_);
 const configuredHashFunction =
 	env.get(terms.CONFIG_PARAMS.AUTHENTICATION_HASHFUNCTION) ?? password.HASH_FUNCTION.SHA256;
 let usersWithRolesMap;
@@ -416,9 +416,13 @@ async function findAndValidateUser(username: string, pw?: string | null, validat
 		if (passwordHashCache.get(pw) === userTmp.password) return user;
 		// if validates, cache the password
 		else {
-			let validated = password.validate(userTmp.password, pw, userTmp.hash_function || password.HASH_FUNCTION.MD5); // if no hashFunction default to legacy MD5
+			let validated: boolean | Promise<boolean> = password.validate(
+				userTmp.password,
+				pw,
+				userTmp.hash_function || password.HASH_FUNCTION.MD5
+			); // if no hashFunction default to legacy MD5
 			// argon2id hash validation is async so await it if it is a promise
-			if (validated?.then) validated = await validated;
+			if (typeof validated === 'object' && (validated as Promise<boolean>)?.then) validated = await validated;
 			if (validated === true) passwordHashCache.set(pw, userTmp.password);
 			else throw new ClientError(AUTHENTICATION_ERROR_MSGS.GENERIC_AUTH_FAIL, HTTP_STATUS_CODES.UNAUTHORIZED);
 		}
@@ -436,7 +440,7 @@ async function getSuperUser(): Promise<User | undefined> {
 }
 
 let invalidateCallbacks = [];
-server.invalidateUser = function (user: User | any) {
+(server as any).invalidateUser = function (user: User | any) {
 	for (let callback of invalidateCallbacks) {
 		try {
 			callback(user);

package/server/DESIGN.md

@@ -0,0 +1,139 @@
+# server/ — Navigation Guide
+
+This layer accepts inbound traffic on every supported protocol (HTTP/1.1, HTTP/2, HTTPS, WebSockets, MQTT, NATS) and routes it through to the Resource layer.
+
+**Read this when:** you're touching request/response, protocol handling, middleware ordering, or WebSocket upgrade behavior.
+
+> **Navigation convention.** This guide references code by **symbol name** (function/const). Use your editor's go-to-symbol or `grep -n '<name>' server/<file>` to jump. Line numbers drift; symbols don't.
+
+---
+
+## Three HTTP stacks coexist — know which one
+
+| Stack                         | File                  | Used for                                                                                                                                                                                        |
+| ----------------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Native**                    | `http.ts`             | Direct socket handling for application-level HTTP/1.1, HTTPS, HTTP/2, and WebSockets. Highest performance. This is the path most user requests take (REST, GraphQL, custom resource endpoints). |
+| **Operations API**            | `operationsServer.ts` | Fastify-based JSON operations API (`{operation: 'create_table', ...}`). Internal/admin surface — not on the hot path for application data.                                                      |
+| **Custom Functions (legacy)** | `fastifyRoutes.ts`    | Legacy custom functions only. Wraps Fastify with autoload. Don't add new code here.                                                                                                             |
+
+A request entering `http.ts` does **not** go through Fastify. The two `handleApplication(scope)` functions (one in each Fastify file) load independently from component config.
+
+---
+
+## File overview
+
+### Core dispatch
+
+| File                             | Purpose                                                                                                                                                                                                       |
+| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `Server.ts`                      | Defines the `Server` interface — the contract that protocol plugins use to register listeners. Has `socket()`, `http()`, `ws()`, `upgrade()`, `contentTypes`, `getUser()`, `operation()`, `replication`, etc. |
+| `http.ts`                        | Native HTTP/WS server. Registration entry points (`onRequest`, `onUpgrade`, `onWebSocket`), per-port middleware chains, UDS support, PROXY protocol. **See section map below.**                               |
+| `middlewareChain.ts`             | Topological sort respecting `before`/`after` constraints on listener registrations (`topoSort`). Falls back to registration order on cycle.                                                                   |
+| `REST.ts`                        | Resource-routed REST handler: URL → `Resource.getResource()` → method dispatch + content negotiation.                                                                                                         |
+| `graphqlQuerying.ts`             | GraphQL query/mutation/subscription execution against Resources.                                                                                                                                              |
+| `mqtt.ts`                        | MQTT broker (connect/sub/pub mapped onto Resource interface).                                                                                                                                                 |
+| `DurableSubscriptionsSession.ts` | Persistent subscription state (resume across reconnects).                                                                                                                                                     |
+
+### Operations & Fastify
+
+| File                  | Purpose                                                                                                                        |
+| --------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
+| `operationsServer.ts` | Boots Fastify for operations API. `buildServer()` constructs the server; `handler()` parses `{operation: ...}` and dispatches. |
+| `fastifyRoutes.ts`    | Legacy custom functions. Discovers routes from each component's `routes/` folder.                                              |
+
+### Helpers
+
+| File                                       | Purpose                                                                         |
+| ------------------------------------------ | ------------------------------------------------------------------------------- |
+| `serverHelpers/Request.ts`                 | Wraps `IncomingMessage` with Harper-specific fields (user, response, headers).  |
+| `serverHelpers/Headers.ts`                 | Header mutation/merge utilities.                                                |
+| `serverHelpers/contentTypes.ts`            | (de)serialization registry; `serialize`, `serializeMessage`, `getDeserializer`. |
+| `serverHelpers/serverUtilities.ts`         | `OperationDefinition` and shared helpers.                                       |
+| `serverHelpers/OperationFunctionObject.ts` | Wraps an operation handler with metadata.                                       |
+| `serverHelpers/JSONStream.ts`              | Streaming JSON output for large responses.                                      |
+| `nodeName.ts`                              | Resolves this node's name (config → hostname).                                  |
+| `static.ts`                                | Static file serving for component-bundled assets.                               |
+| `throttle.ts`                              | Per-IP / per-user request throttling.                                           |
+| `storageReclamation.ts`                    | Disk-pressure signals to downstream consumers.                                  |
+| `serverRegistry.ts`                        | Trivial registry export.                                                        |
+| `status/`                                  | Server status reporting (cluster status, per-port info).                        |
+
+### Threads
+
+| File                       | Purpose                                                  |
+| -------------------------- | -------------------------------------------------------- |
+| `threads/socketRouter.ts`  | Routes accepted sockets to worker threads based on port. |
+| `threads/manageThreads.js` | Thread pool lifecycle.                                   |
+| `threads/threadServer.js`  | Worker entry point — receives sockets via IPC.           |
+| `threads/itc.js`           | Inter-thread comms primitives.                           |
+
+> Workers receive `workerData.noServerStart = true` — never start the server inside a worker.
+
+---
+
+## `http.ts` — symbol map
+
+Every entry is a top-level function or named const. Jump via go-to-symbol or `grep -n 'function <name>' server/http.ts`.
+
+| Symbol                                                                                      | What it does                                                                                                                                                                               |
+| ------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `registerUdsCleanupPaths`, `cleanupUdsFiles`, `writeUdsMetadata`, `cleanupSocketsDirectory` | UDS socket / metadata file lifecycle.                                                                                                                                                      |
+| `handleApplication(scope)`                                                                  | Component entry point — captures `httpOptions` for the scope.                                                                                                                              |
+| `getHttpOptions()`                                                                          | Returns the current scope's `HttpOptions`.                                                                                                                                                 |
+| `deliverSocket()`                                                                           | IPC-delivered socket handoff from `socketRouter`.                                                                                                                                          |
+| `proxyRequest()`                                                                            | Cross-port request routing.                                                                                                                                                                |
+| `registerServer()`                                                                          | Records a server for a port in the `SERVERS` map.                                                                                                                                          |
+| `getPorts()`                                                                                | Resolves listener options → list of `{port, secure}`.                                                                                                                                      |
+| `httpServer()`                                                                              | Main listener registration entry point.                                                                                                                                                    |
+| `getHTTPServer(port, secure, options)`                                                      | **The largest function in the file.** Creates/retrieves the underlying Node HTTP/HTTPS server. Wires `request`, `upgrade`, error handlers, TLS context, and the per-port middleware chain. |
+| `makeCallbackChain()`                                                                       | Builds the per-port handler chain via `middlewareChain.topoSort`.                                                                                                                          |
+| `unhandled()`                                                                               | Terminal 404 handler.                                                                                                                                                                      |
+| `onRequest()`                                                                               | Thin alias of `httpServer({requestOnly: true})`.                                                                                                                                           |
+| `onUpgrade()` / `upgradeListeners` (const)                                                  | Register HTTP upgrade listener; underlying list.                                                                                                                                           |
+| `onWebSocket()` / `websocketListeners` (const)                                              | Register WebSocket listener; auto-adds default upgrade handler the first time it runs for a port. Underlying list of registrations.                                                        |
+| `enableProxyProtocol()`                                                                     | PROXY v1 parsing (Node 24+-compatible workaround).                                                                                                                                         |
+| `defaultNotFound()`                                                                         | Default 404 response.                                                                                                                                                                      |
+| `logRequest()`                                                                              | Per-request access log line.                                                                                                                                                               |
+| `getRequestId()`                                                                            | Generates the per-request correlation ID.                                                                                                                                                  |
+
+### Middleware ordering (`before` / `after`)
+
+Components register listeners with optional `before: 'name'` / `after: 'name'` options. `middlewareChain.topoSort` resolves order; cycles fall back to registration order with a warning. Three lists hold the registrations:
+
+- `httpResponders` — request handlers
+- `upgradeListeners` (in `http.ts`)
+- `websocketListeners` (in `http.ts`)
+
+The default WebSocket upgrade handler is registered automatically inside `onWebSocket()` the first time it runs for a given port.
+
+---
+
+## Resource ↔ HTTP boundary
+
+`REST.ts → http(request, nextHandler)` is the chief integration point: it takes a `Request`, asks the `Resources` registry for a match, builds a `RequestTarget`, and dispatches into the Resource class's static method. Cache headers are translated to `request.expiresAt` / `onlyIfCached` / `noCache` flags within the same function.
+
+---
+
+## "Where is X" cheat sheet
+
+| Question                                            | Where                                                                 |
+| --------------------------------------------------- | --------------------------------------------------------------------- |
+| Where do I register a new HTTP handler?             | `http.ts → httpServer()` (or `onRequest()` for the request-only form) |
+| Where do I register a WebSocket handler?            | `http.ts → onWebSocket()`                                             |
+| How does `before`/`after` middleware ordering work? | `middlewareChain.ts → topoSort`                                       |
+| Where does PROXY protocol get parsed?               | `http.ts → enableProxyProtocol`                                       |
+| Where is the REST request → Resource dispatch?      | `REST.ts → http()`                                                    |
+| Where is the operations API request handled?        | `operationsServer.ts → handler`                                       |
+| How are content types (de)serialized?               | `serverHelpers/contentTypes.ts`                                       |
+| Where do durable subscriptions live?                | `DurableSubscriptionsSession.ts`                                      |
+| How are sockets dispatched to worker threads?       | `threads/socketRouter.ts`                                             |
+| Where is the Operations API wired into Fastify?     | `operationsServer.ts → buildServer`                                   |
+
+---
+
+## Conventions
+
+- Don't add new code to `fastifyRoutes.ts` — it's the legacy custom-functions path.
+- New protocol plugins implement the `Server` interface (in `Server.ts`) and register via `onRequest`/`onUpgrade`/`onWebSocket`.
+- Always pass `name` when registering a listener with `before`/`after` — anonymous entries can't be ordered against.
+- Tests live in `../unitTests/server/`.