@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/server/static.ts

@@ -0,0 +1,187 @@
+import { realpathSync, existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { Scope } from '../components/Scope';
+import send from 'send';
+
+/**
+ * The static plugin handles serving static files from the respective application directory.
+ * It uses the default `EntryHandler` configured via `files` and `urlPath` to watch for file changes and updates the in-memory map of static files.
+ *
+ * Additionally, it supports additional options:
+ * - `index`: If enabled, it will serve `index.html` files from directories.
+ * - `extensions`: An array of file extensions to try when serving files. If a file is not found, it will try appending each extension in order. For example, if set to `['html'], and the request is `/page`, it will try `/page.html` if `/page` is not found.
+ * - `fallthrough`: If true, it will fall through to the next handler if the file is not found. If false, it will return a 404 error.
+ * - `notFound`: Can be specified as a string to serve a custom 404 page, or an object with `file` and `statusCode` properties to serve a custom file with a specific status code. This is useful for hosting SPAs that use client-side routing. Make sure to set `fallthrough` to `false`!
+ *
+ * This plugin dynamically updates its behavior based on the current configuration file. Users can make updates and immediately see the changes reflect in the next request.
+ *
+ * Updates to the `files` or `urlPath` options will clear the in-memory maps and allow them to regenerate based on the new configuration (since the default EntryHandler will regenerate anyways).
+ */
+export function handleApplication(scope: Scope) {
+	// in-memory map of static files
+	// keys are the URL paths, values are the absolute paths to the files
+	const staticFiles = new Map<string, string>();
+	const indexEntries = new Map<string, string>();
+
+	// If the `files` or `urlPath` options change, clear the maps and let them regenerate
+	scope.options.on('change', (key) => {
+		if (key[0] === 'files' || key[0] === 'urlPath') {
+			// If the files or urlPath options change, we need to reinitialize the static files map
+			staticFiles.clear();
+			indexEntries.clear();
+			scope.logger.info(`Static files reinitialized due to change in ${key.join('.')}`);
+			return;
+		}
+	});
+
+	// Handle entry events for the default entry handler based on the `files` and `urlPath` options
+	scope.handleEntry((entry) => {
+		switch (entry.eventType) {
+			// Directories only matter for the `index` files
+			case 'addDir':
+			case 'unlinkDir':
+				// Handle `index.html` for directories for if/when the user enables the `index` option
+				const indexPath = join(entry.absolutePath, 'index.html');
+				if (existsSync(indexPath)) {
+					indexEntries[entry.eventType === 'addDir' ? 'set' : 'delete'](entry.urlPath, indexPath);
+				}
+				break;
+			// Otherwise, user must specify pattern to match individual files
+			case 'add':
+				// Store the file in memory for serving
+				staticFiles.set(entry.urlPath, entry.absolutePath);
+				// If the file is an index.html, also store it in the index entries
+				if (entry.urlPath.endsWith('index.html')) {
+					// Without trailing slash; null -> 301 redirect to trailing slash
+					indexEntries.set(dirname(entry.urlPath), null);
+					// With trailing slash; serves the index.html file
+					indexEntries.set(join(dirname(entry.urlPath), '/'), entry.absolutePath);
+				}
+				break;
+			case 'unlink':
+				// Remove the file from memory when it is deleted
+				staticFiles.delete(entry.urlPath);
+				// If the file is an index.html, remove it from the index entries as well
+				if (entry.urlPath.endsWith('index.html')) {
+					indexEntries.delete(dirname(entry.urlPath));
+				}
+				break;
+		}
+	});
+
+	scope.server.http(
+		(req, next) => {
+			// TODO: Not sure if the isWebSocket check is still necessary
+			if (req.method !== 'GET' || req.isWebSocket) return next(req);
+
+			// Default fallthrough to `true`
+			const fallthrough = scope.options.get(['fallthrough']) ?? true;
+
+			if (typeof fallthrough !== 'boolean') {
+				throw new Error(`Invalid fallthrough option: ${fallthrough}. Must be a boolean.`);
+			}
+
+			// Attempt to retrieve the requested static file from memory
+			let staticFile = staticFiles.get(req.pathname);
+
+			// If the file is not found, try matching index
+			if (!staticFile) {
+				const index = scope.options.get(['index']) ?? true;
+
+				if (typeof index !== 'boolean') {
+					throw new Error(`Invalid index option: ${index}. Must be a boolean.`);
+				}
+
+				if (index) {
+					// Retrieve index entry
+					staticFile = indexEntries.get(req.pathname);
+
+					// If `null`, redirect to trailing slash
+					if (staticFile === null) {
+						return {
+							status: 301,
+							headers: {
+								Location: join(req.pathname, '/'),
+							},
+						};
+					}
+				}
+			}
+
+			// If the file is still not found, try matching extensions
+			if (!staticFile) {
+				const extensions = scope.options.get(['extensions']) ?? [];
+				if (!Array.isArray(extensions) || extensions.some((ext) => typeof ext !== 'string')) {
+					throw new Error(`Invalid extensions option: ${extensions}. Must be an array of strings.`);
+				}
+
+				for (const ext of extensions) {
+					staticFile = staticFiles.get(`${req.pathname}.${ext}`);
+					// break on first match
+					if (staticFile) break;
+				}
+			}
+
+			// If an entry matched, serve it
+			if (staticFile) {
+				// The benefit to using `send` is that it handles a lot of edge cases and headers for us.
+				return {
+					handlesHeaders: true,
+					body: send(req, realpathSync(staticFile)),
+				};
+			}
+
+			// If fallthrough is true pass along the request to the next handler
+			if (fallthrough) {
+				return next(req);
+			}
+
+			// Otherwise, handle not found
+
+			const notFound = scope.options.get(['notFound']);
+
+			validateNotFoundOption(notFound);
+
+			if (!notFound) {
+				return {
+					status: 404,
+					body: 'File not found',
+				};
+			}
+
+			const notFoundPath = join(scope.directory, typeof notFound === 'string' ? notFound : notFound.file);
+			const statusCode = typeof notFound === 'object' ? notFound.statusCode : 404;
+
+			if (!existsSync(notFoundPath)) {
+				throw new Error(`Not found file does not exist: ${notFoundPath}`);
+			}
+
+			return {
+				status: statusCode,
+				handlesHeaders: true,
+				body: send(req, realpathSync(notFoundPath)),
+			};
+		},
+		{ runFirst: true }
+	);
+}
+
+function validateNotFoundOption(
+	notFound: any
+): asserts notFound is undefined | string | { file: string; statusCode: number } {
+	if (notFound === undefined || typeof notFound === 'string') return;
+
+	if (typeof notFound === 'object' && notFound !== null && !Array.isArray(notFound)) {
+		if (!('file' in notFound) || typeof notFound.file !== 'string') {
+			throw new Error(`Invalid \`notFound.file\` option: ${notFound.file}. Must be a string.`);
+		}
+		if (!('statusCode' in notFound) || typeof notFound.statusCode !== 'number') {
+			throw new Error(`Invalid \`notFound.statusCode\` option: ${notFound.statusCode}. Must be a number.`);
+		}
+		return;
+	}
+
+	throw new Error(
+		`Invalid notFound option: ${notFound}. Must be a string or an object with file and statusCode properties.`
+	);
+}

package/server/status/definitions.ts

@@ -0,0 +1,37 @@
+/**
+ * Single source of truth for Harper status system definitions
+ */
+
+// Define status configurations with const assertion for literal types
+export const STATUS_DEFINITIONS = {
+	primary: {
+		allowedValues: null, // Any string is valid
+	},
+	maintenance: {
+		allowedValues: null, // Any string is valid
+	},
+	availability: {
+		allowedValues: ['Available', 'Unavailable'] as const,
+	},
+} as const;
+
+// Derive types from the definitions
+export type StatusDefinitions = typeof STATUS_DEFINITIONS;
+export type StatusId = keyof StatusDefinitions;
+
+// Status value types derived from definitions
+export type StatusValueMap = {
+	[K in StatusId]: StatusDefinitions[K]['allowedValues'] extends readonly (infer U)[] ? U : string;
+};
+
+// Status record structure
+export interface StatusRecord<T extends StatusId = StatusId> {
+	id: T;
+	status: StatusValueMap[T];
+	__createdtime__?: number;
+	__updatedtime__?: number;
+}
+
+// Utility constants
+export const STATUS_IDS = Object.keys(STATUS_DEFINITIONS) as StatusId[];
+export const DEFAULT_STATUS_ID: StatusId = 'primary';

package/server/status/index.ts

@@ -0,0 +1,125 @@
+import { table } from '../../resources/databases.ts';
+import { handleHDBError, hdbErrors } from '../../utility/errors/hdbError.js';
+import { loggerWithTag } from '../../utility/logging/logger.ts';
+import { validateStatus } from '../../validation/statusValidator.ts';
+import { type StatusId, type StatusValueMap, type StatusRecord, DEFAULT_STATUS_ID } from './definitions.ts';
+import { internal as statusInternal, type AggregatedComponentStatus } from '../../components/status/index.ts';
+import { restartNeeded } from '../../components/requestRestart.ts';
+
+export { clearStatus as clear, getStatus as get, setStatus as set };
+
+// Re-export types for convenience
+export type { StatusId, StatusRecord, StatusValueMap } from './definitions.ts';
+
+export { STATUS_IDS, DEFAULT_STATUS_ID } from './definitions.ts';
+
+const { HTTP_STATUS_CODES } = hdbErrors;
+
+// For direct function calls, we don't need the operation fields
+type StatusRequestBody = {
+	id: StatusId;
+};
+
+type StatusWriteRequestBody<T extends StatusId = StatusId> = {
+	id?: T;
+	status: StatusValueMap[T];
+};
+
+// Lazy-initialize the Status table to avoid initialization issues during module import
+let _statusTable: ReturnType<typeof table>;
+
+function getStatusTable() {
+	if (!_statusTable) {
+		_statusTable = table({
+			database: 'system',
+			table: 'hdb_status',
+			replicate: false,
+			attributes: [
+				{
+					name: 'id',
+					isPrimaryKey: true,
+				},
+				{
+					name: 'status',
+				},
+				{
+					name: '__createdtime__',
+				},
+				{
+					name: '__updatedtime__',
+				},
+			],
+		});
+	}
+	return _statusTable;
+}
+
+// Export Status as a getter for compatibility with modules that need direct table access
+export const Status = {
+	get primaryStore() {
+		return getStatusTable().primaryStore;
+	},
+};
+
+const statusLogger = loggerWithTag('status');
+
+function clearStatus({ id }: StatusRequestBody): Promise<boolean> {
+	statusLogger.debug?.('clearStatus', id);
+	return getStatusTable().delete(id);
+}
+
+interface AggregatedComponentStatusWithName extends AggregatedComponentStatus {
+	name: string;
+}
+
+interface AllStatusSummary {
+	systemStatus: Promise<AsyncIterable<StatusRecord>>;
+	componentStatus: AggregatedComponentStatusWithName[];
+	restartRequired: boolean;
+}
+
+async function getAllStatus(): Promise<AllStatusSummary> {
+	statusLogger.debug?.('getAllStatus');
+	const statusRecords = getStatusTable().search([]);
+
+	// Get aggregated component statuses from all threads
+	const aggregatedStatuses = await statusInternal.query.allThreads();
+	const componentStatusArray: AggregatedComponentStatusWithName[] = Array.from(aggregatedStatuses.entries()).map(
+		([name, status]) => ({
+			name,
+			...status,
+		})
+	);
+
+	// Get restart flag status
+	const restartRequired = restartNeeded();
+
+	return {
+		systemStatus: statusRecords as Promise<AsyncIterable<StatusRecord>>,
+		componentStatus: componentStatusArray,
+		restartRequired,
+	};
+}
+
+function getStatus({ id }: Partial<StatusRequestBody>): Promise<StatusRecord | AllStatusSummary> {
+	if (!id) {
+		statusLogger.debug?.('getStatus', 'all');
+		return getAllStatus();
+	}
+
+	statusLogger.debug?.('getStatus', id);
+	return getStatusTable().get(id) as unknown as Promise<StatusRecord>;
+}
+
+function setStatus<T extends StatusId = StatusId>({
+	status,
+	id = DEFAULT_STATUS_ID as T,
+}: StatusWriteRequestBody<T>): Promise<StatusRecord<T>> {
+	const validation = validateStatus({ status, id });
+	if (validation) {
+		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
+	}
+
+	statusLogger.debug?.('setStatus', id, status);
+	return getStatusTable().put(id, { status }) as Promise<StatusRecord<T>>;
+}

package/server/storageReclamation.ts

@@ -0,0 +1,93 @@
+import { statfs } from 'node:fs/promises';
+import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js';
+import { logger } from '../utility/logging/logger.ts';
+import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
+import envMgr from '../utility/environment/environmentManager.js';
+import { convertToMS } from '../utility/common_utils.js';
+envMgr.initSync();
+const reclamationHandlers = new Map<
+	string,
+	{ priority: number; handler: (priority: number) => Promise<void> | void }[]
+>();
+
+const RECLAMATION_THRESHOLD = envMgr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_THRESHOLD) ?? 0.4; // 40% remaining free space is the default
+const RECLAMATION_INTERVAL = convertToMS(envMgr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_INTERVAL)) || 3600000; // 1 hour is the default
+/**
+ * Register a handler to be called when storage free space is low and reclamation is needed. The callback is called
+ * with the priority of the reclamation, which is the ratio of the threshold to the available space ratio. If space is
+ * low, the priority will be greater than 1. If the reclamation is successful, the callback will be called again with
+ * a priority of 0.
+ * @param path
+ * @param handler
+ */
+export function onStorageReclamation(
+	path: string,
+	handler: (priority: number) => Promise<void> | void,
+	skipThreadCheck?: boolean
+) {
+	if (skipThreadCheck || getWorkerIndex() === getWorkerCount() - 1) {
+		// only run on one thread (last one)
+		if (!path) {
+			throw new Error('Storage reclamation path cannot be empty');
+		}
+		if (!reclamationHandlers.has(path)) {
+			reclamationHandlers.set(path, []);
+		}
+		reclamationHandlers.get(path).push({ priority: 0, handler });
+		if (!reclamationTimer) reclamationTimer = setTimeout(runReclamationHandlers, RECLAMATION_INTERVAL).unref();
+	}
+}
+let reclamationTimer: NodeJS.Timeout;
+const defaultGetAvailableSpaceRatio = async (path: string): Promise<number> => {
+	if (statfs) {
+		const fsStats = await statfs(path);
+		return fsStats.bavail / fsStats.blocks;
+	} else {
+		return new Promise((resolve) => {
+			import('hdd-space').then((hddSpace) => {
+				hddSpace.default((space: any) => {
+					for (const volume of space.parts) {
+						if (path.startsWith(volume.place)) return resolve(volume.free / volume.size);
+					}
+					return resolve(1);
+				});
+			});
+		});
+	}
+};
+let getAvailableSpaceRatio: (path: string) => Promise<number> = defaultGetAvailableSpaceRatio;
+
+/**
+ * Run the registered reclamation handlers, if any disk drives are below the threshold
+ */
+export async function runReclamationHandlers() {
+	for (const [path, handlers] of reclamationHandlers) {
+		try {
+			const availableRatio = await getAvailableSpaceRatio(path);
+			const priority = RECLAMATION_THRESHOLD / availableRatio;
+			for (const entry of handlers) {
+				const { priority: previousPriority, handler } = entry;
+				entry.priority = priority;
+				if (priority > 1 || previousPriority > 1) {
+					const resolution = handler(priority > 1 ? priority : 0);
+					if (resolution) {
+						// if the handler returns a promise, wait for it, otherwise it is probably not doing anything worth logging
+						logger.info?.(`Running storage reclamation handler for ${path} with priority ${priority}`);
+						await resolution;
+					}
+				}
+			}
+		} catch (e) {
+			logger.error?.('Error running storage reclamation handlers', e);
+		}
+	}
+	reclamationTimer = setTimeout(runReclamationHandlers, RECLAMATION_INTERVAL).unref();
+}
+
+/**
+ * Set the function used to get the available space ratio (for testing and backfill for Node v16)
+ * @param newGetter
+ */
+export function setAvailableSpaceRatioGetter(newGetter?: (path: string) => Promise<number>) {
+	getAvailableSpaceRatio = newGetter ?? defaultGetAvailableSpaceRatio;
+}

package/server/threads/itc.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const hdbUtils = require('../../utility/common_utils.js');
+const hdbTerms = require('../../utility/hdbTerms.ts');
+const { ITC_ERRORS } = require('../../utility/errors/commonErrors.js');
+const { threadId, isMainThread } = require('worker_threads');
+const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js');
+
+module.exports = {
+	sendItcEvent,
+	validateEvent,
+	SchemaEventMsg,
+	UserEventMsg,
+};
+let serverItcHandlers;
+onMessageFromWorkers(async (event, sender) => {
+	serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js');
+	validateEvent(event);
+	if (serverItcHandlers[event.type]) {
+		await serverItcHandlers[event.type](event);
+	}
+	if (event.requestId && sender)
+		sender.postMessage({
+			type: 'ack',
+			id: event.requestId,
+		});
+});
+
+/**
+ * Emits an ITC event to the ITC server.
+ * @param event
+ */
+function sendItcEvent(event) {
+	if (!isMainThread && event.message) event.message.originator = threadId;
+	return broadcastWithAcknowledgement(event);
+}
+
+/**
+ * Does some basic validation on an ITC event.
+ * @param event
+ * @returns {string}
+ */
+function validateEvent(event) {
+	if (typeof event !== 'object') {
+		return ITC_ERRORS.INVALID_ITC_DATA_TYPE;
+	}
+
+	if (!event.hasOwnProperty('type') || hdbUtils.isEmpty(event.type)) {
+		return ITC_ERRORS.MISSING_TYPE;
+	}
+
+	if (!event.hasOwnProperty('message') || hdbUtils.isEmpty(event.message)) {
+		return ITC_ERRORS.MISSING_MSG;
+	}
+
+	if (!event.message.hasOwnProperty('originator') || hdbUtils.isEmpty(event.message.originator)) {
+		return ITC_ERRORS.MISSING_ORIGIN;
+	}
+
+	if (hdbTerms.ITC_EVENT_TYPES[event.type.toUpperCase()] === undefined) {
+		return ITC_ERRORS.INVALID_EVENT(event.type);
+	}
+}
+
+/**
+ * Constructor function for the message of schema ITC events
+ * @param originator
+ * @param operation
+ * @param schema
+ * @param table
+ * @param attribute
+ * @constructor
+ */
+function SchemaEventMsg(originator, operation, schema, table = undefined, attribute = undefined) {
+	this.originator = originator;
+	this.operation = operation;
+	this.schema = schema;
+	this.table = table;
+	this.attribute = attribute;
+}
+
+/**
+ * Constructor function for the message of user ITC events
+ * @param originator
+ * @constructor
+ */
+function UserEventMsg(originator) {
+	this.originator = originator;
+}