@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/components/OptionsWatcher.ts

@@ -0,0 +1,354 @@
+import { type Logger } from '../utility/logging/logger.ts';
+import { loggerWithTag } from '../utility/logging/harper_logger.js';
+import { EventEmitter, once } from 'events';
+import yaml from 'yaml';
+import chokidar, { type FSWatcher } from 'chokidar';
+import { readFile } from 'node:fs/promises';
+import { isDeepStrictEqual } from 'util';
+import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.js';
+import { cloneDeep } from 'lodash';
+
+export interface Config {
+	[key: string]: ConfigValue;
+}
+
+export type ConfigValue = undefined | null | string | number | boolean | Array<ConfigValue> | Config;
+
+export type OptionsWatcherEventMap = {
+	ready: [config?: ConfigValue];
+	change: [key: string[], value: ConfigValue, config: ConfigValue];
+	remove: [];
+	error: [error: unknown];
+	close: [];
+};
+
+// This is uniquely for errors coming from the chokidar watcher of the config file.
+export class OptionsWatcherConfigFileError extends Error {
+	constructor(configFilePath: string, error: unknown) {
+		super(
+			`Error watching config file ${configFilePath}: ${typeof error === 'object' && error !== null && 'message' in error ? error.message : error}`
+		);
+		this.name = 'OptionsWatcherConfigFileError';
+	}
+}
+
+export class UninitializedOptionsWatcherError extends Error {
+	constructor() {
+		super(
+			'OptionsWatcher has not been initialized yet. Await `ready()` or the `ready` event of the respective OptionsWatcher instance.'
+		);
+		this.name = 'UninitializedOptionsWatcherError';
+	}
+}
+
+export class InvariantUninitializedOptionsWatcherError extends Error {
+	constructor() {
+		super('Invariant: OptionsWatcher has not been initialized yet. This should never happen.');
+		this.name = 'InvariantUninitializedOptionsWatcherError';
+	}
+}
+
+export class InvalidValueTypeError extends Error {
+	constructor(keys: string[], value: unknown) {
+		super(
+			`Invalid value type for key ${keys.join('.')}. Expected object, string, array, number, boolean, or undefined. Received ${typeof value}.`
+		);
+		this.name = 'InvalidValueTypeError';
+	}
+}
+
+export class KeyDoesNotExistError extends Error {
+	constructor(keys: string[], key: string) {
+		super(`Cannot set property ${keys.join('.')} as ${key} does not exist.`);
+		this.name = 'KeyDoesNotExistError';
+	}
+}
+
+export class CannotSetPropertyError extends Error {
+	constructor(keys: string[]) {
+		super(`Cannot set property ${keys.join('.')} as parent is not an object.`);
+		this.name = 'CannotSetPropertyError';
+	}
+}
+
+/**
+ * Watches a YAML configuration file for changes and provides methods to access the configuration.
+ *
+ * @emits ready - When the configuration file is initially loaded and values are available
+ * @emits change - When any value in the configuration changes (with key, new value, and full config)
+ * @emits remove - When the configuration file is removed or the extension is removed from the config
+ * @emits error - When an error occurs reading or parsing the file
+ * @emits close - When the watcher is closed
+ */
+export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
+	#filePath: string;
+	#watcher: FSWatcher;
+	#scopedConfig?: ConfigValue;
+	#rootConfig?: Config;
+	#name: string;
+	#logger: Logger;
+	ready: Promise<any[]>;
+
+	constructor(name: string, filePath: string, logger?: Logger) {
+		super();
+		this.#name = name;
+		this.#filePath = filePath;
+		this.#logger = logger || loggerWithTag(name);
+		this.ready = once(this, 'ready');
+		this.#watcher = chokidar
+			.watch(filePath, { persistent: false })
+			.on('add', this.#handleChange.bind(this))
+			.on('change', this.#handleChange.bind(this))
+			.on('error', this.#handleError.bind(this))
+			.on('unlink', this.#handleUnlink.bind(this))
+			.on('ready', this.#handleChange.bind(this));
+	}
+
+	#handleChange() {
+		readFile(this.#filePath, 'utf-8')
+			.then((contents) => {
+				this.#rootConfig = yaml.parse(contents);
+				// If the extension is in the config file
+				if (this.#rootConfig && this.#name in this.#rootConfig) {
+					// If a config object does not exist
+					if (!this.#scopedConfig) {
+						// set it
+						this.#scopedConfig = this.#rootConfig[this.#name];
+						// and emit a ready event
+						this.emit('ready', this.#scopedConfig);
+					} else {
+						// Otherwise, merge the new config with the old config
+						this.#merge(this.#rootConfig[this.#name], this.#scopedConfig);
+					}
+				} else {
+					// Otherwise, if the extension is not in the config file
+					// This means the plugin was removed from the config file
+					if (this.#scopedConfig) {
+						// and a config exists, remove it
+						this.#scopedConfig = undefined;
+						this.emit('remove');
+					}
+					// Otherwise do nothing - the user may add the config back in later
+				}
+			})
+			.catch((error) => {
+				// If the config file does not exist
+				if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+					// And a config already exists, reset it to the default
+					if (this.#rootConfig) {
+						this.#resetConfig();
+						this.emit('remove');
+					} else {
+						// Otherwise, if no config exists, then just set to default and emit ready
+						this.#resetConfig();
+						this.emit('ready');
+					}
+					return;
+				}
+				this.emit('error', error);
+			});
+	}
+
+	#handleError(error: unknown) {
+		this.emit('error', new OptionsWatcherConfigFileError(this.#filePath, error));
+	}
+
+	#handleUnlink(path: string) {
+		this.#logger.warn?.(
+			`Configuration file ${path} was deleted. Reverting to default configuration. Recreate it to restore the options watcher.`
+		);
+		this.#resetConfig();
+		this.emit('remove');
+	}
+
+	#resetConfig() {
+		this.#rootConfig = DEFAULT_CONFIG;
+		this.#scopedConfig = this.#rootConfig[this.#name];
+	}
+
+	/**
+	 * This merge algorithm is best thought off as a diff and overwrite.
+	 * The new config object will completely overwrite the old config object,
+	 * but we need to recursively iterate over the new entries and emit the
+	 * necessary change events.
+	 *
+	 * All events are considered to be a `change`.
+	 */
+	#merge(newConfigValue: ConfigValue, currentConfigValue: ConfigValue, prevKeys: string[] = []) {
+		// First, ensure current and new config values are Config objects (not null, undefined, or a primitive)
+		if (!this.#isConfig(currentConfigValue) || !this.#isConfig(newConfigValue)) {
+			// If either is not a config, then just set as there is no need to diff/merge
+			this.#setValue(prevKeys, newConfigValue);
+			return;
+		}
+
+		// Check for any missing keys (new config has removed keys from current config)
+		for (const key of Object.keys(currentConfigValue)) {
+			if (!(key in newConfigValue)) {
+				this.#setValue(prevKeys.concat(key), undefined);
+			}
+		}
+
+		// Then, iterate of the keys in the new config and check for any changes to the current config
+		for (const [key, newValue] of Object.entries(newConfigValue)) {
+			const keys = prevKeys.concat(key);
+			const currentValue = this.#getValue(keys);
+
+			// If the new value is not the same type as the current value, then no equivalency check is necessary
+			// Just set the value and continue
+			if (
+				typeof newValue !== typeof currentValue ||
+				// one exception to the above rule is if the `currentValue` is being changed from an array to an object or vice versa
+				// Check for this and shortcut as it can be treated as a type change
+				(Array.isArray(newValue) && !Array.isArray(currentValue)) ||
+				(!Array.isArray(newValue) && Array.isArray(currentValue))
+			) {
+				this.#setValue(keys, newValue);
+				continue;
+			}
+
+			// If the new value is an object (non null nor an array), now merge it with the current value
+			if (!Array.isArray(newValue) && typeof newValue === 'object' && newValue !== null) {
+				if (this.#isConfig(currentValue)) {
+					// Now we're sure currentValue is a Config
+					this.#merge(newValue, currentValue, keys);
+				} else {
+					// If currentValue is not a Config, just set newValue
+					this.#setValue(keys, newValue);
+				}
+				continue;
+			}
+
+			if (!isDeepStrictEqual(newValue, currentValue)) {
+				this.#setValue(keys, newValue);
+			}
+		}
+	}
+
+	#isConfig(value: ConfigValue): value is Config {
+		return typeof value === 'object' && value !== null && value !== undefined && !Array.isArray(value);
+	}
+
+	#getValue(keys: string[]): undefined | ConfigValue {
+		let value: ConfigValue = this.#scopedConfig;
+
+		for (const key of keys) {
+			if (value === null || value === undefined || typeof value !== 'object' || !(key in value)) return undefined;
+
+			value = value[key];
+		}
+
+		return cloneDeep(value);
+	}
+
+	#setValue(keys: string[], value: ConfigValue) {
+		// This method is only called by `merge`, which is only called by `changeHandler` if `this.#config` is defined.
+		// So this should never happen, but just in case, throw an error.
+		// If this ever does get triggered:
+		// - Did something else other than `merge` call this method?
+		// - Did the `merge` method get called differently?
+		// - Did the `merge` method become async and the `this.#config` get set to undefined sometime in between?
+		if (!this.#scopedConfig) {
+			throw new InvariantUninitializedOptionsWatcherError();
+		}
+
+		if (!['object', 'string', 'array', 'number', 'boolean', 'undefined'].includes(typeof value)) {
+			throw new InvalidValueTypeError(keys, value);
+		}
+
+		let obj: ConfigValue = this.#scopedConfig;
+
+		for (const key of keys.slice(0, -1)) {
+			if (obj === null || obj === undefined || typeof obj !== 'object' || !(key in obj)) {
+				throw new KeyDoesNotExistError(keys, key);
+			}
+
+			obj = obj[key];
+		}
+
+		if (obj === null || obj === undefined || typeof obj !== 'object') {
+			throw new CannotSetPropertyError(keys);
+		}
+
+		obj[keys[keys.length - 1]] = value;
+
+		this.emit('change', keys, value, this.#scopedConfig);
+	}
+
+	/**
+	 * Closes the underlying file watcher, emits the `close` event, and removes any listeners on the OptionsWatcher instance
+	 */
+	close() {
+		this.#watcher.close();
+
+		this.emit('close');
+
+		this.removeAllListeners();
+
+		return this;
+	}
+
+	/**
+	 * Get a value from the configuration using an array of strings representing the key.
+	 *
+	 * For example, if the configuration is:
+	 * ```yaml
+	 * foo:
+	 *  bar:
+	 *   baz: 42
+	 * ```
+	 * Then `get(['foo','bar','baz'])` will return `42`.
+	 *
+	 * If the key does not exist, `undefined` will be returned.
+	 * @param key an array of strings representing the key.
+	 * @returns
+	 */
+	get(key: string[]): ConfigValue | undefined {
+		return this.#scopedConfig ? this.#getValue(key) : undefined;
+	}
+
+	/**
+	 * Get the entire configuration object.
+	 *
+	 * @returns A deep clone of the entire configuration object.
+	 */
+	getAll(): ConfigValue | undefined {
+		return cloneDeep(this.#scopedConfig);
+	}
+
+	/**
+	 * Get the entire root configuration object from the config file.
+	 */
+	getRoot(): Config | undefined {
+		return this.#rootConfig;
+	}
+
+	// Not sure if we want to enable runtime changes to the config - any changes to the config should be done in the config file.
+	// /**
+	//  * Set a value in the configuration using a dot-separated key. Any existing value can be replaced with any new value, regardless of type.
+	//  *
+	//  * For example, with the configuration:
+	//  *
+	//  * ```yaml
+	//  * foo:
+	//  *  bar:
+	//  *   baz: 42
+	//  * ```
+	//  *
+	//  * The call `set('foo.bar.baz', 'harper')` will set `foo.bar.baz` to `'harper'`.
+	//  *
+	//  * This method will allow you to set new values in the configuration, but it will not generate nested objects.
+	//  *
+	//  * For example, using the configuration above, `set('foo.fuzz', 'buzz')`, will work fine.
+	//  *
+	//  * But `set('foo.x.y', 0)` will throw an error, because it is attempting to set `y` on the non-existent `x`.
+	//  *
+	//  * This method will emit a `change` event when the value is set.
+	//  *
+	//  * @param key Dot-separated key to set the value for.
+	//  * @param value Value to set.
+	//  */
+	// set(key: string, value: any) {
+	// 	this.setValue(key.split('.'), cloneDeep(value));
+	// }
+}

package/components/PluginModule.ts

@@ -0,0 +1,6 @@
+import { Scope } from './Scope';
+
+export interface PluginModule {
+	handleApplication: (scope: Scope) => void | Promise<void>;
+	defaultTimeout?: number;
+}

package/components/Scope.ts

@@ -0,0 +1,329 @@
+import { type Logger } from '../utility/logging/logger.ts';
+import { loggerWithTag } from '../utility/logging/harper_logger.js';
+import { EventEmitter, once } from 'node:events';
+import { databaseEventsEmitter } from '../resources/databases.ts';
+import { server, type Server } from '../server/Server.ts';
+import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts';
+import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts';
+import { resources, type Resources } from '../resources/Resources.ts';
+import type { FileAndURLPathConfig } from './Component.ts';
+import { FilesOption } from './deriveGlobOptions.ts';
+import { requestRestart } from './requestRestart.ts';
+import { ApplicationScope } from './ApplicationScope.ts';
+
+export class MissingDefaultFilesOptionError extends Error {
+	constructor() {
+		super('No default files option exists. Ensure `files` is specified in config.yaml');
+		this.name = 'MissingDefaultFilesOptionError';
+	}
+}
+
+export type ScopeEventsMap = {
+	all: [...args: unknown[]];
+	close: [];
+	error: [error: unknown];
+	ready: [];
+	[record: string]: [...args: unknown[]];
+};
+
+/**
+ * This class is what is passed to the `handleApplication` function of an extension.
+ *
+ * It is imperative that the instance is "ready" before it's passed to the `handleApplication` function
+ * so that the developer can immediately start using `scope.options`, etc.
+ *
+ */
+export class Scope extends EventEmitter<ScopeEventsMap> {
+	#configFilePath: string;
+	#directory: string;
+	#appName: string;
+	#pluginName: string;
+	#entryHandler?: EntryHandler;
+	#entryHandlers: EntryHandler[];
+	#logger: Logger;
+	#pendingInitialLoads: Set<Promise<void>>;
+	applicationScope?: ApplicationScope;
+
+	options: OptionsWatcher;
+	resources?: Resources;
+	server?: Server;
+	ready: Promise<any[]>;
+	databaseEvents: typeof databaseEventsEmitter;
+
+	constructor(
+		appName: string,
+		pluginName: string,
+		directory: string,
+		configFilePath: string,
+		applicationScope: ApplicationScope
+	) {
+		super();
+
+		this.#appName = appName;
+		this.#pluginName = pluginName;
+		this.#directory = directory;
+		this.#configFilePath = configFilePath;
+		this.#logger = logger || loggerWithTag(this.#appName);
+
+		this.databaseEvents = databaseEventsEmitter;
+		this.applicationScope = applicationScope;
+		this.resources = applicationScope?.resources ?? resources;
+		this.server = applicationScope?.server ?? server;
+
+		this.#entryHandlers = [];
+		this.#pendingInitialLoads = new Set();
+
+		this.ready = once(this, 'ready');
+
+		// Create the options instance for the scope immediately
+		this.options = new OptionsWatcher(pluginName, configFilePath, this.#logger)
+			.on('error', this.#handleError.bind(this))
+			.on('change', this.#optionsWatcherChangeListener.bind(this)())
+			.on('ready', this.#handleOptionsWatcherReady.bind(this));
+	}
+
+	get logger(): Logger {
+		return this.#logger;
+	}
+
+	get appName(): string {
+		return this.#appName;
+	}
+
+	get pluginName(): string {
+		return this.#pluginName;
+	}
+
+	get directory(): string {
+		return this.#directory;
+	}
+
+	get configFilePath(): string {
+		return this.#configFilePath;
+	}
+
+	#handleOptionsWatcherReady(): void {
+		// This previously created the default entry handler immediately, but now we wait for the user to call `handleEntry`
+		// The issue was that since the component loader was awaiting `scope.ready()` and then calling `pluginModule.handleApplication(scope)`,
+		// the default entry handler could start receiving events before the plugin provided its own handler.
+		// We could make the user call `await scope.ready()` in their `handleApplication` function, but that could lead to the same issue and it'd
+		// be harder for the user to understand why.
+
+		this.emit('ready');
+	}
+
+	#handleError(error: unknown): void {
+		this.emit('error', error);
+	}
+
+	close() {
+		for (const entryHandler of this.#entryHandlers) {
+			entryHandler.close();
+		}
+
+		this.options.close();
+
+		this.emit('close');
+
+		this.removeAllListeners();
+
+		return this;
+	}
+
+	#createEntryHandler(config: FilesOption | FileAndURLPathConfig): EntryHandler {
+		const entryHandler = new EntryHandler(this.#pluginName, this.#directory, config, this.#logger)
+			.on('error', this.#handleError.bind(this))
+			.on('add', this.#defaultEntryHandlerListener('add'))
+			.on('change', this.#defaultEntryHandlerListener('change'))
+			.on('unlink', this.#defaultEntryHandlerListener('unlink'))
+			.on('addDir', this.#defaultEntryHandlerListener('addDir'))
+			.on('unlinkDir', this.#defaultEntryHandlerListener('unlinkDir'));
+
+		this.#entryHandlers.push(entryHandler);
+
+		return entryHandler;
+	}
+
+	#defaultEntryHandlerListener(event: keyof EntryHandlerEventMap) {
+		// eslint-disable-next-line @typescript-eslint/no-this-alias
+		const scope = this;
+		return function (this: EntryHandler) {
+			if (this.listenerCount('all') > 0 || this.listenerCount(event) > 1) {
+				return;
+			}
+
+			scope.requestRestart();
+		};
+	}
+
+	#optionsWatcherChangeListener() {
+		// eslint-disable-next-line @typescript-eslint/no-this-alias
+		const scope = this;
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		return function handleOptionsWatcherChange(
+			this: OptionsWatcher,
+			// eslint-disable-next-line @typescript-eslint/no-unused-vars
+			...[key, _, config]: OptionsWatcherEventMap['change']
+		) {
+			if (key[0] === 'files' || key[0] === 'urlPath') {
+				// TODO: validate options
+
+				// If not entry handler exists then likely the config did not have `files` initially
+				// Now, it does, so create a default entry handler.
+				if (!scope.#entryHandler) {
+					scope.#entryHandler = scope.#createEntryHandler(config as FileAndURLPathConfig);
+					return;
+				}
+
+				// Otherwise, if an entry handler exists, update it with the new config
+				scope.#entryHandler.update(config as FileAndURLPathConfig);
+
+				return;
+			}
+
+			// If the user isn't handling option changes, request a restart
+			if (this.listenerCount('change') > 1) {
+				return;
+			}
+
+			scope.#logger.debug?.(`Options changed: ${key.join('.')}, requesting restart`);
+			scope.requestRestart();
+		};
+	}
+
+	#getFilesOption(): FileAndURLPathConfig | undefined {
+		const config = this.options.getAll();
+		if (
+			config &&
+			typeof config === 'object' &&
+			config !== null &&
+			!Array.isArray(config) &&
+			'files' in config /*&& validate config.files*/
+		) {
+			return {
+				files: config.files as FilesOption,
+				urlPath: config.urlPath as string | undefined,
+			};
+		}
+		return undefined;
+	}
+
+	handleEntry(files: FilesOption | FileAndURLPathConfig, handler: onEntryEventHandler): EntryHandler;
+	handleEntry(handler: onEntryEventHandler): EntryHandler;
+	handleEntry(): EntryHandler;
+	handleEntry(
+		filesOrHandler?: FilesOption | FileAndURLPathConfig | onEntryEventHandler,
+		handler?: onEntryEventHandler
+	): EntryHandler {
+		let entryHandler: EntryHandler;
+
+		// Helper to wrap async handlers for tracking
+		const wrapHandler = (
+			targetEntryHandler: EntryHandler,
+			entryEventHandler: onEntryEventHandler
+		): onEntryEventHandler => {
+			const pendingOperations = new Set<Promise<void>>();
+
+			const wrapped: onEntryEventHandler = (entry) => {
+				const result = entryEventHandler(entry);
+				if (result instanceof Promise) {
+					const tracked = result
+						.catch((error) => {
+							this.#logger.error?.('Error in async entry handler:', error);
+							this.#handleError(error);
+							throw error;
+						})
+						.finally(() => pendingOperations.delete(tracked));
+					pendingOperations.add(tracked);
+				}
+			};
+
+			// When the entry handler's initial scan completes, wait for all pending async operations
+			const initialLoadPromise = once(targetEntryHandler, 'ready').then(async () => {
+				if (pendingOperations.size > 0) {
+					await Promise.all(pendingOperations);
+				}
+				targetEntryHandler.emit('initialLoadComplete');
+			});
+
+			// Track this promise so the component loader can await it
+			this.#pendingInitialLoads.add(initialLoadPromise);
+			initialLoadPromise.finally(() => this.#pendingInitialLoads.delete(initialLoadPromise));
+
+			return wrapped;
+		};
+
+		// No arguments
+		if (filesOrHandler === undefined) {
+			// If entry handler already exists, return it
+			if (this.#entryHandler) {
+				entryHandler = this.#entryHandler;
+			} else {
+				// Otherwise, try to create a default entry handler using the files option
+				const filesOption = this.#getFilesOption();
+				if (filesOption) {
+					this.#entryHandler = this.#createEntryHandler(filesOption);
+					entryHandler = this.#entryHandler;
+				} else {
+					this.emit('error', new MissingDefaultFilesOptionError());
+					return;
+				}
+			}
+		}
+		// Provided a handler function
+		else if (typeof filesOrHandler === 'function') {
+			// If an entry handler already exists, return it with the handler attached
+			if (this.#entryHandler) {
+				entryHandler = this.#entryHandler;
+			} else {
+				// Otherwise, try to create a default entry handler using the files option
+				const filesOption = this.#getFilesOption();
+				if (filesOption) {
+					this.#entryHandler = this.#createEntryHandler(filesOption);
+					entryHandler = this.#entryHandler;
+				} else {
+					this.emit('error', new MissingDefaultFilesOptionError());
+					return;
+				}
+			}
+
+			const wrappedHandler = wrapHandler(entryHandler, filesOrHandler);
+			entryHandler.on('all', wrappedHandler);
+		}
+		// otherwise this is a custom config entry handler
+		else {
+			entryHandler = this.#createEntryHandler(filesOrHandler);
+			if (handler) {
+				const wrappedHandler = wrapHandler(entryHandler, handler);
+				entryHandler.on('all', wrappedHandler);
+			}
+		}
+
+		return entryHandler;
+	}
+
+	requestRestart() {
+		this.#logger.debug?.(`Restart requested from ${this.#pluginName} scope for ${this.#appName}`);
+		requestRestart();
+	}
+
+	/**
+	 * Wait for all entry handlers' initial loads to complete.
+	 * This includes waiting for any async operations in entry handler callbacks.
+	 * Called by the component loader after handleApplication completes.
+	 */
+	async waitForInitialLoads(): Promise<void> {
+		if (this.#pendingInitialLoads.size > 0) {
+			await Promise.all(this.#pendingInitialLoads);
+		}
+	}
+
+	/**
+	 * Import a file into the scope's sandbox.
+	 * @param filePath - The path of the file to import.
+	 * @returns A promise that resolves with the imported module or value.
+	 */
+	async import(filePath: string): Promise<unknown> {
+		return this.applicationScope ? this.applicationScope.import(filePath) : import(filePath);
+	}
+}