@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/components/componentLoader.ts

@@ -12,7 +12,7 @@ import {
 import { join, basename, dirname } from 'node:path';
 import { isMainThread } from 'node:worker_threads';
 import { parseDocument } from 'yaml';
-import * as env from '../utility/environment/environmentManager.js';
+import * as env from '../utility/environment/environmentManager.ts';
 import { PACKAGE_ROOT } from '../utility/packageUtils.js';
 import { CONFIG_PARAMS, HDB_ROOT_DIR_NAME, ITC_EVENT_TYPES } from '../utility/hdbTerms.ts';
 import * as graphqlHandler from '../resources/graphql.ts';
@@ -21,28 +21,27 @@ import * as roles from '../resources/roles.ts';
 import * as jsHandler from '../resources/jsResource.ts';
 import * as login from '../resources/login.ts';
 import * as REST from '../server/REST.ts';
-import * as fastifyRoutesHandler from '../server/fastifyRoutes.ts';
 import * as staticFiles from '../server/static.ts';
 import * as loadEnv from '../resources/loadEnv.ts';
-import harperLogger from '../utility/logging/harper_logger.js';
+import harperLogger from '../utility/logging/harper_logger.ts';
 import * as dataLoader from '../resources/dataLoader.ts';
-import { watchDir, getWorkerIndex } from '../server/threads/manageThreads.js';
+import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.js';
+import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts';
 import { scopedImport } from '../security/jsLoader.ts';
 import { server } from '../server/Server.ts';
 import { Resources } from '../resources/Resources.ts';
 import { table } from '../resources/databases.ts';
-import { startSocketServer } from '../server/threads/socketRouter.ts';
-import { getHdbBasePath } from '../utility/environment/environmentManager.js';
-import * as operationsServer from '../server/operationsServer.ts';
+import { getHdbBasePath } from '../utility/environment/environmentManager.ts';
 import * as auth from '../security/auth.ts';
 import * as mqtt from '../server/mqtt.ts';
 import { getConfigObj, getConfigPath } from '../config/configUtils.js';
-import { createReuseportFd } from '../server/serverHelpers/Request.ts';
+import { bootstrapModels } from '../resources/models/bootstrap.ts';
 import { ErrorResource } from '../resources/ErrorResource.ts';
 import { Scope } from './Scope.ts';
 import { ApplicationScope } from './ApplicationScope.ts';
 import { ComponentV1, processResourceExtensionComponent } from './ComponentV1.ts';
 import * as httpComponent from '../server/http.ts';
+import * as mcpComponent from './mcp/index.ts';
 import { Status } from '../server/status/index.ts';
 import { lifecycle as componentLifecycle } from './status/index.ts';
 import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts';
@@ -78,6 +77,7 @@ export function loadComponentDirectories(loadedPluginModules?: Map<any, any>, lo
 	}
 	const hdbAppFolder = process.env.RUN_HDB_APP;
 	if (hdbAppFolder) {
+		if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder);
 		cfsLoaded.push(
 			loadComponent(hdbAppFolder, resources, hdbAppFolder, {
 				isRoot: false,
@@ -91,17 +91,18 @@ export function loadComponentDirectories(loadedPluginModules?: Map<any, any>, lo
 	});
 }
 
-export const TRUSTED_RESOURCE_PLUGINS = {
+export const TRUSTED_RESOURCE_PLUGINS: any = {
 	REST, // for backwards compatibility with older configs
 	rest: REST,
 	graphql: graphqlQueryHandler,
 	graphqlSchema: graphqlHandler,
 	roles,
 	jsResource: jsHandler,
-	fastifyRoutes: fastifyRoutesHandler,
+	get fastifyRoutes() {
+		return require('../server/fastifyRoutes');
+	},
 	login,
 	static: staticFiles,
-	operationsApi: operationsServer,
 	customFunctions: {},
 	http: httpComponent,
 	authentication: auth,
@@ -109,11 +110,28 @@ export const TRUSTED_RESOURCE_PLUGINS = {
 	loadEnv,
 	logging: harperLogger,
 	dataLoader,
+	mcp: mcpComponent,
 	/*
 	static: ...
 	login: ...
 	 */
 };
+if (isMainThread) {
+	TRUSTED_RESOURCE_PLUGINS.operationsApi = require('../server/operationsServer');
+	// Built-in agent component (#626). Only loads if the root config carries an `agent:` block;
+	// the block's `enabled: false` default keeps it inert even when the key is present.
+	TRUSTED_RESOURCE_PLUGINS.agent = require('../agent/agent');
+} else {
+	// The HTTP operations API itself only binds in the main thread, but worker threads still
+	// dispatch operations — most notably, the replication WebSocket handler in workers receives
+	// inter-node operations like `add_node_back` and calls `server.operation(...)`. That requires
+	// `server.operation` / `server.registerOperation` to be wired up here too, and the operation
+	// function map to be initialized, BEFORE component plugins (replication, etc.) load and call
+	// `server.registerOperation?.({...})` at their module top level. Requiring serverUtilities
+	// directly (rather than the full operationsServer) avoids binding the fastify HTTP layer in
+	// workers while still installing the dispatch machinery.
+	require('../server/serverHelpers/serverUtilities');
+}
 
 for (const { name, packageIdentifier } of getEnvBuiltInComponents()) {
 	TRUSTED_RESOURCE_PLUGINS[name] = packageIdentifier;
@@ -121,7 +139,6 @@ for (const { name, packageIdentifier } of getEnvBuiltInComponents()) {
 
 const BUILT_INS = Object.keys(TRUSTED_RESOURCE_PLUGINS);
 
-const portsStarted = [];
 export const loadedPaths = new Map();
 let errorReporter;
 export function setErrorReporter(reporter) {
@@ -304,6 +321,22 @@ export async function loadComponent(
 		}
 		applicationScope.config ??= config;
 
+		// For non-root components with empty/null config (e.g., comment-only YAML),
+		// don't synthesize DEFAULT_CONFIG. Empty config means the component has nothing
+		// to load; falling back to DEFAULT_CONFIG would cause OptionsWatcher to wait
+		// forever for plugins that the file doesn't actually declare.
+		if (isRoot) config ??= DEFAULT_CONFIG;
+		if (!config) {
+			// Empty/comment-only config file on a non-root component: nothing to load.
+			return undefined;
+		}
+
+		// #629 (Phase 2 of #510): populate the model-backend registry from the root
+		// config's `models:` block before any user `handleApplication(scope)` runs,
+		// so `scope.models.embed(...)` works from app-init code as well as Resource
+		// methods. Per-entry errors are logged and skipped by `bootstrapModels`.
+		if (isRoot) bootstrapModels(config);
+
 		if (!isRoot) {
 			try {
 				await symlinkHarperModule(componentDirectory);
@@ -427,7 +460,14 @@ export async function loadComponent(
 
 				// New Plugin API (`handleApplication`)
 				if (resources.isWorker && extensionModule.handleApplication) {
-					const scope = new Scope(appName || 'harper', componentName, componentDirectory, configPath, applicationScope);
+					const scope = new Scope(
+						appName || 'harper',
+						componentName,
+						componentDirectory,
+						configPath,
+						applicationScope,
+						origin
+					);
 
 					onMessageByType(ITC_EVENT_TYPES.SHUTDOWN, () => scope.close());
 
@@ -465,22 +505,8 @@ export async function loadComponent(
 							...componentConfig,
 						})) || extensionModule;
 					if (isRoot && network) {
-						for (const possiblePort of [port, securePort]) {
-							try {
-								if (+possiblePort && !portsStarted.includes(possiblePort)) {
-									const sessionAffinity = env.get(CONFIG_PARAMS.HTTP_SESSIONAFFINITY);
-									if (sessionAffinity)
-										harperLogger.warn('Session affinity is not recommended and may cause memory leaks');
-									if (sessionAffinity || !createReuseportFd) {
-										// if there is a TCP port associated with the plugin, we set up the routing on the main thread for it
-										portsStarted.push(possiblePort);
-										startSocketServer(possiblePort, sessionAffinity);
-									}
-								}
-							} catch (error) {
-								console.error('Error listening on socket', possiblePort, error, componentName);
-							}
-						}
+						if (env.get(CONFIG_PARAMS.HTTP_SESSIONAFFINITY))
+							harperLogger.warn('Session affinity is not supported and will be ignored');
 					}
 				}
 				if (resources.isWorker)
@@ -527,10 +553,16 @@ export async function loadComponent(
 		}
 
 		compName = parentCompName;
-		// Auto restart threads on changes to any app folder. TODO: Make this configurable
 		if (isMainThread && !watchesSetup && autoReload) {
-			watchDir(componentDirectory, async () => {
-				return loadComponentDirectories(); // return the promise
+			let debounceTimer: ReturnType<typeof setTimeout> | null = null;
+			subscribeToRestartRequests(() => {
+				if (debounceTimer) clearTimeout(debounceTimer);
+				debounceTimer = setTimeout(async () => {
+					debounceTimer = null;
+					resetRestartNeeded();
+					await loadComponentDirectories();
+					restartWorkers();
+				}, 500);
 			});
 		}
 		if ((config.extensionModule || config.pluginModule) && (!isMainThread || config.runOnMainThread)) {

package/components/deployLifecycle.ts

@@ -0,0 +1,161 @@
+// Cross-thread deploy lifecycle events.
+//
+// Component deploys (extract + npm install) write into the component directory,
+// which is exactly what every Scope's EntryHandler is watching. Without
+// coordination this drives a restart-request storm — each file change fires
+// scope.requestRestart(), which closes and recreates every component watcher
+// through componentLoader, briefly doubling inotify-handle occupancy and
+// occasionally exhausting the OS limit (harper#488).
+//
+// This module solves that by broadcasting a structured "deploy:start" /
+// "deploy:end" signal to every Harper thread. Each thread's deploy emitter
+// fires locally so Scopes (and any plugin subscribers) can react: Scopes close
+// their EntryHandlers on start, recreate them on end. The result is that
+// during a deploy, no watcher fires events for the deployed component, and
+// after the deploy the recreated watcher emits one fresh add per current file
+// — collapsed by the existing restart debounce into a single restart.
+//
+// State sharing across threads is intentionally narrow: the local Set of
+// in-flight component names is rebuilt from the broadcast stream. Plugins that
+// need to gate their own work on deploy progress import `deployLifecycle` and
+// listen to the events directly.
+
+import { EventEmitter } from 'node:events';
+import { isMainThread } from 'node:worker_threads';
+import { broadcast, broadcastWithAcknowledgement, onMessageByType } from '../server/threads/manageThreads.js';
+
+const DEPLOY_LIFECYCLE_MSG = 'harper:deploy:lifecycle';
+
+export type DeployPhase = 'start' | 'end';
+export type DeployLifecycleEvent = { name: string; phase: DeployPhase };
+
+type DeployLifecycleEventsMap = {
+	'deploy:start': [componentName: string];
+	'deploy:end': [componentName: string];
+};
+
+class DeployLifecycle extends EventEmitter<DeployLifecycleEventsMap> {
+	// Ref-counts in-flight deploys per component. Concurrent deploys of the
+	// same name (rare, but possible if an operator queues two before the first
+	// resolves) overlap: 0→1 fires deploy:start, 1→0 fires deploy:end. Any
+	// intermediate increment/decrement is silent so watchers aren't toggled
+	// out from under an active deploy by a peer ending early.
+	#inFlight = new Map<string, number>();
+
+	isDeployInFlight(componentName: string): boolean {
+		return (this.#inFlight.get(componentName) ?? 0) > 0;
+	}
+
+	// Process a deploy lifecycle event in-process. Called both from the
+	// broadcast receiver (for events originating on another thread) and from
+	// the broadcaster (so the originating thread also reacts locally).
+	_handle(event: DeployLifecycleEvent): void {
+		const current = this.#inFlight.get(event.name) ?? 0;
+		if (event.phase === 'start') {
+			this.#inFlight.set(event.name, current + 1);
+			if (current === 0) this.emit('deploy:start', event.name);
+		} else {
+			const next = Math.max(0, current - 1);
+			if (next === 0) {
+				this.#inFlight.delete(event.name);
+				if (current > 0) this.emit('deploy:end', event.name);
+			} else {
+				this.#inFlight.set(event.name, next);
+			}
+		}
+	}
+
+	// Test-only: clear in-flight state without firing events.
+	_clearForTests(): void {
+		this.#inFlight.clear();
+	}
+}
+
+export const deployLifecycle = new DeployLifecycle();
+
+let receiverInstalled = false;
+function ensureReceiver() {
+	if (receiverInstalled) return;
+	receiverInstalled = true;
+	onMessageByType(DEPLOY_LIFECYCLE_MSG, (msg: { type: string; event: DeployLifecycleEvent }) => {
+		deployLifecycle._handle(msg.event);
+	});
+}
+
+// Install the cross-thread receiver at module load. Receiving threads (workers)
+// don't call the broadcast helpers themselves but must still react to deploy
+// events originating on the main thread — without this, a deploy on main would
+// suppress only the main thread's watchers and the worker watchers would keep
+// firing restart storms (harper#488).
+ensureReceiver();
+
+/**
+ * Announce the start of a deploy for `componentName`. Awaits acknowledgement
+ * from every worker so the caller can rely on all watchers being suppressed
+ * before writing into the component directory.
+ *
+ * Ref-counted via DeployLifecycle so overlapping deploys of the same
+ * component compose correctly (each call must be paired with exactly one
+ * broadcastDeployEnd).
+ */
+export async function broadcastDeployStart(componentName: string): Promise<void> {
+	ensureReceiver();
+	const event: DeployLifecycleEvent = { name: componentName, phase: 'start' };
+	deployLifecycle._handle(event); // local thread first
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	try {
+		// broadcastWithAcknowledgement only resolves once every peer has processed
+		// the message, which is what we want before we start touching files. From
+		// a worker, this still reaches main + sibling workers; from main it reaches
+		// all workers.
+		//
+		// Race against a 5s timeout so an unresponsive worker can't hang the
+		// deploy indefinitely — the deploy is meant to be best-effort coordinated,
+		// not gated on every worker acknowledging.
+		const timeout = new Promise<void>((_resolve, reject) => {
+			timer = setTimeout(() => reject(new Error('Broadcast acknowledgement timed out')), 5000);
+			timer.unref?.();
+		});
+		await Promise.race([broadcastWithAcknowledgement({ type: DEPLOY_LIFECYCLE_MSG, event }), timeout]);
+	} catch (error) {
+		// A broadcast failure here is non-fatal: the deploy can still proceed,
+		// the worst case is a transient restart storm. Don't block the deploy.
+		// (Errors are already surfaced through the existing logger by manageThreads.)
+		void error;
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
+/**
+ * Announce the end of a deploy for `componentName`. Fire-and-forget — the
+ * caller (the deploy operation) is done by this point and doesn't need to
+ * wait for every worker to reopen its watcher.
+ *
+ * Must be called exactly once per matching broadcastDeployStart, even if the
+ * deploy errored — typically from a `finally` block.
+ */
+export function broadcastDeployEnd(componentName: string): void {
+	ensureReceiver();
+	const event: DeployLifecycleEvent = { name: componentName, phase: 'end' };
+	deployLifecycle._handle(event);
+	try {
+		void broadcast({ type: DEPLOY_LIFECYCLE_MSG, event }, false);
+	} catch (error) {
+		void error;
+	}
+}
+
+// Tests need to reset module state between cases. We deliberately do NOT reset
+// `receiverInstalled`: manageThreads.onMessageByType has no deregistration API,
+// so flipping the flag would let a subsequent ensureReceiver() pile a second
+// listener and double-increment the refcount on every broadcast.
+export function _resetForTests(): void {
+	deployLifecycle.removeAllListeners();
+	deployLifecycle._clearForTests();
+}
+
+// Marker so callers (e.g. Application.ts) can tell whether they're running in
+// a context where broadcasting will reach peers — during startup before
+// threads are wired up, broadcasts are no-ops but still safe to call.
+export const supportsCrossThreadBroadcast = isMainThread;

package/components/deploymentOperations.ts

@@ -0,0 +1,173 @@
+'use strict';
+
+// Read-side operations against system.hdb_deployment.
+// Write-side lives in deploymentRecorder.ts; this module only reads.
+
+import { databases } from '../resources/databases.ts';
+import * as terms from '../utility/hdbTerms.ts';
+import { ClientError } from '../utility/errors/hdbError.ts';
+import { getActiveEmitter } from './deploymentRecorder.ts';
+import type { ProgressEmitter } from '../server/serverHelpers/progressEmitter.ts';
+
+const DEPLOYMENT_TABLE = terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME;
+const TERMINAL_STATUSES = new Set(['success', 'failed', 'rolled_back']);
+
+interface ListRequest {
+	project?: string;
+	status?: string;
+	since?: number;
+	until?: number;
+	limit?: number;
+	offset?: number;
+}
+
+interface GetRequest {
+	deployment_id: string;
+	// Set by serverHandlers.js when the client asks for `Accept: text/event-stream`.
+	progress?: ProgressEmitter;
+}
+
+function deploymentTable() {
+	const table = (databases as any).system?.[DEPLOYMENT_TABLE];
+	if (!table) {
+		throw new ClientError(
+			`Deployment tracking is not initialized on this node (system.${DEPLOYMENT_TABLE} missing). ` +
+				`Run upgrade or restart the server to provision the table.`
+		);
+	}
+	return table;
+}
+
+// Strip the blob attribute from a row; the bytes never travel over the operations API.
+// A separate get_deployment_payload operation streams the raw bytes when callers need them.
+function stripBlob(row: any): any {
+	if (!row || typeof row !== 'object') return row;
+	const { payload_blob, ...rest } = row;
+	rest.payload_blob_present = payload_blob != null;
+	return rest;
+}
+
+export async function handleListDeployments(req: ListRequest = {}): Promise<{ deployments: any[]; total: number }> {
+	const table = deploymentTable();
+	const conditions: any[] = [];
+	if (req.project) conditions.push({ attribute: 'project', value: req.project });
+	if (req.status) conditions.push({ attribute: 'status', value: req.status });
+	if (req.since != null)
+		conditions.push({ attribute: 'started_at', value: req.since, comparator: 'greater_than_equal' });
+	if (req.until != null) conditions.push({ attribute: 'started_at', value: req.until, comparator: 'less_than_equal' });
+
+	const collected: any[] = [];
+	for await (const row of table.search(conditions)) {
+		collected.push(stripBlob(row));
+	}
+	// Newest first by started_at; ties broken by deployment_id for stability.
+	collected.sort(
+		(a, b) => (b.started_at ?? 0) - (a.started_at ?? 0) || String(a.deployment_id).localeCompare(b.deployment_id)
+	);
+
+	const total = collected.length;
+	const offset = Math.max(0, req.offset ?? 0);
+	const limit = req.limit != null ? Math.max(0, req.limit) : collected.length;
+	return { deployments: collected.slice(offset, offset + limit), total };
+}
+
+export async function handleGetDeployment(req: GetRequest): Promise<any> {
+	if (!req || !req.deployment_id) {
+		throw new ClientError(`'deployment_id' is required`);
+	}
+	const table = deploymentTable();
+	const row = await table.get(req.deployment_id);
+	if (!row) {
+		throw new ClientError(`No deployment found with id '${req.deployment_id}'`);
+	}
+
+	// SSE content-negotiated branch — when serverHandlers.js detects
+	// `Accept: text/event-stream` it attaches a ProgressEmitter as req.progress and wraps
+	// our return as the operation's final SSE event. We replay event_log on connect, then
+	// tail the deployment's live emitter (if it's still running on this node) until it
+	// reaches a terminal status. The final return value becomes the SSE `done` event.
+	if (req.progress && typeof (req.progress as any).emit === 'function') {
+		const sse = req.progress;
+		const liveEmitter = getActiveEmitter(req.deployment_id);
+
+		// Subscribe FIRST, before reading the row, so any event the recorder emits between
+		// "now" and the moment we finish reading the historical log still lands. Buffer
+		// those live events; dedupe by recording the timestamp of the last replayed entry
+		// and skipping any live event whose timestamp is <= that. Forward everything else.
+		let lastReplayedTs = 0;
+		let resolveLive: (() => void) | null = null;
+		let liveDone = false;
+		const liveBuffer: Array<{ t: number; event: { event: string; data: unknown } }> = [];
+		const forwardLive = (e: { event: string; data: unknown }) => {
+			sse.emit(e.event, e.data);
+			// A terminal signal — either explicit success/error event from the lifecycle, or
+			// the recorder's `_recorder_finished` sentinel emitted before it unsubscribes.
+			const isTerminalEvent =
+				e.event === '_recorder_finished' ||
+				e.event === 'error' ||
+				(e.event === 'phase' &&
+					e.data &&
+					typeof e.data === 'object' &&
+					(e.data as { phase?: string }).phase === 'success');
+			if (isTerminalEvent && !liveDone) {
+				liveDone = true;
+				resolveLive?.();
+			}
+		};
+		const unsubscribe = liveEmitter
+			? liveEmitter.subscribe((event) => {
+					const t = Date.now();
+					if (resolveLive) {
+						// Replay phase finished; we own the live forward path now.
+						if (t > lastReplayedTs) forwardLive(event);
+					} else {
+						// Still replaying; stash for dedup-and-forward once replay completes.
+						liveBuffer.push({ t, event });
+					}
+				})
+			: null;
+
+		try {
+			for (const entry of row.event_log ?? []) {
+				sse.emit(entry.event, entry.data);
+				if (typeof entry.t === 'number') lastReplayedTs = Math.max(lastReplayedTs, entry.t);
+			}
+
+			if (liveEmitter && !TERMINAL_STATUSES.has(row.status)) {
+				await new Promise<void>((resolve) => {
+					resolveLive = resolve;
+					// Flush anything that arrived during replay, filtering to events the replay missed.
+					for (const buffered of liveBuffer) {
+						if (buffered.t > lastReplayedTs) forwardLive(buffered.event);
+					}
+					if (liveDone) resolve();
+					// Safety net — if the in-memory emitter is dropped (recorder finished or
+					// the process recycled) before signaling, poll the row's status as a
+					// fallback so the client never hangs indefinitely.
+					const pollTimer = setInterval(async () => {
+						if (liveDone) {
+							clearInterval(pollTimer);
+							return;
+						}
+						const live = getActiveEmitter(req.deployment_id);
+						if (!live || live !== liveEmitter) {
+							clearInterval(pollTimer);
+							const latest = await table.get(req.deployment_id);
+							if (latest && TERMINAL_STATUSES.has(latest.status) && !liveDone) {
+								liveDone = true;
+								resolve();
+							}
+						}
+					}, 500);
+				});
+			}
+			// Re-read the row so the final SSE payload reflects the post-deploy state.
+			const finalRow = await table.get(req.deployment_id);
+			return stripBlob(finalRow ?? row);
+		} finally {
+			unsubscribe?.();
+		}
+	}
+
+	return stripBlob(row);
+}