@harperfast/harper 5.2.0 → 5.2.1

package/components/RuntimeModuleTracker.ts

@@ -0,0 +1,189 @@
+import { createHash } from 'node:crypto';
+import { lstatSync, readFileSync, readlinkSync } from 'node:fs';
+import { lstat, readFile, readlink } from 'node:fs/promises';
+import { createRequire, Module } from 'node:module';
+import { dirname, isAbsolute, relative, resolve, sep } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+type Source = string | Buffer;
+
+const MODULE_COMPARE_CONCURRENCY = 16;
+const RESOLUTION_EXTENSIONS = ['', '.js', '.json', '.node', '.ts', '.tsx', '.cjs', '.mjs'];
+
+type Resolution = {
+	resolvedUrl: string;
+	higherPriorityCandidates?: Map<string, string>;
+};
+
+export class RuntimeModuleTracker {
+	#getRoot: () => string | undefined;
+	#modules = new Map<string, string>();
+	#resolutions = new Map<string, Resolution>();
+	#nativeRuntime = false;
+	#deployInFlight = false;
+	#loadedDuringDeploy = false;
+	#comparison?: Promise<boolean>;
+
+	constructor(getRoot: () => string | undefined) {
+		this.#getRoot = getRoot;
+	}
+
+	beginDeploy(): void {
+		if (this.#deployInFlight) return;
+		this.#deployInFlight = true;
+		this.#loadedDuringDeploy = false;
+		this.#comparison = undefined;
+	}
+
+	markNativeRuntime(): void {
+		this.#nativeRuntime = true;
+		if (this.#deployInFlight) this.#loadedDuringDeploy = true;
+	}
+
+	recordModule(moduleUrl: string, source: Source): void {
+		const modulePath = this.#localPath(moduleUrl);
+		if (!modulePath) return;
+		if (!this.#modules.has(modulePath)) this.#modules.set(modulePath, digest(source));
+		if (this.#deployInFlight) this.#loadedDuringDeploy = true;
+	}
+
+	recordResolution(specifier: string, referrer: string, resolvedUrl: string): void {
+		const referrerPath = this.#localPath(referrer);
+		const resolvedPath = this.#localPath(resolvedUrl);
+		if (!referrerPath || !resolvedPath) return;
+		const key = `${referrerPath}\0${specifier}`;
+		if (!this.#resolutions.has(key)) {
+			const candidates = higherPriorityResolutionCandidates(specifier, referrerPath, resolvedPath);
+			this.#resolutions.set(key, {
+				resolvedUrl,
+				higherPriorityCandidates: candidates
+					? new Map(candidates.map((candidate) => [candidate, candidateState(candidate)]))
+					: undefined,
+			});
+			if (this.#deployInFlight) this.#loadedDuringDeploy = true;
+		}
+	}
+
+	finishDeploy(): Promise<boolean> {
+		if (!this.#deployInFlight) return this.#comparison ?? Promise.resolve(false);
+		this.#deployInFlight = false;
+		this.#comparison = this.#compare();
+		return this.#comparison;
+	}
+
+	async #compare(): Promise<boolean> {
+		if (this.#nativeRuntime || this.#loadedDuringDeploy) return true;
+		const modules = [...this.#modules];
+		for (let start = 0; start < modules.length; start += MODULE_COMPARE_CONCURRENCY) {
+			const changed = await Promise.all(
+				modules.slice(start, start + MODULE_COMPARE_CONCURRENCY).map(async ([modulePath, previousDigest]) => {
+					try {
+						return digest(await readFile(modulePath)) !== previousDigest;
+					} catch {
+						return true;
+					}
+				})
+			);
+			if (changed.includes(true)) return true;
+		}
+		for (const [key, resolution] of this.#resolutions) {
+			const separator = key.indexOf('\0');
+			const referrerPath = key.slice(0, separator);
+			const specifier = key.slice(separator + 1);
+			if (
+				resolution.higherPriorityCandidates &&
+				(
+					await Promise.all(
+						[...resolution.higherPriorityCandidates].map(
+							async ([candidate, previousState]) => (await candidateStateAsync(candidate)) !== previousState
+						)
+					)
+				).includes(true)
+			)
+				return true;
+			try {
+				// Node does not invalidate this cache when a component tree is replaced in place.
+				if (!invalidateResolutionCache(specifier, referrerPath, resolution.resolvedUrl)) return true;
+				const resolved = createRequire(pathToFileURL(referrerPath)).resolve(specifier);
+				const resolvedUrl = isAbsolute(resolved) ? pathToFileURL(resolved).toString() : resolved;
+				if (resolvedUrl !== resolution.resolvedUrl) return true;
+			} catch {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	#localPath(moduleUrl: string): string | undefined {
+		if (!moduleUrl.startsWith('file:')) return;
+		const root = this.#getRoot();
+		if (!root) return;
+		const modulePath = fileURLToPath(moduleUrl);
+		const relativePath = relative(resolve(root), modulePath);
+		if (
+			relativePath === '' ||
+			(!relativePath.startsWith(`..${sep}`) && relativePath !== '..' && !isAbsolute(relativePath))
+		)
+			return modulePath;
+	}
+}
+
+function higherPriorityResolutionCandidates(
+	specifier: string,
+	referrerPath: string,
+	resolvedPath: string
+): string[] | undefined {
+	if (!specifier.startsWith('.')) return;
+	const basePath = resolve(dirname(referrerPath), specifier);
+	const candidates = [...new Set(RESOLUTION_EXTENSIONS.map((extension) => basePath + extension))];
+	candidates.push(resolve(basePath, 'package.json'));
+	for (const extension of RESOLUTION_EXTENSIONS.slice(1)) candidates.push(resolve(basePath, `index${extension}`));
+	const resolvedIndex = candidates.indexOf(resolvedPath);
+	if (resolvedIndex === -1) return;
+	return candidates.slice(0, resolvedIndex);
+}
+
+function candidateState(path: string): string {
+	try {
+		const stats = lstatSync(path);
+		if (stats.isSymbolicLink()) return `link:${readlinkSync(path)}`;
+		if (stats.isDirectory()) return 'directory';
+		if (!stats.isFile()) return 'other';
+		return path.endsWith('package.json') ? `file:${digest(readFileSync(path))}` : 'file';
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 'missing';
+		return `error:${(error as NodeJS.ErrnoException).code ?? 'unknown'}`;
+	}
+}
+
+async function candidateStateAsync(path: string): Promise<string> {
+	try {
+		const stats = await lstat(path);
+		if (stats.isSymbolicLink()) return `link:${await readlink(path)}`;
+		if (stats.isDirectory()) return 'directory';
+		if (!stats.isFile()) return 'other';
+		return path.endsWith('package.json') ? `file:${digest(await readFile(path))}` : 'file';
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 'missing';
+		return `error:${(error as NodeJS.ErrnoException).code ?? 'unknown'}`;
+	}
+}
+
+function invalidateResolutionCache(specifier: string, referrerPath: string, previousResolvedUrl: string): boolean {
+	const pathCache = (Module as unknown as { _pathCache?: Record<string, string> })._pathCache;
+	if (!pathCache) return false;
+	const previousPath = previousResolvedUrl.startsWith('file:')
+		? fileURLToPath(previousResolvedUrl)
+		: previousResolvedUrl;
+	const relativeKey = `${specifier}\0${dirname(referrerPath)}`;
+	if (pathCache[relativeKey] === previousPath) delete pathCache[relativeKey];
+	if (specifier.startsWith('.')) return true;
+	for (const cacheKey of Object.keys(pathCache)) {
+		if (cacheKey.startsWith(`${specifier}\0`) && pathCache[cacheKey] === previousPath) delete pathCache[cacheKey];
+	}
+	return true;
+}
+
+function digest(source: Source): string {
+	return createHash('sha256').update(source).digest('base64');
+}

package/components/Scope.ts

@@ -38,8 +38,10 @@ export type ScopeEventsMap = {
 	// file-driven work to avoid acting on intermediate states.
 	'deploy:start': [componentName: string];
 	// Fired after deploy I/O completes (success or failure). The scope's
-	// EntryHandlers have been recreated by this point; subsequent `add`/`change`
-	// events reflect the post-deploy tree.
+	// EntryHandlers have been resumed by this point; their replacement watcher
+	// generation compares the post-deploy scan with the retained pre-deploy
+	// snapshot, so subsequent events are the logical differences of that tree,
+	// not a replay of every surviving file as an `add`.
 	'deploy:end': [componentName: string];
 	[record: string]: [...args: unknown[]];
 };
@@ -64,11 +66,8 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	#pendingInitialLoads: Set<Promise<void>>;
 	#deployStartHandler: (name: string) => void;
 	#deployEndHandler: (name: string) => void;
-	// While a deploy of this component is in flight, EntryHandler events do not
-	// drive requestRestart() — the deploy itself produces hundreds of file
-	// changes that would otherwise pile up. A single coalesced restart is
-	// triggered by the post-deploy re-scan instead.
 	#deployInFlight: boolean = false;
+	#restartRequestedDuringDeploy: boolean = false;
 	applicationScope?: ApplicationScope;
 
 	options: OptionsWatcher;
@@ -113,6 +112,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		this.#directory = directory;
 		this.#configFilePath = configFilePath;
 		this.#logger = loggerWithTag(this.#appName);
+		this.#deployInFlight = deployLifecycle.isDeployInFlight(this.#appName);
 
 		this.databaseEvents = databaseEventsEmitter;
 		this.applicationScope = applicationScope;
@@ -156,6 +156,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		this.options = new OptionsWatcher(pluginName, configFilePath, this.#logger, isRootConfig)
 			.on('error', this.#handleError.bind(this))
 			.on('change', this.#optionsWatcherChangeListener.bind(this)())
+			.on('remove', this.#optionsWatcherRemoveListener())
 			.on('ready', this.#handleOptionsWatcherReady.bind(this));
 
 		// Bridge cross-thread deploy lifecycle events for this component. The
@@ -254,7 +255,8 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	}
 
 	#handleError(error: unknown): void {
-		this.emit('error', error);
+		if (this.listenerCount('error') > 0) this.emit('error', error);
+		else this.#logger.error?.('Error in component scope:', error);
 	}
 
 	async close(): Promise<this> {
@@ -301,6 +303,8 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	#onDeployStart(componentName: string): void {
 		this.#deployInFlight = true;
+		this.#restartRequestedDuringDeploy = false;
+		this.applicationScope?.beginDeploy();
 		// Pause each EntryHandler so it stops emitting events for the
 		// intermediate filesystem state the deploy is writing, and so it
 		// releases its inotify handles while npm install is unpacking
@@ -315,25 +319,27 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	#onDeployEnd(componentName: string): void {
 		this.#deployInFlight = false;
+		const restartRequestedDuringDeploy = this.#restartRequestedDuringDeploy;
+		this.#restartRequestedDuringDeploy = false;
 
-		// Resume each EntryHandler BEFORE notifying plugins. Otherwise a plugin
-		// throwing in its deploy:end handler would abort this function and
-		// leave the watchers permanently paused (surfaced by Gemini review).
-		// The fresh chokidar watcher does an initial scan and emits add events
-		// for the post-deploy tree; the existing per-event listener calls
-		// scope.requestRestart() for each, and the restart debounce in
-		// componentLoader collapses them into a single restart cycle. Plugin
-		// handlers stay attached across the pause.
+		// Resume before notifying plugins so a throwing deploy:end listener cannot strand the watchers.
 		for (const entryHandler of this.#entryHandlers) {
-			void entryHandler.resume();
+			void entryHandler.resume().catch(() => {});
 		}
+		if (restartRequestedDuringDeploy) this.requestRestart();
+		void this.applicationScope
+			?.finishDeploy()
+			.then((runtimeChanged) => {
+				if (runtimeChanged) this.requestRestart();
+			})
+			.catch((error) => {
+				this.#logger.error?.(`Could not verify the loaded runtime after deploying ${this.#appName}:`, error);
+				this.requestRestart();
+			});
 
 		this.#safeEmit('deploy:end', componentName);
 	}
 
-	// Swallow and log listener exceptions so one buggy plugin can't stop us
-	// from running the rest of the deploy-lifecycle bookkeeping. EventEmitter
-	// by default rethrows from synchronous listeners.
 	#safeEmit(event: 'deploy:start' | 'deploy:end', componentName: string): void {
 		try {
 			this.emit(event, componentName);
@@ -350,6 +356,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 			.on('unlink', this.#defaultEntryHandlerListener('unlink'))
 			.on('addDir', this.#defaultEntryHandlerListener('addDir'))
 			.on('unlinkDir', this.#defaultEntryHandlerListener('unlinkDir'));
+		if (this.#deployInFlight) entryHandler.pause();
 
 		this.#entryHandlers.push(entryHandler);
 
@@ -391,8 +398,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 					return;
 				}
 
-				// Otherwise, if an entry handler exists, update it with the new config
-				scope.#entryHandler.update(config as FileAndURLPathConfig);
+				void scope.#entryHandler.update(config as FileAndURLPathConfig).catch(() => {});
 
 				return;
 			}
@@ -407,6 +413,23 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		};
 	}
 
+	#optionsWatcherRemoveListener() {
+		// eslint-disable-next-line @typescript-eslint/no-this-alias
+		const scope = this;
+		// Deleting a component's config block emits `remove` (not `change`), so without
+		// this listener a running component keeps serving until some unrelated restart —
+		// even though absence is the canonical disabled state for opt-in built-ins like
+		// the /v1 models gateway. Mirrors the change listener: a plugin that registers
+		// its own `remove` handler owns the response and no restart is requested.
+		return function handleOptionsWatcherRemove(this: OptionsWatcher) {
+			if (this.listenerCount('remove') > 1) {
+				return;
+			}
+			scope.#logger.debug?.('Options removed, requesting restart');
+			scope.requestRestart();
+		};
+	}
+
 	#getFilesOption(): FileAndURLPathConfig | undefined {
 		const config = this.options.getAll();
 		if (
@@ -520,9 +543,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	requestRestart() {
 		if (this.#deployInFlight) {
-			// Suppressed: a deploy is rewriting this component's files. The
-			// post-deploy re-scan in #onDeployEnd will trigger the coalesced
-			// restart instead.
+			this.#restartRequestedDuringDeploy = true;
 			this.#logger.debug?.(`Restart suppressed (deploy in flight) for ${this.#appName}`);
 			return;
 		}
@@ -541,6 +562,29 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		}
 	}
 
+	waitForDeployCompletion(timeoutMs = 6 * 60 * 60 * 1000): Promise<void> {
+		if (!this.#deployInFlight) return Promise.resolve();
+		return new Promise((resolve, reject) => {
+			const timer = setTimeout(() => {
+				deployLifecycle.off('deploy:end', handleDeployEnd);
+				reject(new Error(`Timed out waiting for deployment of ${this.#appName} to complete`));
+			}, timeoutMs);
+			timer.unref?.();
+			const handleDeployEnd = (componentName: string) => {
+				if (componentName !== this.#appName || this.#deployInFlight) return;
+				deployLifecycle.off('deploy:end', handleDeployEnd);
+				clearTimeout(timer);
+				resolve();
+			};
+			deployLifecycle.on('deploy:end', handleDeployEnd);
+			if (!this.#deployInFlight) {
+				deployLifecycle.off('deploy:end', handleDeployEnd);
+				clearTimeout(timer);
+				resolve();
+			}
+		});
+	}
+
 	/**
 	 * Import a file into the scope's sandbox.
 	 * @param filePath - The path of the file to import.

package/components/componentLoader.ts

@@ -29,6 +29,7 @@ import * as scheduler from '../resources/scheduler/scheduler.ts';
 import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.js';
 import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts';
 import { trackScopeClose } from './scopeShutdown.ts';
+import { deployLifecycle } from './deployLifecycle.ts';
 import { toScopeMount, nestScopeMount, type ScopeMount } from './scopeMount.ts';
 import { scopedImport } from '../security/jsLoader.ts';
 import { server } from '../server/Server.ts';
@@ -170,6 +171,16 @@ export const TRUSTED_RESOURCE_PLUGINS: any = {
 		return require('../server/fastifyRoutes');
 	},
 	login,
+	// String entry: the loader `await import()`s these lazily when the component is actually
+	// processed, so the gateway's module graph is not pulled into componentLoader's own
+	// evaluation. `#src/*` is used rather than a relative path because it resolves under both
+	// conditions (source under --conditions=typestrip, dist otherwise); a relative extensionless
+	// require would only resolve against dist.
+	//
+	// The component is only *processed* when a `modelsGateway` key exists in the config, since
+	// the root loop skips absent keys (`if (!componentConfig) continue`). `defaultConfig.yaml`
+	// ships no such key, so an instance that never opts in pays nothing for this entry.
+	modelsGateway: '#src/resources/models/v1/index',
 	static: staticFiles,
 	customFunctions: {},
 	http: httpComponent,
@@ -312,6 +323,7 @@ function symlinkHarperModule(componentDirectory: string) {
  */
 function sequentiallyHandleApplication(scope: Scope, plugin: PluginModule) {
 	return scope.ready.then(async () => {
+		await scope.waitForDeployCompletion();
 		// Timeout priority is user config, plugin default, finally 30 seconds
 		const timeout = scope.options.get(['timeout']) || plugin.defaultTimeout || 30_000; // default 30 second timeout
 		if (typeof timeout !== 'number') {
@@ -333,35 +345,76 @@ function sequentiallyHandleApplication(scope: Scope, plugin: PluginModule) {
 				}, timeout + 5_000); // extra time for lock acquisition
 			});
 		}
-		let loadTimeout: NodeJS.Timeout;
 		try {
 			// note that handleApplication can throw sync or async errors, need to run finally block for both
-			await Promise.race([
+			await withDeployAwareTimeout(
 				Promise.resolve(plugin.handleApplication(scope)).then(async () => {
 					// Wait for any initial entry handler loads to complete
 					// This ensures all async operations (like secureImport) finish before the component is marked as loaded
 					await scope.waitForInitialLoads();
 				}),
-				new Promise(
-					(_, reject) =>
-						(loadTimeout = setTimeout(
-							() =>
-								reject(
-									new Error(
-										`handleApplication timed out after ${timeout}ms for ${scope.pluginName} on behalf of ${scope.appName}`
-									)
-								),
-							timeout
-						))
-				),
-			]);
+				scope,
+				timeout
+			);
 		} finally {
 			Status.primaryStore.unlock(scope.pluginName);
-			clearTimeout(loadTimeout);
 		}
 	});
 }
 
+function withDeployAwareTimeout<T>(operation: Promise<T>, scope: Scope, timeout: number): Promise<T> {
+	return new Promise((resolve, reject) => {
+		const absoluteTimeout = timeout + 6 * 60 * 60 * 1000;
+		let remaining = timeout;
+		let activeSince = 0;
+		let timer: NodeJS.Timeout | undefined;
+		let absoluteTimer: NodeJS.Timeout;
+		const cleanup = () => {
+			if (timer) clearTimeout(timer);
+			clearTimeout(absoluteTimer);
+			deployLifecycle.off('deploy:start', handleDeployStart);
+			deployLifecycle.off('deploy:end', handleDeployEnd);
+		};
+		const rejectTimeout = (limit = timeout) => {
+			cleanup();
+			reject(
+				new Error(`handleApplication timed out after ${limit}ms for ${scope.pluginName} on behalf of ${scope.appName}`)
+			);
+		};
+		absoluteTimer = setTimeout(() => rejectTimeout(absoluteTimeout), absoluteTimeout);
+		absoluteTimer.unref?.();
+		const arm = () => {
+			if (deployLifecycle.isDeployInFlight(scope.appName)) return;
+			if (remaining <= 0) return rejectTimeout();
+			activeSince = Date.now();
+			timer = setTimeout(rejectTimeout, remaining);
+		};
+		function handleDeployStart(componentName: string) {
+			if (componentName !== scope.appName || !timer) return;
+			remaining -= Date.now() - activeSince;
+			clearTimeout(timer);
+			timer = undefined;
+		}
+		function handleDeployEnd(componentName: string) {
+			if (componentName === scope.appName) arm();
+		}
+
+		deployLifecycle.on('deploy:start', handleDeployStart);
+		deployLifecycle.on('deploy:end', handleDeployEnd);
+		operation.then(
+			(value) => {
+				cleanup();
+				resolve(value);
+			},
+			(error) => {
+				cleanup();
+				reject(error);
+			}
+		);
+		arm();
+	});
+}
+
 export interface LoadComponentOptions {
 	isRoot?: boolean;
 	applicationScope?: ApplicationScope;
@@ -405,6 +458,7 @@ export async function loadComponent(
 		appName,
 		mount,
 	} = options;
+	applicationScope.runtimeRoot ??= resolvedFolder;
 	applicationScope.allowedPath ??= realpathSync(componentDirectory);
 	if (providedLoadedComponents) loadedComponents = providedLoadedComponents;
 	try {