@harperfast/harper 5.2.0-beta.4 → 5.2.1

package/components/RuntimeModuleTracker.ts

@@ -0,0 +1,189 @@
+import { createHash } from 'node:crypto';
+import { lstatSync, readFileSync, readlinkSync } from 'node:fs';
+import { lstat, readFile, readlink } from 'node:fs/promises';
+import { createRequire, Module } from 'node:module';
+import { dirname, isAbsolute, relative, resolve, sep } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+type Source = string | Buffer;
+
+const MODULE_COMPARE_CONCURRENCY = 16;
+const RESOLUTION_EXTENSIONS = ['', '.js', '.json', '.node', '.ts', '.tsx', '.cjs', '.mjs'];
+
+type Resolution = {
+	resolvedUrl: string;
+	higherPriorityCandidates?: Map<string, string>;
+};
+
+export class RuntimeModuleTracker {
+	#getRoot: () => string | undefined;
+	#modules = new Map<string, string>();
+	#resolutions = new Map<string, Resolution>();
+	#nativeRuntime = false;
+	#deployInFlight = false;
+	#loadedDuringDeploy = false;
+	#comparison?: Promise<boolean>;
+
+	constructor(getRoot: () => string | undefined) {
+		this.#getRoot = getRoot;
+	}
+
+	beginDeploy(): void {
+		if (this.#deployInFlight) return;
+		this.#deployInFlight = true;
+		this.#loadedDuringDeploy = false;
+		this.#comparison = undefined;
+	}
+
+	markNativeRuntime(): void {
+		this.#nativeRuntime = true;
+		if (this.#deployInFlight) this.#loadedDuringDeploy = true;
+	}
+
+	recordModule(moduleUrl: string, source: Source): void {
+		const modulePath = this.#localPath(moduleUrl);
+		if (!modulePath) return;
+		if (!this.#modules.has(modulePath)) this.#modules.set(modulePath, digest(source));
+		if (this.#deployInFlight) this.#loadedDuringDeploy = true;
+	}
+
+	recordResolution(specifier: string, referrer: string, resolvedUrl: string): void {
+		const referrerPath = this.#localPath(referrer);
+		const resolvedPath = this.#localPath(resolvedUrl);
+		if (!referrerPath || !resolvedPath) return;
+		const key = `${referrerPath}\0${specifier}`;
+		if (!this.#resolutions.has(key)) {
+			const candidates = higherPriorityResolutionCandidates(specifier, referrerPath, resolvedPath);
+			this.#resolutions.set(key, {
+				resolvedUrl,
+				higherPriorityCandidates: candidates
+					? new Map(candidates.map((candidate) => [candidate, candidateState(candidate)]))
+					: undefined,
+			});
+			if (this.#deployInFlight) this.#loadedDuringDeploy = true;
+		}
+	}
+
+	finishDeploy(): Promise<boolean> {
+		if (!this.#deployInFlight) return this.#comparison ?? Promise.resolve(false);
+		this.#deployInFlight = false;
+		this.#comparison = this.#compare();
+		return this.#comparison;
+	}
+
+	async #compare(): Promise<boolean> {
+		if (this.#nativeRuntime || this.#loadedDuringDeploy) return true;
+		const modules = [...this.#modules];
+		for (let start = 0; start < modules.length; start += MODULE_COMPARE_CONCURRENCY) {
+			const changed = await Promise.all(
+				modules.slice(start, start + MODULE_COMPARE_CONCURRENCY).map(async ([modulePath, previousDigest]) => {
+					try {
+						return digest(await readFile(modulePath)) !== previousDigest;
+					} catch {
+						return true;
+					}
+				})
+			);
+			if (changed.includes(true)) return true;
+		}
+		for (const [key, resolution] of this.#resolutions) {
+			const separator = key.indexOf('\0');
+			const referrerPath = key.slice(0, separator);
+			const specifier = key.slice(separator + 1);
+			if (
+				resolution.higherPriorityCandidates &&
+				(
+					await Promise.all(
+						[...resolution.higherPriorityCandidates].map(
+							async ([candidate, previousState]) => (await candidateStateAsync(candidate)) !== previousState
+						)
+					)
+				).includes(true)
+			)
+				return true;
+			try {
+				// Node does not invalidate this cache when a component tree is replaced in place.
+				if (!invalidateResolutionCache(specifier, referrerPath, resolution.resolvedUrl)) return true;
+				const resolved = createRequire(pathToFileURL(referrerPath)).resolve(specifier);
+				const resolvedUrl = isAbsolute(resolved) ? pathToFileURL(resolved).toString() : resolved;
+				if (resolvedUrl !== resolution.resolvedUrl) return true;
+			} catch {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	#localPath(moduleUrl: string): string | undefined {
+		if (!moduleUrl.startsWith('file:')) return;
+		const root = this.#getRoot();
+		if (!root) return;
+		const modulePath = fileURLToPath(moduleUrl);
+		const relativePath = relative(resolve(root), modulePath);
+		if (
+			relativePath === '' ||
+			(!relativePath.startsWith(`..${sep}`) && relativePath !== '..' && !isAbsolute(relativePath))
+		)
+			return modulePath;
+	}
+}
+
+function higherPriorityResolutionCandidates(
+	specifier: string,
+	referrerPath: string,
+	resolvedPath: string
+): string[] | undefined {
+	if (!specifier.startsWith('.')) return;
+	const basePath = resolve(dirname(referrerPath), specifier);
+	const candidates = [...new Set(RESOLUTION_EXTENSIONS.map((extension) => basePath + extension))];
+	candidates.push(resolve(basePath, 'package.json'));
+	for (const extension of RESOLUTION_EXTENSIONS.slice(1)) candidates.push(resolve(basePath, `index${extension}`));
+	const resolvedIndex = candidates.indexOf(resolvedPath);
+	if (resolvedIndex === -1) return;
+	return candidates.slice(0, resolvedIndex);
+}
+
+function candidateState(path: string): string {
+	try {
+		const stats = lstatSync(path);
+		if (stats.isSymbolicLink()) return `link:${readlinkSync(path)}`;
+		if (stats.isDirectory()) return 'directory';
+		if (!stats.isFile()) return 'other';
+		return path.endsWith('package.json') ? `file:${digest(readFileSync(path))}` : 'file';
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 'missing';
+		return `error:${(error as NodeJS.ErrnoException).code ?? 'unknown'}`;
+	}
+}
+
+async function candidateStateAsync(path: string): Promise<string> {
+	try {
+		const stats = await lstat(path);
+		if (stats.isSymbolicLink()) return `link:${await readlink(path)}`;
+		if (stats.isDirectory()) return 'directory';
+		if (!stats.isFile()) return 'other';
+		return path.endsWith('package.json') ? `file:${digest(await readFile(path))}` : 'file';
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 'missing';
+		return `error:${(error as NodeJS.ErrnoException).code ?? 'unknown'}`;
+	}
+}
+
+function invalidateResolutionCache(specifier: string, referrerPath: string, previousResolvedUrl: string): boolean {
+	const pathCache = (Module as unknown as { _pathCache?: Record<string, string> })._pathCache;
+	if (!pathCache) return false;
+	const previousPath = previousResolvedUrl.startsWith('file:')
+		? fileURLToPath(previousResolvedUrl)
+		: previousResolvedUrl;
+	const relativeKey = `${specifier}\0${dirname(referrerPath)}`;
+	if (pathCache[relativeKey] === previousPath) delete pathCache[relativeKey];
+	if (specifier.startsWith('.')) return true;
+	for (const cacheKey of Object.keys(pathCache)) {
+		if (cacheKey.startsWith(`${specifier}\0`) && pathCache[cacheKey] === previousPath) delete pathCache[cacheKey];
+	}
+	return true;
+}
+
+function digest(source: Source): string {
+	return createHash('sha256').update(source).digest('base64');
+}

package/components/Scope.ts

@@ -11,6 +11,7 @@ import type { FileAndURLPathConfig } from './Component.ts';
 import { FilesOption } from './deriveGlobOptions.ts';
 import { requestRestart } from './requestRestart.ts';
 import { resolveBaseURLPath } from './resolveBaseURLPath.ts';
+import { composeMountedUrlPath, type ScopeMount } from './scopeMount.ts';
 import { ApplicationScope } from './ApplicationScope.ts';
 import {
 	getSecretsForComponent,
@@ -37,8 +38,10 @@ export type ScopeEventsMap = {
 	// file-driven work to avoid acting on intermediate states.
 	'deploy:start': [componentName: string];
 	// Fired after deploy I/O completes (success or failure). The scope's
-	// EntryHandlers have been recreated by this point; subsequent `add`/`change`
-	// events reflect the post-deploy tree.
+	// EntryHandlers have been resumed by this point; their replacement watcher
+	// generation compares the post-deploy scan with the retained pre-deploy
+	// snapshot, so subsequent events are the logical differences of that tree,
+	// not a replay of every surviving file as an `add`.
 	'deploy:end': [componentName: string];
 	[record: string]: [...args: unknown[]];
 };
@@ -63,11 +66,8 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	#pendingInitialLoads: Set<Promise<void>>;
 	#deployStartHandler: (name: string) => void;
 	#deployEndHandler: (name: string) => void;
-	// While a deploy of this component is in flight, EntryHandler events do not
-	// drive requestRestart() — the deploy itself produces hundreds of file
-	// changes that would otherwise pile up. A single coalesced restart is
-	// triggered by the post-deploy re-scan instead.
 	#deployInFlight: boolean = false;
+	#restartRequestedDuringDeploy: boolean = false;
 	applicationScope?: ApplicationScope;
 
 	options: OptionsWatcher;
@@ -81,6 +81,18 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	// process-global side effects should validate fully but skip activation.
 	isTransientValidation?: boolean;
 
+	/**
+	 * Routing the operator declared for this application in the root config. Applied automatically
+	 * to handlers registered through `scope.server`, so plugins normally don't touch it — the
+	 * router strips the mount before a handler runs and everything inside the application
+	 * addresses itself mount-relative.
+	 *
+	 * Read it only when a plugin emits an absolute URL back to the client (e.g. a redirect
+	 * `Location`, via `externalBasePath()`) or bypasses the routed chain entirely (legacy
+	 * fastify routes register on the bare server).
+	 */
+	mount?: ScopeMount;
+
 	constructor(
 		appName: string,
 		pluginName: string,
@@ -88,16 +100,19 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		configFilePath: string,
 		applicationScope: ApplicationScope,
 		origin: string = appName,
-		isRootConfig?: boolean
+		isRootConfig?: boolean,
+		mount?: ScopeMount
 	) {
 		super();
 
+		this.mount = mount;
 		this.#appName = appName;
 		this.#pluginName = pluginName;
 		this.#origin = typeof origin === 'string' ? origin : appName;
 		this.#directory = directory;
 		this.#configFilePath = configFilePath;
 		this.#logger = loggerWithTag(this.#appName);
+		this.#deployInFlight = deployLifecycle.isDeployInFlight(this.#appName);
 
 		this.databaseEvents = databaseEventsEmitter;
 		this.applicationScope = applicationScope;
@@ -117,14 +132,10 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 					const method = Reflect.get(target, prop, receiver);
 					if (typeof method === 'function') {
 						return (listener: any, options?: any) => {
-							const scopeConfig = (scopeRef.options?.getAll() as any) ?? {};
 							return method.call(target, listener, {
 								name: pluginName,
-								// resolve to the same base the entry pipeline uses ('assets' -> '/assets/',
-								// './x' -> '/<name>/x/') so route matching sees a real pathname prefix (#1583)
-								urlPath: scopeConfig.urlPath ? resolveBaseURLPath(pluginName, scopeConfig.urlPath) : undefined,
-								host: scopeConfig.host || undefined,
 								...options,
+								...scopeRef.routeFor(options),
 							});
 						};
 					}
@@ -145,6 +156,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		this.options = new OptionsWatcher(pluginName, configFilePath, this.#logger, isRootConfig)
 			.on('error', this.#handleError.bind(this))
 			.on('change', this.#optionsWatcherChangeListener.bind(this)())
+			.on('remove', this.#optionsWatcherRemoveListener())
 			.on('ready', this.#handleOptionsWatcherReady.bind(this));
 
 		// Bridge cross-thread deploy lifecycle events for this component. The
@@ -184,6 +196,41 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		return this.#pluginName;
 	}
 
+	/**
+	 * Turns a mount-relative base path into the absolute path a client sees, by prefixing the
+	 * application's mount. Use it for anything sent back to the client — a redirect `Location`, a
+	 * generated link — since the router strips the mount before a handler runs and a handler's own
+	 * view of the path therefore excludes it.
+	 */
+	externalBasePath(baseURLPath: string): string {
+		return this.mount?.urlPath ? `${this.mount.urlPath}${baseURLPath}` : baseURLPath;
+	}
+
+	/**
+	 * The route a handler registered through `scope.server` with these options will answer on — the
+	 * single place the application mount is applied. `scope.server` uses it, and a plugin that must
+	 * identify its own route (e.g. REST deduplicating registration per mount) calls it rather than
+	 * recomposing the parts, so there is one definition of "which route is this".
+	 *
+	 * An explicit call option wins over config, but either way `urlPath` is *resolved* rather than
+	 * passed through: plugins that spread their whole config section into these options (REST does)
+	 * would otherwise hand the router a raw value — './' became the literal, unmatchable route '/.'.
+	 */
+	routeFor(options?: { urlPath?: string; host?: string }): { host?: string; urlPath?: string } {
+		const scopeConfig = (this.options?.getAll() as any) ?? {};
+		const rawUrlPath = options?.urlPath ?? scopeConfig.urlPath;
+		// resolve to the same base the entry pipeline uses ('assets' -> '/assets/', './x' ->
+		// '/<name>/x/') so route matching sees a real pathname prefix (#1583), then prefix the
+		// application's mount. The mount is applied ONLY here, at the routing boundary: the router
+		// strips it before the handler runs, so entry URL paths — and the resource paths
+		// graphqlSchema/jsResource derive from them — stay mount-relative.
+		const pluginUrlPath = rawUrlPath ? resolveBaseURLPath(this.#pluginName, rawUrlPath) : undefined;
+		return {
+			host: this.mount?.host || options?.host || scopeConfig.host || undefined,
+			urlPath: composeMountedUrlPath(this.mount?.urlPath, this.#pluginName, pluginUrlPath) || undefined,
+		};
+	}
+
 	get directory(): string {
 		return this.#directory;
 	}
@@ -208,7 +255,8 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	}
 
 	#handleError(error: unknown): void {
-		this.emit('error', error);
+		if (this.listenerCount('error') > 0) this.emit('error', error);
+		else this.#logger.error?.('Error in component scope:', error);
 	}
 
 	async close(): Promise<this> {
@@ -255,6 +303,8 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	#onDeployStart(componentName: string): void {
 		this.#deployInFlight = true;
+		this.#restartRequestedDuringDeploy = false;
+		this.applicationScope?.beginDeploy();
 		// Pause each EntryHandler so it stops emitting events for the
 		// intermediate filesystem state the deploy is writing, and so it
 		// releases its inotify handles while npm install is unpacking
@@ -269,25 +319,27 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	#onDeployEnd(componentName: string): void {
 		this.#deployInFlight = false;
+		const restartRequestedDuringDeploy = this.#restartRequestedDuringDeploy;
+		this.#restartRequestedDuringDeploy = false;
 
-		// Resume each EntryHandler BEFORE notifying plugins. Otherwise a plugin
-		// throwing in its deploy:end handler would abort this function and
-		// leave the watchers permanently paused (surfaced by Gemini review).
-		// The fresh chokidar watcher does an initial scan and emits add events
-		// for the post-deploy tree; the existing per-event listener calls
-		// scope.requestRestart() for each, and the restart debounce in
-		// componentLoader collapses them into a single restart cycle. Plugin
-		// handlers stay attached across the pause.
+		// Resume before notifying plugins so a throwing deploy:end listener cannot strand the watchers.
 		for (const entryHandler of this.#entryHandlers) {
-			void entryHandler.resume();
+			void entryHandler.resume().catch(() => {});
 		}
+		if (restartRequestedDuringDeploy) this.requestRestart();
+		void this.applicationScope
+			?.finishDeploy()
+			.then((runtimeChanged) => {
+				if (runtimeChanged) this.requestRestart();
+			})
+			.catch((error) => {
+				this.#logger.error?.(`Could not verify the loaded runtime after deploying ${this.#appName}:`, error);
+				this.requestRestart();
+			});
 
 		this.#safeEmit('deploy:end', componentName);
 	}
 
-	// Swallow and log listener exceptions so one buggy plugin can't stop us
-	// from running the rest of the deploy-lifecycle bookkeeping. EventEmitter
-	// by default rethrows from synchronous listeners.
 	#safeEmit(event: 'deploy:start' | 'deploy:end', componentName: string): void {
 		try {
 			this.emit(event, componentName);
@@ -304,6 +356,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 			.on('unlink', this.#defaultEntryHandlerListener('unlink'))
 			.on('addDir', this.#defaultEntryHandlerListener('addDir'))
 			.on('unlinkDir', this.#defaultEntryHandlerListener('unlinkDir'));
+		if (this.#deployInFlight) entryHandler.pause();
 
 		this.#entryHandlers.push(entryHandler);
 
@@ -345,8 +398,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 					return;
 				}
 
-				// Otherwise, if an entry handler exists, update it with the new config
-				scope.#entryHandler.update(config as FileAndURLPathConfig);
+				void scope.#entryHandler.update(config as FileAndURLPathConfig).catch(() => {});
 
 				return;
 			}
@@ -361,6 +413,23 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		};
 	}
 
+	#optionsWatcherRemoveListener() {
+		// eslint-disable-next-line @typescript-eslint/no-this-alias
+		const scope = this;
+		// Deleting a component's config block emits `remove` (not `change`), so without
+		// this listener a running component keeps serving until some unrelated restart —
+		// even though absence is the canonical disabled state for opt-in built-ins like
+		// the /v1 models gateway. Mirrors the change listener: a plugin that registers
+		// its own `remove` handler owns the response and no restart is requested.
+		return function handleOptionsWatcherRemove(this: OptionsWatcher) {
+			if (this.listenerCount('remove') > 1) {
+				return;
+			}
+			scope.#logger.debug?.('Options removed, requesting restart');
+			scope.requestRestart();
+		};
+	}
+
 	#getFilesOption(): FileAndURLPathConfig | undefined {
 		const config = this.options.getAll();
 		if (
@@ -474,9 +543,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	requestRestart() {
 		if (this.#deployInFlight) {
-			// Suppressed: a deploy is rewriting this component's files. The
-			// post-deploy re-scan in #onDeployEnd will trigger the coalesced
-			// restart instead.
+			this.#restartRequestedDuringDeploy = true;
 			this.#logger.debug?.(`Restart suppressed (deploy in flight) for ${this.#appName}`);
 			return;
 		}
@@ -495,6 +562,29 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		}
 	}
 
+	waitForDeployCompletion(timeoutMs = 6 * 60 * 60 * 1000): Promise<void> {
+		if (!this.#deployInFlight) return Promise.resolve();
+		return new Promise((resolve, reject) => {
+			const timer = setTimeout(() => {
+				deployLifecycle.off('deploy:end', handleDeployEnd);
+				reject(new Error(`Timed out waiting for deployment of ${this.#appName} to complete`));
+			}, timeoutMs);
+			timer.unref?.();
+			const handleDeployEnd = (componentName: string) => {
+				if (componentName !== this.#appName || this.#deployInFlight) return;
+				deployLifecycle.off('deploy:end', handleDeployEnd);
+				clearTimeout(timer);
+				resolve();
+			};
+			deployLifecycle.on('deploy:end', handleDeployEnd);
+			if (!this.#deployInFlight) {
+				deployLifecycle.off('deploy:end', handleDeployEnd);
+				clearTimeout(timer);
+				resolve();
+			}
+		});
+	}
+
 	/**
 	 * Import a file into the scope's sandbox.
 	 * @param filePath - The path of the file to import.