@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/components/RuntimeModuleTracker.ts

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto';
-import { lstatSync, readFileSync, readlinkSync } from 'node:fs';
+import { lstatSync, readFileSync, readlinkSync, realpathSync } from 'node:fs';
 import { lstat, readFile, readlink } from 'node:fs/promises';
 import { createRequire, Module } from 'node:module';
 import { dirname, isAbsolute, relative, resolve, sep } from 'node:path';
@@ -43,7 +43,7 @@ export class RuntimeModuleTracker {
 	recordModule(moduleUrl: string, source: Source): void {
 		const modulePath = this.#localPath(moduleUrl);
 		if (!modulePath) return;
-		if (!this.#modules.has(modulePath)) this.#modules.set(modulePath, digest(source));
+		if (!this.#modules.has(modulePath)) this.#modules.set(modulePath, moduleDigest(modulePath, source));
 		if (this.#deployInFlight) this.#loadedDuringDeploy = true;
 	}
 
@@ -78,7 +78,7 @@ export class RuntimeModuleTracker {
 			const changed = await Promise.all(
 				modules.slice(start, start + MODULE_COMPARE_CONCURRENCY).map(async ([modulePath, previousDigest]) => {
 					try {
-						return digest(await readFile(modulePath)) !== previousDigest;
+						return moduleDigest(modulePath, await readFile(modulePath)) !== previousDigest;
 					} catch {
 						return true;
 					}
@@ -138,8 +138,21 @@ function higherPriorityResolutionCandidates(
 	const candidates = [...new Set(RESOLUTION_EXTENSIONS.map((extension) => basePath + extension))];
 	candidates.push(resolve(basePath, 'package.json'));
 	for (const extension of RESOLUTION_EXTENSIONS.slice(1)) candidates.push(resolve(basePath, `index${extension}`));
-	const resolvedIndex = candidates.indexOf(resolvedPath);
-	if (resolvedIndex === -1) return;
+	let resolvedIndex = candidates.indexOf(resolvedPath);
+	if (resolvedIndex === -1) {
+		let canonicalResolvedPath = resolvedPath;
+		try {
+			canonicalResolvedPath = realpathSync(resolvedPath);
+		} catch {}
+		resolvedIndex = candidates.findIndex((candidate) => {
+			try {
+				return realpathSync(candidate) === canonicalResolvedPath;
+			} catch {
+				return false;
+			}
+		});
+	}
+	if (resolvedIndex === -1) return candidates.slice(0, RESOLUTION_EXTENSIONS.length + 1);
 	return candidates.slice(0, resolvedIndex);
 }
 
@@ -149,7 +162,7 @@ function candidateState(path: string): string {
 		if (stats.isSymbolicLink()) return `link:${readlinkSync(path)}`;
 		if (stats.isDirectory()) return 'directory';
 		if (!stats.isFile()) return 'other';
-		return path.endsWith('package.json') ? `file:${digest(readFileSync(path))}` : 'file';
+		return path.endsWith('package.json') ? `file:${moduleDigest(path, readFileSync(path))}` : 'file';
 	} catch (error) {
 		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 'missing';
 		return `error:${(error as NodeJS.ErrnoException).code ?? 'unknown'}`;
@@ -162,7 +175,7 @@ async function candidateStateAsync(path: string): Promise<string> {
 		if (stats.isSymbolicLink()) return `link:${await readlink(path)}`;
 		if (stats.isDirectory()) return 'directory';
 		if (!stats.isFile()) return 'other';
-		return path.endsWith('package.json') ? `file:${digest(await readFile(path))}` : 'file';
+		return path.endsWith('package.json') ? `file:${moduleDigest(path, await readFile(path))}` : 'file';
 	} catch (error) {
 		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 'missing';
 		return `error:${(error as NodeJS.ErrnoException).code ?? 'unknown'}`;
@@ -187,3 +200,21 @@ function invalidateResolutionCache(specifier: string, referrerPath: string, prev
 function digest(source: Source): string {
 	return createHash('sha256').update(source).digest('base64');
 }
+
+function moduleDigest(path: string, source: Source): string {
+	if (path.endsWith('package.json')) {
+		try {
+			return digest(JSON.stringify(canonicalizeJSON(JSON.parse(source.toString()))));
+		} catch {}
+	}
+	return digest(source);
+}
+
+function canonicalizeJSON(value: unknown): unknown {
+	if (Array.isArray(value)) return value.map(canonicalizeJSON);
+	if (!value || typeof value !== 'object') return value;
+	const canonical: Record<string, unknown> = Object.create(null);
+	for (const key of Object.keys(value).sort())
+		canonical[key] = canonicalizeJSON((value as Record<string, unknown>)[key]);
+	return canonical;
+}

package/components/Scope.ts

@@ -1,7 +1,7 @@
 import { type Logger } from '../utility/logging/logger.ts';
 import { loggerWithTag } from '../utility/logging/harper_logger.ts';
 import { EventEmitter, once } from 'node:events';
-import { databaseEventsEmitter, table } from '../resources/databases.ts';
+import { databaseEventsEmitter, scopedTableFactory } from '../resources/databases.ts';
 import { server, type Server } from '../server/Server.ts';
 import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts';
 import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts';
@@ -20,6 +20,7 @@ import {
 } from './componentSecrets.ts';
 import type { SecretsView } from './componentSecrets.ts';
 import { deployLifecycle } from './deployLifecycle.ts';
+import { thisThreadOwnsApplication } from '../server/threads/isolatedApplications.ts';
 
 export class MissingDefaultFilesOptionError extends Error {
 	constructor() {
@@ -68,6 +69,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	#deployEndHandler: (name: string) => void;
 	#deployInFlight: boolean = false;
 	#restartRequestedDuringDeploy: boolean = false;
+	#optionsReady: boolean = false;
 	applicationScope?: ApplicationScope;
 
 	options: OptionsWatcher;
@@ -241,7 +243,10 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	ensureTable<TableResourceType = unknown>(options: any): TableResourceType {
 		options.origin = this.#origin;
-		return table<TableResourceType>(options);
+		return scopedTableFactory(
+			this.applicationScope?.branches,
+			thisThreadOwnsApplication(this.applicationScope?.name)
+		)<TableResourceType>(options);
 	}
 
 	#handleOptionsWatcherReady(): void {
@@ -251,7 +256,21 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		// We could make the user call `await scope.ready()` in their `handleApplication` function, but that could lead to the same issue and it'd
 		// be harder for the user to understand why.
 
+		// A second `ready` means the scope had no config of its own and now does — a config file
+		// that was unreadable when this worker booted, or one recreated after deletion. Re-emitting
+		// reaches nobody: componentLoader is long past its `await scope.ready`, so the component is
+		// running on the defaults until something restarts it. Same recovery as the `remove`
+		// listener, and the same convention — a plugin with its own `ready` handler owns it.
+		const started = this.#optionsReady;
+		this.#optionsReady = true;
+		const restartNeeded = started && this.listenerCount('ready') === 0;
+
 		this.emit('ready');
+
+		if (restartNeeded) {
+			this.#logger.debug?.('Options arrived after the scope started, requesting restart');
+			this.requestRestart();
+		}
 	}
 
 	#handleError(error: unknown): void {
@@ -461,33 +480,55 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 			targetEntryHandler: EntryHandler,
 			entryEventHandler: onEntryEventHandler
 		): onEntryEventHandler => {
-			const pendingOperations = new Set<Promise<void>>();
+			// Retained until the drain below reports them: an operation dropped as it settles is invisible
+			// to that drain, so a handler rejecting before `ready` would report nothing at all.
+			let initialOperations: Promise<void>[] | null = [];
 
 			const wrapped: onEntryEventHandler = (entry) => {
-				const result = entryEventHandler(entry);
+				let result;
+				try {
+					result = entryEventHandler(entry);
+				} catch (error) {
+					// A synchronous throw is the same failure as a rejection. Left to propagate, EntryHandler
+					// catches it for its own reporting and the initial load completes as if nothing failed.
+					result = Promise.reject(error);
+				}
 				if (result instanceof Promise) {
-					const tracked = result
-						.catch((error) => {
-							this.#logger.error?.('Error in async entry handler:', error);
+					const tracked = result.catch((error) => {
+						this.#logger.error?.('Error in async entry handler:', error);
+						try {
 							this.#handleError(error);
-							throw error;
-						})
-						.finally(() => pendingOperations.delete(tracked));
-					pendingOperations.add(tracked);
+						} catch (reportingError) {
+							// Reporting must not replace the failure it is reporting.
+							this.#logger.error?.('Error reporting an entry handler failure:', reportingError);
+						}
+						throw error;
+					});
+					// Nothing awaits `tracked` until the drain, which leaves an early rejection unhandled.
+					tracked.catch(() => {});
+					initialOperations?.push(tracked);
 				}
 			};
 
-			// When the entry handler's initial scan completes, wait for all pending async operations
+			// When the entry handler's initial scan completes, wait for all of its async operations
 			const initialLoadPromise = once(targetEntryHandler, 'ready').then(async () => {
-				if (pendingOperations.size > 0) {
-					await Promise.all(pendingOperations);
+				const operations = initialOperations ?? [];
+				initialOperations = null;
+				// Drained, not fail-fast: the loader holds the plugin-wide load lock until this settles, so a
+				// first failure reported while a sibling still runs would let the next application in.
+				for (const result of await Promise.allSettled(operations)) {
+					if (result.status === 'rejected') throw result.reason;
 				}
 				targetEntryHandler.emit('initialLoadComplete');
 			});
 
 			// Track this promise so the component loader can await it
 			this.#pendingInitialLoads.add(initialLoadPromise);
-			initialLoadPromise.finally(() => this.#pendingInitialLoads.delete(initialLoadPromise));
+			// Two-branch cleanup, not `.finally`: its derivative would reject with nobody left to handle it.
+			const forgetInitialLoad = () => {
+				this.#pendingInitialLoads.delete(initialLoadPromise);
+			};
+			initialLoadPromise.then(forgetInitialLoad, forgetInitialLoad);
 
 			return wrapped;
 		};

package/components/awaitRestart.ts

@@ -0,0 +1,84 @@
+/**
+ * Waiting for a worker restart on behalf of an operation that reports whether it finished.
+ *
+ * No single clock bounds a restart: each replacement has its own startup backstop, a worker draining
+ * in-flight work may hold its exit for as long as the drain ceiling allows, and a wide pool rolls a
+ * few workers at a time. So the wait follows the restart's own progress and gives up only when
+ * nothing has moved for `idleTimeoutMs`, or at the absolute `ceilingMs` — a caller holding an API
+ * response open cannot wait forever.
+ */
+
+/** How long the restart may go without reporting progress before the wait gives up. */
+export const RESTART_IDLE_TIMEOUT_MS = 60_000;
+/** Absolute cap on the wait, matching the default shutdown-drain ceiling. */
+export const RESTART_WAIT_CEILING_MS = 600_000;
+
+export interface RestartOutcome {
+	completed: boolean;
+	/** Workers left running the previous code because their replacement never came up. */
+	workersKeptOnOldCode?: number;
+	/** Replacements that never reported starting after their predecessor was already gone. */
+	replacementsNotStarted?: number;
+	/** The wait ended because the restart stopped reporting progress. */
+	stalled?: boolean;
+	/** The restart was handed to another thread, which reports its own completion. */
+	handedOff?: boolean;
+	/** No restart was performed: the process is already shutting down. */
+	declined?: boolean;
+	error?: any;
+}
+
+/** Reports progress; a deadline says the reporter is busy until then (e.g. a drain it asked for). */
+export type RestartProgress = (untilMs?: number) => void;
+
+export function awaitRestart(
+	startRestart: (onProgress: RestartProgress) => any,
+	{ idleTimeoutMs = RESTART_IDLE_TIMEOUT_MS, ceilingMs = RESTART_WAIT_CEILING_MS } = {}
+): Promise<RestartOutcome> {
+	let idleTimer: any, ceilingTimer: any;
+	let settled = false;
+	let reportStalled: (outcome: RestartOutcome) => void;
+	const done = (outcome: RestartOutcome) => {
+		settled = true;
+		clearTimeout(idleTimer);
+		clearTimeout(ceilingTimer);
+		return outcome;
+	};
+	const onProgress: RestartProgress = (untilMs) => {
+		// The restart keeps reporting past the point where the wait gave up; re-arming a timer then
+		// would leave a live handle behind for as long as the last reported deadline.
+		if (settled) return;
+		clearTimeout(idleTimer);
+		// A reported deadline is what the reporter says it needs, so honor it over the idle window; the
+		// absolute ceiling is what keeps that bounded.
+		const wait = Math.max(idleTimeoutMs, untilMs === undefined ? 0 : untilMs - Date.now());
+		idleTimer = setTimeout(() => reportStalled(done({ completed: false, stalled: true })), wait);
+	};
+	const stalled = new Promise<RestartOutcome>((resolve) => {
+		reportStalled = resolve;
+	});
+	const ceiling = new Promise<RestartOutcome>((resolve) => {
+		ceilingTimer = setTimeout(() => resolve(done({ completed: false })), ceilingMs);
+	});
+	onProgress();
+	const restarted = Promise.resolve()
+		.then(() => startRestart(onProgress))
+		.then(
+			(result) =>
+				done(
+					result && typeof result === 'object'
+						? result.declined
+							? { completed: false, declined: true }
+							: {
+									completed: true,
+									workersKeptOnOldCode: result.workersKeptOnOldCode ?? 0,
+									replacementsNotStarted: result.replacementsNotStarted ?? 0,
+								}
+						: // Only the thread performing the restart can report on it; from a worker
+							// restartWorkers() hands it off and resolves with nothing.
+							{ completed: false, handedOff: true }
+				),
+			(error) => done({ completed: false, error })
+		);
+	return Promise.race([restarted, stalled, ceiling]);
+}