@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/server/http.ts

@@ -18,7 +18,13 @@ import { createServer as createSecureServerHttp1 } from 'node:https';
 import { createServer, IncomingMessage, validateHeaderName, validateHeaderValue } from 'node:http';
 import { createServer as createNetServer } from 'node:net';
 import { Request, BunRequest, UwsRequest, isBun } from './serverHelpers/Request.ts';
-import { appendHeader, Headers, toWriteHeadHeaders } from './serverHelpers/Headers.ts';
+import {
+	appendHeader,
+	bridgeChainHeadersToNodeResponse,
+	Headers,
+	mergeChainHeadersIntoFallback,
+	toWriteHeadHeaders,
+} from './serverHelpers/Headers.ts';
 import {
 	decodeProxyHeader,
 	applyProxyHeader,
@@ -34,7 +40,7 @@ import { server, type ServerOptions, type HttpOptions, type UpgradeOptions, Upgr
 import { setPortServerMap, SERVERS, socketOptionDefaults } from './serverRegistry.ts';
 import { getComponentName } from '../components/componentLoader.ts';
 import { throttle } from './throttle.ts';
-import { makeCallbackChain as buildCallbackChain, describeChains } from './middlewareChain.ts';
+import { makeCallbackChain as buildCallbackChain, describeChains, type HttpEntry } from './middlewareChain.ts';
 import { WebSocketServer } from 'ws';
 
 const { errorToString, errorForLog } = harperLogger;
@@ -490,7 +496,7 @@ export function httpServer(listener, options) {
 			httpResponders[options?.runFirst ? 'unshift' : 'push'](entry);
 		} else if (isBun) {
 			// On Bun, store non-function listeners (e.g. Fastify's http.Server) for fallback delegation
-			fallbackServers[port] = listener;
+			registerFallbackServer(port, listener);
 		} else if ((httpServers[port] as any)?.uws) {
 			// uWS HTTP path (#914, HARPER_UWS_HTTP): the port is backed by uWebSockets.js, not a Node
 			// http server, so a raw non-function listener (e.g. Fastify's http.Server via
@@ -499,12 +505,12 @@ export function httpServer(listener, options) {
 			// port. Divert it to the fallback map like the Bun path; makeUwsHandler delegates unhandled
 			// requests to it via inject(). The { uws: true } marker is guaranteed present here: the
 			// getServer(port) call above (same loop iteration) sets it before this branch runs.
-			fallbackServers[port] = listener;
+			registerFallbackServer(port, listener);
 		} else {
 			listener.isSecure = secure;
 			registerServer(listener, port, false);
 		}
-		httpChain[port] = makeCallbackChain(httpResponders, port);
+		buildChains(httpChain, httpResponders, port);
 	}
 
 	return servers;
@@ -543,10 +549,47 @@ export function pipeBodyToResponse(
 		} else {
 			recordAction(performance.now() - endTime, 'transfer', handlerPath, method);
 			recordAction(bytesSent, 'bytes-sent', handlerPath, method);
+			endConnectionIfClientExpectsClose(nodeResponse);
 		}
 	});
 }
 
+/**
+ * Bun's node:http never derives keep-alive from the request, so a stream-ended response delivers its
+ * full body and then leaves the client attached until its own timeout (#2210). Ending the *request's*
+ * socket is the only remedy that works there, and it is graceful, so nothing is truncated (measured
+ * on plain TCP and TLS). DESIGN.md carries the measurements and the alternatives that don't work.
+ */
+function endConnectionIfClientExpectsClose(nodeResponse: any) {
+	if (!isBun) return;
+	const nodeRequest = nodeResponse.req;
+	if (nodeRequest?.httpVersionMajor !== 1) return;
+	const socket = nodeRequest.socket;
+	// a peer that RSTs right after the terminal chunk can leave this callback holding a destroyed
+	// socket; end()ing it would throw from inside stream internals, i.e. an uncaughtException
+	if (!socket || socket.destroyed) return;
+	const connection = nodeRequest.headers?.connection;
+	const tokens = typeof connection === 'string' ? connection.split(',') : [];
+	if (hasConnectionToken(tokens, 'close')) {
+		socket.end();
+		return;
+	}
+	// HTTP/1.0 persistence needs an explicit keep-alive AND a length to read to, so a 1.0 response
+	// that got no Content-Length is delimited by the close — the same line Node draws
+	if (
+		nodeRequest.httpVersionMinor === 0 &&
+		!(hasConnectionToken(tokens, 'keep-alive') && nodeResponse.hasHeader?.('content-length'))
+	)
+		socket.end();
+}
+
+function hasConnectionToken(tokens: string[], wanted: string) {
+	for (const token of tokens) {
+		if (token.trim().toLowerCase() === wanted) return true;
+	}
+	return false;
+}
+
 function getHTTPServer(port: number, secure: boolean, options: ServerOptions) {
 	const { mtls: isMtls, usageType } = options || {};
 	const isOperationsServer = usageType === 'operations-api';
@@ -642,9 +685,7 @@ function getHTTPServer(port: number, secure: boolean, options: ServerOptions) {
 					// This means the HDB stack didn't handle the request, and we can then cascade the request
 					// to the server-level handler, forming the bridge to the slower legacy fastify framework that expects
 					// to interact with a node HTTP server object.
-					for (const headerPair of response.headers || []) {
-						nodeResponse.setHeader(headerPair[0], headerPair[1]);
-					}
+					bridgeChainHeadersToNodeResponse(response.headers, nodeResponse);
 					nodeRequest.baseRequest = request;
 					nodeResponse.baseResponse = response;
 					return httpServers[port].emit('unhandled', nodeRequest, nodeResponse);
@@ -947,7 +988,7 @@ function getHTTPServer(port: number, secure: boolean, options: ServerOptions) {
  * and a Fastify fallback is registered for the port, it delegates via inject() (see injectToFastify),
  * mirroring the Bun path — so legacy Fastify routes work behind uWS too.
  */
-function makeUwsHandler(port: number | string, isOperationsServer: boolean, requestQueueLimit?: number) {
+export function makeUwsHandler(port: number | string, isOperationsServer: boolean, requestQueueLimit?: number) {
 	// Build a fresh response descriptor rather than mutating what the chain returned: a handler may
 	// return a WHATWG `Response` (read-only `status`/`body` accessors), which the Bun path also never
 	// mutates. `headers` is normalized in place the same way the Bun path does.
@@ -983,6 +1024,7 @@ function makeUwsHandler(port: number | string, isOperationsServer: boolean, requ
 					if (Array.isArray(v)) respHeaders.set(k, k.toLowerCase() === 'set-cookie' ? v : v.join(', '));
 					else respHeaders.set(k, String(v));
 				}
+				mergeChainHeadersIntoFallback(headers, respHeaders);
 				if (universalHeaders.length > 0) applyUniversalHeaders(respHeaders);
 				logHttpRequest(request, injectResult.statusCode, requestId, performance.now() - startTime);
 				const responseStream = injectResult.stream();
@@ -1006,6 +1048,7 @@ function makeUwsHandler(port: number | string, isOperationsServer: boolean, requ
 			}
 			logHttpRequest(request, 404, requestId, performance.now() - startTime);
 			const notFoundHeaders = new Headers({ 'content-type': 'text/plain' });
+			mergeChainHeadersIntoFallback(headers, notFoundHeaders);
 			if (universalHeaders.length > 0) applyUniversalHeaders(notFoundHeaders);
 			return { status: 404, headers: notFoundHeaders, body: 'Not found\n' };
 		}
@@ -1180,10 +1223,11 @@ function getBunHTTPServer(port: number, secure: boolean, options: ServerOptions)
 						// Delegate to the fallback server (e.g. Fastify) via node:http compatibility.
 						// We create a Node-compatible IncomingMessage/ServerResponse and emit 'request'
 						// on the fallback server, then capture the response.
-						return await bunDelegateToNodeServer(fallbackServer, webRequest, request);
+						return await bunDelegateToNodeServer(fallbackServer, webRequest, request, response.headers);
 					}
 					logHttpRequest(request, 404, requestId, performance.now() - startTime);
 					const notFoundHeaders = new globalThis.Headers();
+					mergeChainHeadersIntoFallback(response.headers, notFoundHeaders);
 					if (universalHeaders.length > 0) applyUniversalHeaders(notFoundHeaders);
 					return new Response('Not found\n', { status: 404, headers: notFoundHeaders });
 				}
@@ -1367,6 +1411,10 @@ let fastifyInstances: Record<string | number, any> = {};
 export function registerFastifyInstance(port: string | number, instance: any) {
 	fastifyInstances[port] = instance;
 }
+
+export function registerFallbackServer(port: string | number, listener: any) {
+	fallbackServers[port] = listener;
+}
 const INTERNAL_USER_HEADER = 'x-harper-internal-pre-auth-user';
 
 /**
@@ -1397,10 +1445,11 @@ function injectToFastify(
 	return fastify.inject({ method: req.method, url: req.url, headers, payload: req.body, payloadAsStream: true });
 }
 
-async function bunDelegateToNodeServer(
+export async function bunDelegateToNodeServer(
 	nodeServer: any,
 	webRequest: globalThis.Request,
-	bunRequest?: any
+	bunRequest?: any,
+	chainHeaders?: any
 ): Promise<Response> {
 	// Check if there's a Fastify instance registered for this port (preferred path)
 	for (const port in fallbackServers) {
@@ -1421,13 +1470,19 @@ async function bunDelegateToNodeServer(
 			});
 			const webHeaders = new globalThis.Headers();
 			for (const [k, v] of Object.entries(injectResult.headers)) {
-				if (v != null) webHeaders.set(k, Array.isArray(v) ? v.join(', ') : String(v));
+				if (v == null) continue;
+				if (Array.isArray(v)) {
+					if (k.toLowerCase() === 'set-cookie') {
+						for (const single of v) webHeaders.append(k, String(single));
+					} else webHeaders.set(k, v.join(', '));
+				} else webHeaders.set(k, String(v));
 			}
 			// Propagate Connection: close so Bun closes the TCP connection after this response,
 			// preventing stale keep-alive sockets from causing silent hangs on subsequent requests.
 			if (webRequest.headers.get('connection')?.toLowerCase() === 'close') {
 				webHeaders.set('connection', 'close');
 			}
+			mergeChainHeadersIntoFallback(chainHeaders, webHeaders);
 			if (universalHeaders.length > 0) applyUniversalHeaders(webHeaders);
 			const responseStream = injectResult.stream();
 			// Event-stream responses (MCP SSE) must reach the client incrementally — return
@@ -1462,6 +1517,7 @@ async function bunDelegateToNodeServer(
 	}
 	// No Fastify instance found — return 404
 	const notFoundHeaders = new globalThis.Headers();
+	mergeChainHeadersIntoFallback(chainHeaders, notFoundHeaders);
 	if (universalHeaders.length > 0) applyUniversalHeaders(notFoundHeaders);
 	return new Response('Not found\n', { status: 404, headers: notFoundHeaders });
 }
@@ -1470,14 +1526,39 @@ type SerializedRoute = { host?: string; urlPath?: string; order: string[] };
 // Resolved order captured at chain-build time, keyed identically to httpChain/upgradeChains/
 // websocketChains (kind → port → routes). Reporting the stored build-time order rather than
 // recomputing from current responders guarantees get_status matches the callback chain actually
-// serving that port — including cases where a late `port: 'all'` registration rebuilds only the
-// 'all' chain and leaves a concrete port's chain (and this description) unchanged (#1573).
+// serving that port: buildChains() writes both in the same pass, so a description can never
+// describe an order the port isn't running (#1573).
 const resolvedChainDescriptions: Record<string, Record<string, SerializedRoute[]>> = {
 	http: {},
 	upgrade: {},
 	websocket: {},
 };
 
+// Preserve the original port type because route selection uses strict equality.
+const builtChainPorts: Record<string, Map<string, number | string>> = {
+	http: new Map(),
+	upgrade: new Map(),
+	websocket: new Map(),
+};
+
+// A late registration on 'all' must rebuild every already-bound concrete port (#2418).
+function buildChains(
+	chains: Record<string, Function>,
+	listeners: HttpEntry[],
+	port: number | string,
+	requestArgIndex: number = 0,
+	kind: string = 'http'
+) {
+	builtChainPorts[kind].set(String(port), port);
+	if (port !== 'all') {
+		chains[port] = makeCallbackChain(listeners, port, requestArgIndex, kind);
+		return;
+	}
+	for (const builtPort of builtChainPorts[kind].values()) {
+		chains[builtPort] = makeCallbackChain(listeners, builtPort, requestArgIndex, kind);
+	}
+}
+
 function makeCallbackChain(
 	responders: typeof httpResponders,
 	portNum: number | string,
@@ -1603,7 +1684,7 @@ function onUpgrade(listener: UpgradeListener, options: UpgradeOptions) {
 			host: options?.host || undefined,
 		};
 		upgradeListeners[options?.runFirst ? 'unshift' : 'push'](entry);
-		upgradeChains[port] = makeCallbackChain(upgradeListeners, port, 0, 'upgrade');
+		buildChains(upgradeChains, upgradeListeners, port, 0, 'upgrade');
 	}
 }
 
@@ -1742,10 +1823,10 @@ function onWebSocket(listener: (ws: WebSocket) => void, options: OnWebSocketOpti
 			host: options?.host || undefined,
 		};
 		websocketListeners[options?.runFirst ? 'unshift' : 'push'](wsEntry);
-		websocketChains[port] = makeCallbackChain(websocketListeners, port, 1, 'websocket');
+		buildChains(websocketChains, websocketListeners, port, 1, 'websocket');
 
 		// mqtt doesn't invoke the http handler so this needs to be here to load up the http chains.
-		httpChain[port] = makeCallbackChain(httpResponders, port);
+		buildChains(httpChain, httpResponders, port);
 	}
 
 	return servers;

package/server/itc/serverHandlers.js

@@ -12,7 +12,7 @@ const harperBridge =
 	require('../../dataLayer/harperBridge/harperBridge.ts');
 const process = require('process');
 const { isMainThread, threadId, workerData } = require('node:worker_threads');
-const { resetDatabases, closeDatabase } = require('../../resources/databases.ts');
+const { resetDatabases, closeDatabase, reloadBranchAt } = require('../../resources/databases.ts');
 
 /**
  * This object/functions are passed to the ITC client instance and dynamically added as event handlers.
@@ -76,6 +76,13 @@ schemaHandler.addListener = function (listener) {
  */
 async function syncSchemaMetadata(msg) {
 	try {
+		// A change to a scope-private branch is not a change to any database in the global map, so the
+		// rescan below has nothing to find; a thread holding that branch open reloads it instead.
+		if (msg.branchPath) {
+			// No write barrier here: the symbol-keyed put below has never been one (harper#2522).
+			reloadBranchAt(msg.branchPath);
+			return;
+		}
 		// TODO: Eventually should indicate which database/table changed so we don't have to scan everything
 		let databases = resetDatabases();
 		if (msg.table && msg.database)

package/server/jobs/jobProcess.ts

@@ -9,6 +9,8 @@ import harperLogger from '../../utility/logging/harper_logger.ts';
 import * as globalSchema from '../../utility/globalSchema.ts';
 import * as user from '../../security/user.ts';
 import * as serverUtils from '../serverHelpers/serverUtilities.ts';
+import { runWithDispatchedOperation } from '../serverHelpers/operationAuthorizationState.ts';
+import { stripSuppliedParsedSqlObject } from '../serverHelpers/requestSanitization.ts';
 import moment from 'moment';
 import * as jobs from './jobs.ts';
 import { cloneDeep } from 'lodash';
@@ -54,12 +56,16 @@ const JOB_ID = JOB_NAME.substring(4);
 			throw new Error('Did not find job request in hdb_job table, unable to proceed');
 		}
 		request = cloneDeep(request);
+		// The worker re-enters from the persisted row rather than re-dispatching, so a row queued before
+		// the dispatch-time strip existed — or written directly — is sanitized here.
+		stripSuppliedParsedSqlObject(request);
 
 		const operation = serverUtils.getOperationFunction(request);
 		harperLogger.trace('Running operation:', request.operation, 'for job', JOB_ID);
 
-		// Run the job operation.
-		const results = await operation.job_operation_function(request);
+		const results = await runWithDispatchedOperation(request.operation, () =>
+			operation.job_operation_function(request)
+		);
 		harperLogger.trace('Result from job:', JOB_ID, results);
 
 		jobObj.status = hdbTerms.JOB_STATUS_ENUM.COMPLETE;

package/server/jobs/jobs.ts

@@ -11,6 +11,7 @@ import * as search from '../../dataLayer/search.ts';
 import Search_Object from '../../dataLayer/SearchObject.ts';
 import searchByHashObj from '../../dataLayer/SearchByHashObject.ts';
 import SQL_Search_Object from '../../dataLayer/SqlSearchObject.ts';
+import { runWithOperationAuthorizationBypass } from '../serverHelpers/operationAuthorizationState.ts';
 import * as hdbTerms from '../../utility/hdbTerms.ts';
 import JobObject from './JobObject.ts';
 import UpdateObject from '../../dataLayer/UpdateObject.ts';
@@ -249,7 +250,9 @@ export async function getJobsInDateRange(jsonBody: any) {
 			const hdbSql = require('../../sqlTranslator/index');
 			pSqlEvaluate = promisify(hdbSql.evaluateSQL);
 		}
-		return await pSqlEvaluate(sqlSearchObj);
+		// Harper's own statement, not the caller's. A carrier is not equivalent here — see DESIGN.md.
+		// Wraps only this statement: a later caller-dependent one must not inherit the bypass.
+		return await runWithOperationAuthorizationBypass(true, () => pSqlEvaluate(sqlSearchObj));
 	} catch (e) {
 		log.error(
 			`there was a problem searching for jobs from date ${jsonBody.from_date} to date ${jsonBody.to_date} ${e}`

package/server/liveSubscriptionAuth.ts

@@ -23,8 +23,24 @@ interface LiveSubscription {
 	authExpiresAt?: number;
 	/** Returns true if the principal is still authorized for this subscription. */
 	recheck: () => Promise<boolean>;
-	/** Stop delivery and tear down the subscription. */
-	terminate: () => void;
+	/** Stop delivery and tear down. May be async (e.g. a shared-feed refcount release). */
+	terminate: () => void | Promise<void>;
+}
+
+function errorMessage(error: unknown): string {
+	try {
+		return error instanceof Error ? error.message : String(error);
+	} catch {
+		return '<error message unavailable>';
+	}
+}
+
+function safeLog(log: ((message: string) => void) | undefined, message: string): void {
+	try {
+		log?.(message);
+	} catch {
+		/* a broken logger must not turn a contained failure into a new one */
+	}
 }
 
 const registry = new Set<LiveSubscription>();
@@ -32,9 +48,15 @@ let sweepTimer: any = null;
 let itcListenerInstalled = false;
 let sweeping = false;
 
+const NOOP_HANDLE = { unregister: () => {} };
+
+function triggerSweep(): void {
+	void sweep().catch((error) => safeLog(hdbLogger.error, `liveSubscriptionAuth: sweep failed: ${errorMessage(error)}`));
+}
+
 function ensureStarted(): void {
 	if (!sweepTimer) {
-		sweepTimer = setInterval(() => void sweep(), RECHECK_INTERVAL_MS);
+		sweepTimer = setInterval(triggerSweep, RECHECK_INTERVAL_MS);
 		// don't keep the worker alive solely for the recheck timer
 		sweepTimer.unref?.();
 	}
@@ -44,7 +66,7 @@ function ensureStarted(): void {
 			// user/role cache before invoking listeners, so recheck() observes the new permissions.
 			const handlers = require('./itc/serverHandlers.js');
 			if (handlers?.userHandler?.addListener) {
-				handlers.userHandler.addListener(() => void sweep());
+				handlers.userHandler.addListener(triggerSweep);
 				itcListenerInstalled = true;
 			}
 		} catch (error) {
@@ -61,32 +83,49 @@ function stopIfIdle(): void {
 }
 
 /**
- * Register a live subscription for continuous re-authorization. The returned subscription's normal
- * teardown (end()/close) automatically unregisters it, so callers don't need to.
+ * Register a live subscription for continuous re-authorization.
+ *
+ * Without `revoke`, teardown is end()/close()/emit('close') on `subscription`, and the
+ * subscription's own teardown unregisters the entry so callers need not.
+ *
+ * With `revoke`, that becomes the teardown and `subscription` is never touched — no `end` wrapping,
+ * no 'close' listener — because a feed shared by many subscribers must stay revocable per subscriber,
+ * so the registry can neither own the shared object nor let every registrant mutate it. A `revoke`
+ * caller owns the entry's lifetime: nothing detects a leaked registration, and a forgotten
+ * `unregister()` degrades sweep latency for every other tracked subscriber. It also owns teardown
+ * recovery — the entry is untracked before `revoke` runs and `revoke` is invoked exactly once, so one
+ * that throws, rejects or never settles is logged and never retried. A `recheck` shared across
+ * subscribers must not mutate state shared across them: `registerLiveSubscriptionForContext` in
+ * resources/Resource.ts mutates `context.user`, which is safe only while each context has exactly
+ * one subscriber.
  */
-export function registerLiveSubscription(opts: {
-	subscription: any;
-	username: string;
-	authExpiresAt?: number;
-	recheck: () => Promise<boolean>;
-}): void {
-	const { subscription, username, authExpiresAt, recheck } = opts;
-	if (!subscription || typeof subscription !== 'object' || subscription.closed) return;
+export function registerLiveSubscription(
+	opts: {
+		username: string;
+		authExpiresAt?: number;
+		recheck: () => Promise<boolean>;
+	} & (
+		| { subscription: any; revoke?: undefined }
+		// requiring one of the two modes stops a caller that supplies neither from type-checking
+		// into a silent no-op registration; a revoke-only registrant may own no subscription object
+		| { subscription?: any; revoke: () => void | Promise<void> }
+	)
+): { unregister: () => void } {
+	const { subscription, username, authExpiresAt, recheck, revoke } = opts;
+	if (!revoke && (!subscription || typeof subscription !== 'object' || subscription.closed)) return NOOP_HANDLE;
 
 	const entry: LiveSubscription = {
 		username,
 		authExpiresAt,
 		recheck,
-		terminate: () => {
-			// end() removes the subscription from the broadcast loop and closes its iterable queue.
-			try {
+		terminate:
+			revoke ??
+			(() => {
+				// end() removes the subscription from the broadcast loop and closes its iterable queue.
 				if (subscription.end) subscription.end();
 				else if (subscription.close) subscription.close();
 				else subscription.emit?.('close');
-			} catch {
-				/* ignore */
-			}
-		},
+			}),
 	};
 	registry.add(entry);
 
@@ -94,51 +133,90 @@ export function registerLiveSubscription(opts: {
 		registry.delete(entry);
 		stopIfIdle();
 	};
-	// Both transports ultimately call end() on normal teardown (MQTT unsubscribe/disconnect; SSE close
-	// is wired to end()); wrap it so a closed stream never leaks a registry entry. Also listen for
-	// 'close' to cover any iterable that closes without an end().
-	const originalEnd = typeof subscription.end === 'function' ? subscription.end.bind(subscription) : null;
-	if (originalEnd) {
-		subscription.end = function (...args: any[]) {
-			unregister();
-			return originalEnd(...args);
-		};
+
+	if (!revoke) {
+		// Both transports ultimately call end() on normal teardown (MQTT unsubscribe/disconnect; SSE close
+		// is wired to end()); wrap it so a closed stream never leaks a registry entry. Also listen for
+		// 'close' to cover any iterable that closes without an end(). Skipped when `revoke` is supplied:
+		// the caller owns unregistration, and a subscription shared by many subscribers must not be
+		// mutated once per registrant.
+		const originalEnd = typeof subscription.end === 'function' ? subscription.end.bind(subscription) : null;
+		if (originalEnd) {
+			subscription.end = function (...args: any[]) {
+				unregister();
+				return originalEnd(...args);
+			};
+		}
+		subscription.on?.('close', unregister);
 	}
-	subscription.on?.('close', unregister);
 
 	ensureStarted();
+	return { unregister };
+}
+
+/** Untrack first: a `terminate` that hangs or fails must not wedge the sweep or be re-entered by a later one. */
+function terminateEntry(
+	entry: LiveSubscription,
+	reason: string,
+	notice: ((message: string) => void) | undefined = hdbLogger.info
+): void {
+	registry.delete(entry);
+	safeLog(notice, `liveSubscriptionAuth: revoking subscription for ${entry.username} (${reason})`);
+	const failed = (error: unknown) =>
+		safeLog(
+			hdbLogger.error,
+			`liveSubscriptionAuth: terminate failed for ${entry.username} (${reason}): ${errorMessage(error)}`
+		);
+	try {
+		// an async terminate's rejection would otherwise surface as an unhandled rejection on the timer's stack
+		Promise.resolve(entry.terminate()).catch(failed);
+	} catch (error) {
+		failed(error);
+	}
 }
 
 async function sweep(): Promise<void> {
 	if (sweeping) return; // a slow recheck must not overlap with the next tick/event
 	sweeping = true;
+	// the per-subscriber lines are info, which the shipped default (logging.level: warn) drops; one
+	// aggregate keeps the pass visible without making a mass role change a warn per subscriber
+	const revokedByReason = new Map<string, number>();
+	const countRevocation = (reason: string) => revokedByReason.set(reason, (revokedByReason.get(reason) ?? 0) + 1);
 	try {
-		const now = Date.now();
-		for (const entry of registry) {
+		// snapshot bounds the pass to entries present at its start; the has() guards cover the rest
+		for (const entry of Array.from(registry)) {
+			if (!registry.has(entry)) continue;
 			try {
-				const expired = entry.authExpiresAt != null && now >= entry.authExpiresAt * 1000;
+				const expired = entry.authExpiresAt != null && Date.now() >= entry.authExpiresAt * 1000;
 				const stillAuthorized = expired ? false : await entry.recheck();
-				const revoke = expired || !stillAuthorized;
-				if (revoke) {
-					registry.delete(entry);
-					entry.terminate();
+				if (!registry.has(entry)) continue;
+				if (expired || !stillAuthorized) {
+					const reason = expired ? 'token expired' : 'no longer authorized';
+					terminateEntry(entry, reason);
+					countRevocation(reason);
 				}
 			} catch (error) {
 				// fail closed: if authorization can't be confirmed, revoke
-				registry.delete(entry);
-				try {
-					entry.terminate();
-				} catch {
-					/* ignore */
+				if (registry.has(entry)) {
+					terminateEntry(entry, `recheck error: ${errorMessage(error)}`, hdbLogger.warn);
+					countRevocation('recheck error');
 				}
-				hdbLogger.warn?.(
-					`liveSubscriptionAuth: revoked subscription for ${entry.username} after recheck error: ${(error as Error).message}`
-				);
 			}
 		}
 	} finally {
 		sweeping = false;
 		stopIfIdle();
+		if (revokedByReason.size > 0) {
+			let total = 0;
+			for (const count of revokedByReason.values()) total += count;
+			const breakdown = Array.from(revokedByReason, ([reason, count]) => `${reason}: ${count}`).join(', ');
+			safeLog(
+				hdbLogger.warn,
+				// "revoking", like the per-subscriber line: teardown is dispatched, not awaited, and a
+				// failure surfaces on its own error line
+				`liveSubscriptionAuth: revoking ${total} live subscription${total === 1 ? '' : 's'} (${breakdown})`
+			);
+		}
 	}
 }
 
@@ -146,3 +224,8 @@ async function sweep(): Promise<void> {
 export function _liveSubscriptionCount(): number {
 	return registry.size;
 }
+
+/** Test-only: run a sweep synchronously, bypassing the interval/ITC triggers. */
+export function _sweepNow(): Promise<void> {
+	return sweep();
+}

package/server/loadRootComponents.js

@@ -1,19 +1,62 @@
 const { isMainThread } = require('worker_threads');
 const { getTables } = require('../resources/databases.ts');
-const { loadComponentDirectories, loadComponent } = require('../components/componentLoader.ts');
+const { loadComponentDirectories, loadComponent, readyComponentModules } = require('../components/componentLoader.ts');
 const { resetResources } = require('../resources/Resources.ts');
 const configUtils = require('../config/configUtils.ts');
 const { dirname } = require('path');
 const { loadCertificates } = require('../security/keys.ts');
-const { installApplications } = require('../components/Application.ts');
+const { installApplications, recoverInterruptedActivations } = require('../components/Application.ts');
 const { errorForLog } = require('../utility/logging/harper_logger.ts');
+const { CONFIG_PARAMS } = require('../utility/hdbTerms.ts');
 
 let loadedComponents = new Map();
+
+/**
+ * Map every component directory to the reason activation recovery could not run. Used only when the scan
+ * fails globally, where the safe answer is "none of these are known-good" rather than "all of these are".
+ */
+async function failEveryComponentClosed(cause) {
+	const failures = new Map();
+	const reason = cause instanceof Error ? cause : new Error(`Activation recovery could not run: ${String(cause)}`);
+	try {
+		const { readdir } = require('node:fs/promises');
+		const componentsRoot = configUtils.getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
+		for (const entry of await readdir(componentsRoot, { withFileTypes: true })) {
+			if (!entry.name.startsWith('.') && (entry.isDirectory() || entry.isSymbolicLink())) {
+				failures.set(entry.name, reason);
+			}
+		}
+	} catch (error) {
+		console.error(errorForLog(error));
+	}
+	return failures;
+}
 /**
  * This is main entry point for loading the main set of global server modules that power Harper.
  * @returns {Promise<void>}
  */
 async function loadRootComponents(isWorkerThread = false) {
+	// Interrupted activations are settled FIRST, before installApplications() — not merely before the
+	// component scan. installApplications() installs whatever the root config names, so a candidate left
+	// half-swapped by a crash has to be resolved before that runs, or boot reinstalls over it. Failures are
+	// per component and returned, not thrown: the loader fails those components closed and loads the rest.
+	// Undefined, not an empty Map: a worker skips this branch entirely, and handing the loader an empty map
+	// would assert "nothing is unreconciled" on a thread that never checked. The loader treats undefined as
+	// "no verdict from boot" instead.
+	let interruptedActivationFailures;
+	try {
+		if (isMainThread && !process.env.HARPER_SAFE_MODE) {
+			interruptedActivationFailures = await recoverInterruptedActivations(
+				configUtils.getConfigPath(CONFIG_PARAMS.COMPONENTSROOT)
+			);
+		}
+	} catch (error) {
+		// The scan itself failed, so WHICH components are unsettled is unknown — loading them all would
+		// defeat the fail-closed contract this pass exists for. Every component present is failed closed
+		// instead, and a later reload cycle retries once the cause is gone.
+		console.error(errorForLog(error));
+		interruptedActivationFailures = await failEveryComponentClosed(error);
+	}
 	try {
 		if (isMainThread && !process.env.HARPER_SAFE_MODE) await installApplications();
 	} catch (error) {
@@ -33,13 +76,12 @@ async function loadRootComponents(isWorkerThread = false) {
 	});
 	if (!process.env.HARPER_SAFE_MODE) {
 		// once the global plugins are loaded, we now load all the CF and run applications (and their components)
-		await loadComponentDirectories(loadedComponents, resources);
-	}
-	let allReady = [];
-	for (let [serverModule] of loadedComponents) {
-		if (serverModule.ready) allReady.push(serverModule.ready());
+		const readyComponentPromises = new WeakMap();
+		await loadComponentDirectories(loadedComponents, resources, readyComponentPromises, interruptedActivationFailures);
+		await readyComponentModules(loadedComponents.keys(), readyComponentPromises);
+		return;
 	}
-	if (allReady.length > 0) await Promise.all(allReady);
+	await readyComponentModules(loadedComponents.keys());
 }
 
 module.exports.loadRootComponents = loadRootComponents;