@harperfast/harper 5.2.0 → 5.2.2

package/components/deployLifecycle.ts

@@ -9,25 +9,39 @@
 //
 // This module solves that by broadcasting a structured "deploy:start" /
 // "deploy:end" signal to every Harper thread. Each thread's deploy emitter
-// fires locally so Scopes (and any plugin subscribers) can react: Scopes close
-// their EntryHandlers on start, recreate them on end. The result is that
+// fires locally so Scopes (and any plugin subscribers) can react: Scopes pause
+// their EntryHandlers on start and resume them on end. The result is that
 // during a deploy, no watcher fires events for the deployed component, and
-// after the deploy the recreated watcher emits one fresh add per current file
-// — collapsed by the existing restart debounce into a single restart.
+// after the deploy the watcher compares the new tree with its retained snapshot
+// and emits the same logical add/change/unlink events consumers saw before #1806.
 //
-// State sharing across threads is intentionally narrow: the local Set of
-// in-flight component names is rebuilt from the broadcast stream. Plugins that
-// need to gate their own work on deploy progress import `deployLifecycle` and
-// listen to the events directly.
+// State sharing across threads is intentionally narrow: local deployment-owner
+// records are rebuilt from the broadcast stream and reclaimed when their owner
+// thread exits. Plugins that need to gate their own work on deploy progress
+// import `deployLifecycle` and listen to the events directly.
 
 import { EventEmitter } from 'node:events';
-import { isMainThread } from 'node:worker_threads';
-import { broadcast, broadcastWithAcknowledgement, onMessageByType } from '../server/threads/manageThreads.js';
+import { randomUUID } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
+import { isMainThread, threadId } from 'node:worker_threads';
+import {
+	broadcast,
+	broadcastWithAcknowledgement,
+	isThreadRunning,
+	onMessageByType,
+	onThreadExit,
+} from '../server/threads/manageThreads.js';
+import harperLogger from '../utility/logging/harper_logger.ts';
 
 const DEPLOY_LIFECYCLE_MSG = 'harper:deploy:lifecycle';
 
 export type DeployPhase = 'start' | 'end';
-export type DeployLifecycleEvent = { name: string; phase: DeployPhase };
+export type DeployLifecycleEvent = {
+	name: string;
+	phase: DeployPhase;
+	deploymentId?: string;
+	ownerThreadId?: number;
+};
 
 type DeployLifecycleEventsMap = {
 	'deploy:start': [componentName: string];
@@ -35,44 +49,109 @@ type DeployLifecycleEventsMap = {
 };
 
 class DeployLifecycle extends EventEmitter<DeployLifecycleEventsMap> {
-	// Ref-counts in-flight deploys per component. Concurrent deploys of the
-	// same name (rare, but possible if an operator queues two before the first
-	// resolves) overlap: 0→1 fires deploy:start, 1→0 fires deploy:end. Any
-	// intermediate increment/decrement is silent so watchers aren't toggled
-	// out from under an active deploy by a peer ending early.
-	#inFlight = new Map<string, number>();
+	#deployments = new Map<string, { name: string; ownerThreadId: number }>();
+	#byComponent = new Map<string, Set<string>>();
+	#deadOwners = new Set<number>();
+	#legacyDeployments = new Map<string, string[]>();
+	#legacySequence = 0;
 
 	isDeployInFlight(componentName: string): boolean {
-		return (this.#inFlight.get(componentName) ?? 0) > 0;
+		return (this.#byComponent.get(componentName)?.size ?? 0) > 0;
 	}
 
 	// Process a deploy lifecycle event in-process. Called both from the
 	// broadcast receiver (for events originating on another thread) and from
 	// the broadcaster (so the originating thread also reacts locally).
 	_handle(event: DeployLifecycleEvent): void {
-		const current = this.#inFlight.get(event.name) ?? 0;
+		const ownerThreadId = event.ownerThreadId ?? threadId;
+		let deploymentId = event.deploymentId;
 		if (event.phase === 'start') {
-			this.#inFlight.set(event.name, current + 1);
-			if (current === 0) this.emit('deploy:start', event.name);
-		} else {
-			const next = Math.max(0, current - 1);
-			if (next === 0) {
-				this.#inFlight.delete(event.name);
-				if (current > 0) this.emit('deploy:end', event.name);
-			} else {
-				this.#inFlight.set(event.name, next);
+			if (this.#deadOwners.has(ownerThreadId)) return;
+			if (!deploymentId) {
+				deploymentId = `legacy:${threadId}:${++this.#legacySequence}`;
+				const legacy = this.#legacyDeployments.get(event.name) ?? [];
+				legacy.push(deploymentId);
+				this.#legacyDeployments.set(event.name, legacy);
+			}
+			if (this.#deployments.has(deploymentId)) return;
+			const active = this.#byComponent.get(event.name) ?? new Set<string>();
+			this.#deployments.set(deploymentId, { name: event.name, ownerThreadId });
+			this.#byComponent.set(event.name, active);
+			active.add(deploymentId);
+			if (active.size === 1) this.#emitSafely('deploy:start', event.name);
+			return;
+		}
+		if (!deploymentId) deploymentId = this.#legacyDeployments.get(event.name)?.pop();
+		if (deploymentId) this.#removeDeployment(deploymentId);
+	}
+
+	_reclaimOwner(ownerThreadId: number): void {
+		this.#deadOwners.add(ownerThreadId);
+		for (const [deploymentId, deployment] of this.#deployments) {
+			if (deployment.ownerThreadId === ownerThreadId) this.#removeDeployment(deploymentId);
+		}
+	}
+
+	async _reclaimOwnerAfterTermination(
+		ownerThreadId: number,
+		waitForTermination: (ownerThreadId: number) => Promise<void> = waitForOwnerTermination
+	): Promise<void> {
+		if (ownerThreadId === 0 || ownerThreadId === threadId) return;
+		this.#deadOwners.add(ownerThreadId);
+		await waitForTermination(ownerThreadId);
+		this._reclaimOwner(ownerThreadId);
+	}
+
+	#removeDeployment(deploymentId: string): void {
+		const deployment = this.#deployments.get(deploymentId);
+		if (!deployment) return;
+		this.#deployments.delete(deploymentId);
+		const active = this.#byComponent.get(deployment.name);
+		if (!active) return;
+		active.delete(deploymentId);
+		if (active.size === 0) {
+			this.#byComponent.delete(deployment.name);
+			this.#emitSafely('deploy:end', deployment.name);
+		}
+	}
+
+	#emitSafely(event: 'deploy:start' | 'deploy:end', componentName: string): void {
+		for (const listener of this.rawListeners(event)) {
+			try {
+				listener.call(this, componentName);
+			} catch (error) {
+				harperLogger.error(`Listener for ${event} threw for ${componentName}:`, error);
 			}
 		}
 	}
 
 	// Test-only: clear in-flight state without firing events.
 	_clearForTests(): void {
-		this.#inFlight.clear();
+		this.#deployments.clear();
+		this.#byComponent.clear();
+		this.#deadOwners.clear();
+		this.#legacyDeployments.clear();
+		this.#legacySequence = 0;
 	}
 }
 
 export const deployLifecycle = new DeployLifecycle();
 
+async function waitForOwnerTermination(ownerThreadId: number): Promise<void> {
+	let timeoutReported = false;
+	for (;;) {
+		try {
+			if (!(await isThreadRunning(ownerThreadId))) return;
+		} catch (error) {
+			if (!timeoutReported) {
+				timeoutReported = true;
+				harperLogger.warn(`Retrying liveness check for exited thread ${ownerThreadId}:`, error);
+			}
+		}
+		await delay(25, undefined, { ref: false });
+	}
+}
+
 let receiverInstalled = false;
 function ensureReceiver() {
 	if (receiverInstalled) return;
@@ -88,6 +167,11 @@ function ensureReceiver() {
 // suppress only the main thread's watchers and the worker watchers would keep
 // firing restart storms (harper#488).
 ensureReceiver();
+onThreadExit((deadThreadId: number) => {
+	void deployLifecycle
+		._reclaimOwnerAfterTermination(deadThreadId)
+		.catch((error) => harperLogger.error(`Could not reclaim deployments from thread ${deadThreadId}:`, error));
+});
 
 /**
  * Announce the start of a deploy for `componentName`. Awaits acknowledgement
@@ -98,9 +182,10 @@ ensureReceiver();
  * component compose correctly (each call must be paired with exactly one
  * broadcastDeployEnd).
  */
-export async function broadcastDeployStart(componentName: string): Promise<void> {
+export async function broadcastDeployStart(componentName: string): Promise<string> {
 	ensureReceiver();
-	const event: DeployLifecycleEvent = { name: componentName, phase: 'start' };
+	const deploymentId = randomUUID();
+	const event: DeployLifecycleEvent = { name: componentName, phase: 'start', deploymentId, ownerThreadId: threadId };
 	deployLifecycle._handle(event); // local thread first
 	let timer: ReturnType<typeof setTimeout> | undefined;
 	try {
@@ -125,6 +210,7 @@ export async function broadcastDeployStart(componentName: string): Promise<void>
 	} finally {
 		if (timer) clearTimeout(timer);
 	}
+	return deploymentId;
 }
 
 /**
@@ -135,9 +221,9 @@ export async function broadcastDeployStart(componentName: string): Promise<void>
  * Must be called exactly once per matching broadcastDeployStart, even if the
  * deploy errored — typically from a `finally` block.
  */
-export function broadcastDeployEnd(componentName: string): void {
+export function broadcastDeployEnd(componentName: string, deploymentId: string): void {
 	ensureReceiver();
-	const event: DeployLifecycleEvent = { name: componentName, phase: 'end' };
+	const event: DeployLifecycleEvent = { name: componentName, phase: 'end', deploymentId, ownerThreadId: threadId };
 	deployLifecycle._handle(event);
 	try {
 		void broadcast({ type: DEPLOY_LIFECYCLE_MSG, event }, false);

package/components/deploymentRecorder.ts

@@ -13,6 +13,7 @@ import { randomUUID } from 'node:crypto';
 import { createHash } from 'node:crypto';
 import { Readable, Transform, pipeline } from 'node:stream';
 import { databases } from '../resources/databases.ts';
+import { transaction, contextStorage } from '../resources/transaction.ts';
 import { createBlob, isSaving, deleteBlob, BLOB_UNAVAILABLE_STATUS } from '../resources/blob.ts';
 import * as terms from '../utility/hdbTerms.ts';
 import type { CredentialReference } from './secretOperations.ts';
@@ -68,6 +69,39 @@ interface CreateOptions {
 	// no-custody transient token.
 	credentials?: CredentialReference[] | null;
 	emitter?: ProgressEmitter;
+	// Open-transaction budget (ms) for the payload-ingest write, below. Mirrors the
+	// `deployment_timeout` operation parameter already used for the peer-side row/blob waits
+	// (DEFAULT_AWAIT_ROW_TIMEOUT_MS) so operators have one existing knob for all of a deploy's
+	// size-driven timeouts, rather than a new config surface for this one.
+	ingestTimeoutMs?: number;
+}
+
+/**
+ * Run `write()` in a fresh transaction while preserving audit attribution. Recorder writes must
+ * commit independently of a caller's transaction so an older staged row cannot overwrite them;
+ * authorization is enforced by the deploy operation rather than these internal writes.
+ *
+ * `timeoutBudget` is sticky across RocksDB `getReadTxn()` calls, so reads inside `write()` cannot
+ * silently restore the generic default. The transaction uses the larger of this budget and the
+ * configured global limit.
+ */
+function withIsolatedTransaction<T>(write: () => Promise<T>, timeoutMs?: number): Promise<T> {
+	const ambient = contextStorage.getStore();
+	const context = {
+		user: ambient?.user,
+		originatingOperation: ambient?.originatingOperation,
+		session: ambient?.session,
+		signal: ambient?.signal,
+	};
+	return transaction(context, (txn) => {
+		if (timeoutMs != null) {
+			// TODO(harper#2057): only extends the RocksDB DatabaseTransaction head; on
+			// HARPER_STORAGE_ENGINE=lmdb, Table.txnForContext() chains a separate LMDBTransaction
+			// that this budget doesn't reach.
+			txn.timeoutBudget = timeoutMs;
+		}
+		return write();
+	});
 }
 
 export class DeploymentRecorder {
@@ -78,10 +112,13 @@ export class DeploymentRecorder {
 	private pendingPut: Promise<void> | null = null;
 	private dirty = false;
 	private sealed = false;
+	private flushSuppressed = false;
+	private readonly ingestTimeoutMs?: number;
 
-	private constructor(deploymentId: string, initial: Record<string, any>) {
+	private constructor(deploymentId: string, initial: Record<string, any>, ingestTimeoutMs?: number) {
 		this.deploymentId = deploymentId;
 		this.record = initial;
+		this.ingestTimeoutMs = ingestTimeoutMs;
 	}
 
 	static async create(options: CreateOptions): Promise<DeploymentRecorder> {
@@ -107,7 +144,7 @@ export class DeploymentRecorder {
 			credentials: options.credentials ?? null,
 			error: null,
 		};
-		const recorder = new DeploymentRecorder(deploymentId, record);
+		const recorder = new DeploymentRecorder(deploymentId, record, options.ingestTimeoutMs);
 		await recorder.put();
 		if (options.emitter) {
 			recorder.subscribeTo(options.emitter);
@@ -158,9 +195,8 @@ export class DeploymentRecorder {
 	// the record dirty; the chained continuation issues a follow-up put once the prior one
 	// settles. This keeps event_log writes O(1) puts per burst rather than O(N) per event.
 	private scheduleFlush(): void {
-		if (this.sealed) {
-			// Sealed: accumulate state in memory but don't write. finish() does the single
-			// terminal write. See seal() for why. The emitter still emits live SSE events.
+		if (this.sealed || this.flushSuppressed) {
+			// Accumulate state until finish() or the active ingest writes it without a competing put.
 			this.dirty = true;
 			return;
 		}
@@ -175,6 +211,7 @@ export class DeploymentRecorder {
 				this.scheduleFlush();
 			}
 		});
+		this.pendingPut.catch((error) => logger.warn?.('Failed to persist deployment progress', error));
 	}
 
 	/**
@@ -191,6 +228,26 @@ export class DeploymentRecorder {
 	 * materialized by the time they reach us, so they take the simpler buffer path.
 	 */
 	async ingestPayload(source: Readable | Buffer | string): Promise<void> {
+		this.flushSuppressed = true;
+		try {
+			while (this.pendingPut) {
+				try {
+					await this.pendingPut;
+				} catch {
+					/* already logged; the ingest write carries the current row state */
+				}
+			}
+			await this.ingestPayloadWithoutFlush(source);
+		} finally {
+			this.flushSuppressed = false;
+			if (this.dirty && !this.sealed) {
+				this.dirty = false;
+				this.scheduleFlush();
+			}
+		}
+	}
+
+	private async ingestPayloadWithoutFlush(source: Readable | Buffer | string): Promise<void> {
 		const hash = createHash('sha256');
 
 		// In-memory sources: the bytes are already resident, so hashing them and creating a
@@ -202,7 +259,10 @@ export class DeploymentRecorder {
 			this.record.payload_blob = createBlob(buffer, { type: 'application/gzip' });
 			this.record.payload_hash = hash.digest('hex');
 			this.record.payload_size = buffer.length;
-			await this.put();
+			// Same blob-durability gate as the streaming path below applies here too (the commit
+			// waits for the buffer to land on disk), so a large base64-in-JSON/CBOR body needs the
+			// same extended budget.
+			await this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 			return;
 		}
 
@@ -263,6 +323,16 @@ export class DeploymentRecorder {
 		});
 		const blob = createBlob(tap, { type: 'application/gzip' });
 		this.record.payload_blob = blob;
+		// This row's commit is gated on the blob's durable file write (resources/blob.ts,
+		// startPreCommitBlobsForRecord's local-write branch) so a peer never sees a row that
+		// references a not-yet-durable blob — the write transaction is legitimately held open for
+		// as long as the payload takes to land on disk. The deploy system is explicitly designed
+		// to carry arbitrarily large, uncapped components (see the module comment above), so that
+		// duration scales with payload size, not with anything bounded. Run this one put() under
+		// an extended open-time budget instead of the generic storage.maxTransactionOpenTime meant
+		// for ordinary short requests (a large-but-otherwise-healthy ingest tripped that generic
+		// guard on a loaded self-hosted runner — see the Large Deploy Payload Test regression).
+		//
 		// Persisting the row encodes the blob, which synchronously starts the file write that
 		// drains the tap, so `isSaving(blob)` is set as soon as put() is *called*. Await the put,
 		// the source draining (tapDone), and the file flush (saving) together: handing all three
@@ -270,7 +340,7 @@ export class DeploymentRecorder {
 		// a large upload mid-stream (which rejects both tapDone and the blob write) fails the
 		// deploy loudly here instead of leaking an unhandledRejection. This also makes hash/size
 		// correctness independent of the put()/store write timing and of isSaving() being defined.
-		const putDone = this.put();
+		const putDone = this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 		const saving = isSaving(blob) ?? Promise.resolve();
 		try {
 			await Promise.all([putDone, tapDone, saving]);
@@ -288,7 +358,7 @@ export class DeploymentRecorder {
 				// ensures the null-reference write lands after the original reference write
 				// (so it wins) and the blob's file lock is released before we unlink it.
 				await Promise.allSettled([putDone, saving]);
-				await this.put();
+				await this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 				deleteBlob(blob);
 			} catch (cleanupError) {
 				logger.warn?.('Failed to clean up partial deploy payload blob', cleanupError);
@@ -301,7 +371,7 @@ export class DeploymentRecorder {
 		this.record.payload_size = blob.size ?? byteCount;
 		// Persist the now-known hash + size. The blob is already saved, so this re-put does
 		// not re-stream — saveBlob short-circuits on the existing fileId.
-		await this.put();
+		await this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 	}
 
 	/**
@@ -436,14 +506,14 @@ export class DeploymentRecorder {
 		return this.record;
 	}
 
-	private async put(): Promise<void> {
+	private async put(timeoutMs?: number): Promise<void> {
 		const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME];
 		if (!table) {
 			// Table missing means the upgrade directive hasn't run yet (or the table got dropped).
 			// We tolerate this — tracking is observability; the deploy itself must still succeed.
 			return;
 		}
-		await table.put(this.record);
+		await withIsolatedTransaction(() => table.put(this.record), timeoutMs);
 	}
 }
 
@@ -455,11 +525,32 @@ export class DeploymentRecorder {
 // via the `deployment_timeout` operation parameter.
 export const DEFAULT_AWAIT_ROW_TIMEOUT_MS = 120_000;
 
+// Default open-time budget for the origin's payload-ingest write transaction (ingestPayload,
+// below). Distinct from DEFAULT_AWAIT_ROW_TIMEOUT_MS (that one bounds a *replication* wait;
+// this one bounds *local disk I/O* for a component the deploy system explicitly supports at
+// arbitrarily large, uncapped sizes — see the module comment above and the large-payload
+// regression test). 10 minutes comfortably covers a multi-GB ingest even on a slow or loaded
+// disk while still being a bounded safety net, not a disabled one. Overridable via the same
+// `deployment_timeout` operation parameter as the peer-side waits, so operators sizing for
+// even larger components (or a known-slow disk) have one knob, not two.
+export const DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS = 600_000;
+
 export function coerceTimeoutMs(value: unknown, fallback: number): number {
 	const requested = Number(value);
 	return Number.isFinite(requested) && requested >= 0 ? requested : fallback;
 }
 
+// Floored at the default, never lowered by it: `deployment_timeout` also governs the
+// peer-side row/blob wait (awaitDeploymentRow), where 0 means "poll once, don't wait" — a
+// legitimate fast-fail setting for peers that must never also starve the origin's own disk
+// write down to 0ms.
+export function ingestTransactionTimeoutMs(deploymentTimeout: unknown): number {
+	return Math.max(
+		coerceTimeoutMs(deploymentTimeout, DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS),
+		DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS
+	);
+}
+
 /**
  * Peer-side helper — wait for the hdb_deployment row to arrive via table replication,
  * then return it. The row is committed on origin before `replicateOperation` is

package/components/mcp/tools/operations.ts

@@ -303,6 +303,12 @@ export function makeOperationToolHandler(operationName: string) {
 			operation: operationName,
 			hdb_user: context.user,
 		};
+		// Authorization bypass is internal dispatch state, never client-supplied tool
+		// arguments — mirrors handlePostRequest's stripping in serverHandlers.js. Without
+		// this, a caller-supplied `bypass_auth`/`bypassAuth` argument would reach handlers
+		// (e.g. createTokens) that trust it directly from the body.
+		delete body.bypass_auth;
+		delete body.bypassAuth;
 		if (STREAMING_OPERATIONS.has(operationName)) {
 			// Reject before the handler runs at all — see STREAMING_OPERATIONS' comment for why
 			// destroying an already-returned stream isn't sufficient cleanup for these producers.

package/components/operations.js

@@ -441,6 +441,10 @@ async function deployComponent(req) {
 				// Reference form only — the rollback source for re-resolving the credential.
 				credentials: credentialReferences.length ? credentialReferences : null,
 				emitter,
+				// Reuse the same `deployment_timeout` knob operators already have for the
+				// peer-side row/blob waits to size the origin's payload-ingest write
+				// transaction too — see DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS in deploymentRecorder.
+				ingestTimeoutMs: req.deployment_timeout,
 			});
 	if (recorder) req._deploymentId = recorder.deploymentId;
 
@@ -646,12 +650,10 @@ async function deployComponent(req) {
 			// deploy checks its own local isNewComponent, since directory state (and therefore
 			// whether the component was already active) can differ per node.
 			//
-			// An existing, already-active component being redeployed does NOT force a restart
-			// here: some updates (e.g. static files only) may not need one at all, and when one
-			// genuinely is needed, that component's already-running file watcher (Scope/
-			// EntryHandler, see deployLifecycle.ts) independently detects the post-deploy file
-			// changes and requests the restart itself.
-			if (application.isNewComponent) {
+			// An existing component's watched files are handled by Scope/EntryHandler. Package
+			// metadata is deliberately outside most plugin globs, so compare it across the atomic
+			// swap as well: a dependency or module-entry change also invalidates loaded code.
+			if (application.isNewComponent || application.packageMetadataChanged) {
 				const { requestRestart } = require('./requestRestart.ts');
 				requestRestart();
 			}

package/config/configUtils.ts

@@ -7,7 +7,6 @@ import YAML from 'yaml';
 import path from 'path';
 import { threadId } from 'node:worker_threads';
 import { randomBytes } from 'node:crypto';
-import { performance } from 'node:perf_hooks';
 import isNumber from 'is-number';
 import propertiesReaderModule from 'properties-reader';
 import _ from 'lodash';
@@ -50,6 +49,14 @@ let flatDefaultConfigObj;
 let flatConfigObj;
 let configObj;
 
+// Canonical param names that live in CONFIG_PARAM_MAP but do NOT correspond to a path in the
+// harper-config.yaml schema (BOOT_PROP_PARAMS is boot-props-file-only bookkeeping — see its own
+// comment). Splitting one of these on '_' and writing it into the nested configObj tree would
+// silently create a bogus top-level section (e.g. 'settings_path' -> configObj.settings.path),
+// which componentLoader.ts treats as a real component to load, since it iterates every truthy
+// top-level key of the root config.
+const NON_NESTED_CONFIG_PARAMS = new Set<string>(Object.values(hdbTerms.BOOT_PROP_PARAMS));
+
 export function resolvePath(relativePath: string) {
 	if (relativePath?.startsWith('~/')) {
 		return path.join(hdbUtils.getHomeDir(), relativePath.slice(1));
@@ -87,12 +94,17 @@ export function getConfigPath(param: string) {
 // Every worker thread runs its own RootConfigWatcher (chokidar), so a write on one thread
 // routinely races a hot-reload read on another; Windows Defender / AV real-time scanning can
 // hold a similar transient handle. Retry with exponential backoff to ride out the race -
-// callers are synchronous, so the wait is a synchronous busy-loop rather than a real sleep.
-const RENAME_RETRY_MAX_ATTEMPTS = 8;
+// callers are synchronous, so the wait is a synchronous sleep rather than an async one.
+// The budget must outlast a single AV real-time scan pass (seconds, not hundreds of ms):
+// the previous ~910ms budget was exhausted twice in a row by the same test on a CI runner
+// (harper#2036), so the worst case is now ~3.6s.
+const RENAME_RETRY_MAX_ATTEMPTS = 12;
 const RENAME_RETRY_INITIAL_DELAY_MS = 10;
-const RENAME_RETRY_MAX_DELAY_MS = 200;
+const RENAME_RETRY_MAX_DELAY_MS = 500;
+// Never notified; exists only so Atomics.wait can time out (a synchronous, CPU-idle sleep).
+const renameRetrySleepBuffer = new Int32Array(new SharedArrayBuffer(4));
 
-function atomicWriteFile(
+export function atomicWriteFile(
 	filePath,
 	content,
 	{
@@ -112,11 +124,10 @@ function atomicWriteFile(
 		} catch (err) {
 			if (retries > 0 && (err.code === 'EPERM' || err.code === 'EACCES')) {
 				retries--;
-				// Sleep synchronously (all call sites are sync) to allow the reader to close the
-				// file. Uses performance.now() rather than Date.now(): the latter tracks wall-clock
-				// time and can jump backward (NTP sync), which would turn this into an unbounded spin.
-				const start = performance.now();
-				while (performance.now() - start < delayMs) {}
+				// Sleep synchronously (all call sites are sync) to allow the holder to close the
+				// file. Atomics.wait yields the thread to the OS instead of spinning the CPU,
+				// which is what makes a multi-second worst-case budget affordable.
+				if (delayMs > 0) Atomics.wait(renameRetrySleepBuffer, 0, 0, delayMs);
 				delayMs = Math.min(delayMs * 2, maxDelayMs);
 				continue;
 			}
@@ -190,8 +201,7 @@ export function createConfigFile(args, skipFsValidation = false) {
 	// Validates config doc and if required sets default values for some parameters.
 	validateConfig(configDoc, skipFsValidation);
 
-	const configObj = configDoc.toJSON();
-	flatConfigObj = flattenConfig(configObj);
+	flatConfigObj = setActiveConfig(configDoc.toJSON());
 
 	// Create new config file and write config doc to it.
 	const hdbRoot = configDoc.getIn(['rootPath']) as string;
@@ -497,9 +507,9 @@ export function initConfig(force = false) {
 
 		// Validates config doc and if required sets default values for some parameters.
 		validateConfig(configDoc);
-		const configObj = configDoc.toJSON();
-		(server as any).config = configObj;
-		flatConfigObj = flattenConfig(configObj);
+		const parsedConfig = configDoc.toJSON();
+		(server as any).config = parsedConfig;
+		flatConfigObj = setActiveConfig(parsedConfig);
 
 		// If config has old version of logrotate enabled let user know it has been deprecated.
 		if (flatConfigObj['logging_rotation_rotate']) {
@@ -687,6 +697,41 @@ export function updateConfigObject(param: string, value: any) {
 	}
 
 	flatConfigObj[configObjKey.toLowerCase()] = value;
+
+	// Keep the nested config tree in sync too: componentLoader (root components) and other
+	// nested-path readers derive behavior — e.g. a component's network port/protocol — from
+	// getConfigObj()'s tree, not the flattened map, so an override that only touched flatConfigObj
+	// was invisible to them. configObj is unset only before a live config has ever been installed
+	// (setActiveConfig), i.e. during install; don't auto-vivify it, since getConfigObj() reads its
+	// falsiness as "not yet initialized" and an empty object would short-circuit that lazy init
+	// permanently.
+	if (configObj != null && !NON_NESTED_CONFIG_PARAMS.has(configObjKey)) {
+		const pathSegments = configObjKey.split('_');
+		let node = configObj;
+		for (let i = 0; i < pathSegments.length - 1; i++) {
+			const segment = pathSegments[i];
+			if (node[segment] === undefined) {
+				// Auto-vivifying ancestors only to delete the leaf would leave empty sections behind,
+				// which componentLoader.ts would treat as components to load.
+				if (value === undefined) return;
+				node[segment] = {};
+			} else if (typeof node[segment] !== 'object' || node[segment] === null) {
+				// A legacy scalar shorthand for this key (e.g. `threads: 4`); descending would replace
+				// it with {} out from under other readers. Leaves the two views disagreeing on this
+				// param, so say so rather than diverging silently.
+				logger.trace(
+					`Config param '${configObjKey}' not mirrored into the nested config: '${segment}' holds a scalar value`
+				);
+				return;
+			}
+			node = node[segment];
+		}
+		// squashObj never writes an undefined value as an enumerable key either; root-config readers
+		// (e.g. bin/run.ts) iterate configObj's own keys and assume every one holds a real value.
+		const leaf = pathSegments[pathSegments.length - 1];
+		if (value === undefined) delete node[leaf];
+		else node[leaf] = value;
+	}
 }
 
 /**
@@ -852,7 +897,7 @@ export function updateConfigValue(
 	}
 	atomicWriteFile(configFileLocation, String(configDoc));
 	if (update_config_obj) {
-		flatConfigObj = flattenConfig(configDoc.toJSON());
+		flatConfigObj = setActiveConfig(configDoc.toJSON());
 	}
 	logger.trace(`Config parameter: ${param} updated with value: ${value}`);
 }
@@ -871,9 +916,25 @@ function backupConfigFile(configPath, hdbRoot) {
 	}
 }
 
+/**
+ * Flattens `obj` and installs it as the live config — the nested tree getConfigObj() hands out and
+ * updateConfigObject() mirrors overrides into. Only callers that own the live config may do this:
+ * flattenConfig() also runs on docs that are discarded moments later (the defaults doc, the
+ * user-supplied doc installer.ts reads for HDB_CONFIG), and no reader of configObj can tell one of
+ * those from the real thing.
+ * @param obj
+ * @returns the flattened config
+ */
+function setActiveConfig(obj) {
+	const flatObj = flattenConfig(obj);
+	configObj = obj;
+	return flatObj;
+}
+
 const PRESERVED_PROPERTIES = ['databases'];
 /**
  * Flattens the JSON version of Harper config with underscores separating each parent/child key.
+ * Does NOT install `obj` as the live config — see setActiveConfig() for that.
  * @param obj
  * @returns {null}
  */
@@ -882,10 +943,7 @@ export function flattenConfig(obj) {
 	if (obj?.operationsApi?.network) obj.operationsApi.network = { ...obj.http, ...obj.operationsApi.network };
 	if (obj?.operationsApi) obj.operationsApi.tls = { ...obj.tls, ...obj.operationsApi.tls };
 
-	configObj = obj;
-	const flatObj = squashObj(obj);
-
-	return flatObj;
+	return squashObj(obj);
 
 	function squashObj(obj) {
 		let result = {};