@harperfast/harper 5.2.1 → 5.2.2

package/bin/copyDb.ts

@@ -454,15 +454,22 @@ function countRecords(rawDbi): { records: number; unversioned: number } {
  * the RocksDB lock — run in-process (inspector) on a live instance, or offline.
  */
 export function verifyMigratedDatabase(databasePath: string): Record<string, { records: number; unversioned: number }> {
-	const rootStore = RocksDatabase.open(databasePath, {});
-	const dbisDb = RocksDatabase.open(databasePath, {
-		name: INTERNAL_DBIS_NAME,
-		sharedStructuresKey: Symbol.for('structures'),
-	});
+	// Every open handle, so a failure at any point (e.g. the second open throwing on lock
+	// contention) cannot leak an earlier handle that would hold the RocksDB lock on the very
+	// diagnostic path operators use after a broken migration.
+	const handles: RocksDatabase[] = [];
 	const report: Record<string, { records: number; unversioned: number }> = {};
 	try {
+		handles.push(RocksDatabase.open(databasePath, {}));
+		const dbisDb = RocksDatabase.open(databasePath, {
+			name: INTERNAL_DBIS_NAME,
+			sharedStructuresKey: Symbol.for('structures'),
+		});
+		handles.push(dbisDb);
 		for (const { key, value: attribute } of dbisDb.getRange({})) {
 			if (typeof key === 'symbol' || !attribute?.isPrimaryKey) continue;
+			// per-table handles close per-iteration so a many-table sweep does not hold every CF
+			// handle open at once; only the two pre-loop opens need the leak-safety array
 			const rawDbi = RocksDatabase.open(databasePath, { name: key, encoding: false });
 			try {
 				report[key] = countRecords(rawDbi);
@@ -471,8 +478,13 @@ export function verifyMigratedDatabase(databasePath: string): Record<string, { r
 			}
 		}
 	} finally {
-		dbisDb.close();
-		rootStore.close();
+		for (const handle of handles.reverse()) {
+			try {
+				handle.close();
+			} catch (error) {
+				console.error('Error closing verification store', error);
+			}
+		}
 	}
 	return report;
 }

package/components/deploymentRecorder.ts

@@ -13,6 +13,7 @@ import { randomUUID } from 'node:crypto';
 import { createHash } from 'node:crypto';
 import { Readable, Transform, pipeline } from 'node:stream';
 import { databases } from '../resources/databases.ts';
+import { transaction, contextStorage } from '../resources/transaction.ts';
 import { createBlob, isSaving, deleteBlob, BLOB_UNAVAILABLE_STATUS } from '../resources/blob.ts';
 import * as terms from '../utility/hdbTerms.ts';
 import type { CredentialReference } from './secretOperations.ts';
@@ -68,6 +69,39 @@ interface CreateOptions {
 	// no-custody transient token.
 	credentials?: CredentialReference[] | null;
 	emitter?: ProgressEmitter;
+	// Open-transaction budget (ms) for the payload-ingest write, below. Mirrors the
+	// `deployment_timeout` operation parameter already used for the peer-side row/blob waits
+	// (DEFAULT_AWAIT_ROW_TIMEOUT_MS) so operators have one existing knob for all of a deploy's
+	// size-driven timeouts, rather than a new config surface for this one.
+	ingestTimeoutMs?: number;
+}
+
+/**
+ * Run `write()` in a fresh transaction while preserving audit attribution. Recorder writes must
+ * commit independently of a caller's transaction so an older staged row cannot overwrite them;
+ * authorization is enforced by the deploy operation rather than these internal writes.
+ *
+ * `timeoutBudget` is sticky across RocksDB `getReadTxn()` calls, so reads inside `write()` cannot
+ * silently restore the generic default. The transaction uses the larger of this budget and the
+ * configured global limit.
+ */
+function withIsolatedTransaction<T>(write: () => Promise<T>, timeoutMs?: number): Promise<T> {
+	const ambient = contextStorage.getStore();
+	const context = {
+		user: ambient?.user,
+		originatingOperation: ambient?.originatingOperation,
+		session: ambient?.session,
+		signal: ambient?.signal,
+	};
+	return transaction(context, (txn) => {
+		if (timeoutMs != null) {
+			// TODO(harper#2057): only extends the RocksDB DatabaseTransaction head; on
+			// HARPER_STORAGE_ENGINE=lmdb, Table.txnForContext() chains a separate LMDBTransaction
+			// that this budget doesn't reach.
+			txn.timeoutBudget = timeoutMs;
+		}
+		return write();
+	});
 }
 
 export class DeploymentRecorder {
@@ -78,10 +112,13 @@ export class DeploymentRecorder {
 	private pendingPut: Promise<void> | null = null;
 	private dirty = false;
 	private sealed = false;
+	private flushSuppressed = false;
+	private readonly ingestTimeoutMs?: number;
 
-	private constructor(deploymentId: string, initial: Record<string, any>) {
+	private constructor(deploymentId: string, initial: Record<string, any>, ingestTimeoutMs?: number) {
 		this.deploymentId = deploymentId;
 		this.record = initial;
+		this.ingestTimeoutMs = ingestTimeoutMs;
 	}
 
 	static async create(options: CreateOptions): Promise<DeploymentRecorder> {
@@ -107,7 +144,7 @@ export class DeploymentRecorder {
 			credentials: options.credentials ?? null,
 			error: null,
 		};
-		const recorder = new DeploymentRecorder(deploymentId, record);
+		const recorder = new DeploymentRecorder(deploymentId, record, options.ingestTimeoutMs);
 		await recorder.put();
 		if (options.emitter) {
 			recorder.subscribeTo(options.emitter);
@@ -158,9 +195,8 @@ export class DeploymentRecorder {
 	// the record dirty; the chained continuation issues a follow-up put once the prior one
 	// settles. This keeps event_log writes O(1) puts per burst rather than O(N) per event.
 	private scheduleFlush(): void {
-		if (this.sealed) {
-			// Sealed: accumulate state in memory but don't write. finish() does the single
-			// terminal write. See seal() for why. The emitter still emits live SSE events.
+		if (this.sealed || this.flushSuppressed) {
+			// Accumulate state until finish() or the active ingest writes it without a competing put.
 			this.dirty = true;
 			return;
 		}
@@ -175,6 +211,7 @@ export class DeploymentRecorder {
 				this.scheduleFlush();
 			}
 		});
+		this.pendingPut.catch((error) => logger.warn?.('Failed to persist deployment progress', error));
 	}
 
 	/**
@@ -191,6 +228,26 @@ export class DeploymentRecorder {
 	 * materialized by the time they reach us, so they take the simpler buffer path.
 	 */
 	async ingestPayload(source: Readable | Buffer | string): Promise<void> {
+		this.flushSuppressed = true;
+		try {
+			while (this.pendingPut) {
+				try {
+					await this.pendingPut;
+				} catch {
+					/* already logged; the ingest write carries the current row state */
+				}
+			}
+			await this.ingestPayloadWithoutFlush(source);
+		} finally {
+			this.flushSuppressed = false;
+			if (this.dirty && !this.sealed) {
+				this.dirty = false;
+				this.scheduleFlush();
+			}
+		}
+	}
+
+	private async ingestPayloadWithoutFlush(source: Readable | Buffer | string): Promise<void> {
 		const hash = createHash('sha256');
 
 		// In-memory sources: the bytes are already resident, so hashing them and creating a
@@ -202,7 +259,10 @@ export class DeploymentRecorder {
 			this.record.payload_blob = createBlob(buffer, { type: 'application/gzip' });
 			this.record.payload_hash = hash.digest('hex');
 			this.record.payload_size = buffer.length;
-			await this.put();
+			// Same blob-durability gate as the streaming path below applies here too (the commit
+			// waits for the buffer to land on disk), so a large base64-in-JSON/CBOR body needs the
+			// same extended budget.
+			await this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 			return;
 		}
 
@@ -263,6 +323,16 @@ export class DeploymentRecorder {
 		});
 		const blob = createBlob(tap, { type: 'application/gzip' });
 		this.record.payload_blob = blob;
+		// This row's commit is gated on the blob's durable file write (resources/blob.ts,
+		// startPreCommitBlobsForRecord's local-write branch) so a peer never sees a row that
+		// references a not-yet-durable blob — the write transaction is legitimately held open for
+		// as long as the payload takes to land on disk. The deploy system is explicitly designed
+		// to carry arbitrarily large, uncapped components (see the module comment above), so that
+		// duration scales with payload size, not with anything bounded. Run this one put() under
+		// an extended open-time budget instead of the generic storage.maxTransactionOpenTime meant
+		// for ordinary short requests (a large-but-otherwise-healthy ingest tripped that generic
+		// guard on a loaded self-hosted runner — see the Large Deploy Payload Test regression).
+		//
 		// Persisting the row encodes the blob, which synchronously starts the file write that
 		// drains the tap, so `isSaving(blob)` is set as soon as put() is *called*. Await the put,
 		// the source draining (tapDone), and the file flush (saving) together: handing all three
@@ -270,7 +340,7 @@ export class DeploymentRecorder {
 		// a large upload mid-stream (which rejects both tapDone and the blob write) fails the
 		// deploy loudly here instead of leaking an unhandledRejection. This also makes hash/size
 		// correctness independent of the put()/store write timing and of isSaving() being defined.
-		const putDone = this.put();
+		const putDone = this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 		const saving = isSaving(blob) ?? Promise.resolve();
 		try {
 			await Promise.all([putDone, tapDone, saving]);
@@ -288,7 +358,7 @@ export class DeploymentRecorder {
 				// ensures the null-reference write lands after the original reference write
 				// (so it wins) and the blob's file lock is released before we unlink it.
 				await Promise.allSettled([putDone, saving]);
-				await this.put();
+				await this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 				deleteBlob(blob);
 			} catch (cleanupError) {
 				logger.warn?.('Failed to clean up partial deploy payload blob', cleanupError);
@@ -301,7 +371,7 @@ export class DeploymentRecorder {
 		this.record.payload_size = blob.size ?? byteCount;
 		// Persist the now-known hash + size. The blob is already saved, so this re-put does
 		// not re-stream — saveBlob short-circuits on the existing fileId.
-		await this.put();
+		await this.put(ingestTransactionTimeoutMs(this.ingestTimeoutMs));
 	}
 
 	/**
@@ -436,14 +506,14 @@ export class DeploymentRecorder {
 		return this.record;
 	}
 
-	private async put(): Promise<void> {
+	private async put(timeoutMs?: number): Promise<void> {
 		const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME];
 		if (!table) {
 			// Table missing means the upgrade directive hasn't run yet (or the table got dropped).
 			// We tolerate this — tracking is observability; the deploy itself must still succeed.
 			return;
 		}
-		await table.put(this.record);
+		await withIsolatedTransaction(() => table.put(this.record), timeoutMs);
 	}
 }
 
@@ -455,11 +525,32 @@ export class DeploymentRecorder {
 // via the `deployment_timeout` operation parameter.
 export const DEFAULT_AWAIT_ROW_TIMEOUT_MS = 120_000;
 
+// Default open-time budget for the origin's payload-ingest write transaction (ingestPayload,
+// below). Distinct from DEFAULT_AWAIT_ROW_TIMEOUT_MS (that one bounds a *replication* wait;
+// this one bounds *local disk I/O* for a component the deploy system explicitly supports at
+// arbitrarily large, uncapped sizes — see the module comment above and the large-payload
+// regression test). 10 minutes comfortably covers a multi-GB ingest even on a slow or loaded
+// disk while still being a bounded safety net, not a disabled one. Overridable via the same
+// `deployment_timeout` operation parameter as the peer-side waits, so operators sizing for
+// even larger components (or a known-slow disk) have one knob, not two.
+export const DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS = 600_000;
+
 export function coerceTimeoutMs(value: unknown, fallback: number): number {
 	const requested = Number(value);
 	return Number.isFinite(requested) && requested >= 0 ? requested : fallback;
 }
 
+// Floored at the default, never lowered by it: `deployment_timeout` also governs the
+// peer-side row/blob wait (awaitDeploymentRow), where 0 means "poll once, don't wait" — a
+// legitimate fast-fail setting for peers that must never also starve the origin's own disk
+// write down to 0ms.
+export function ingestTransactionTimeoutMs(deploymentTimeout: unknown): number {
+	return Math.max(
+		coerceTimeoutMs(deploymentTimeout, DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS),
+		DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS
+	);
+}
+
 /**
  * Peer-side helper — wait for the hdb_deployment row to arrive via table replication,
  * then return it. The row is committed on origin before `replicateOperation` is

package/components/operations.js

@@ -441,6 +441,10 @@ async function deployComponent(req) {
 				// Reference form only — the rollback source for re-resolving the credential.
 				credentials: credentialReferences.length ? credentialReferences : null,
 				emitter,
+				// Reuse the same `deployment_timeout` knob operators already have for the
+				// peer-side row/blob waits to size the origin's payload-ingest write
+				// transaction too — see DEFAULT_INGEST_TRANSACTION_TIMEOUT_MS in deploymentRecorder.
+				ingestTimeoutMs: req.deployment_timeout,
 			});
 	if (recorder) req._deploymentId = recorder.deploymentId;
 

package/config/configUtils.ts

@@ -49,6 +49,14 @@ let flatDefaultConfigObj;
 let flatConfigObj;
 let configObj;
 
+// Canonical param names that live in CONFIG_PARAM_MAP but do NOT correspond to a path in the
+// harper-config.yaml schema (BOOT_PROP_PARAMS is boot-props-file-only bookkeeping — see its own
+// comment). Splitting one of these on '_' and writing it into the nested configObj tree would
+// silently create a bogus top-level section (e.g. 'settings_path' -> configObj.settings.path),
+// which componentLoader.ts treats as a real component to load, since it iterates every truthy
+// top-level key of the root config.
+const NON_NESTED_CONFIG_PARAMS = new Set<string>(Object.values(hdbTerms.BOOT_PROP_PARAMS));
+
 export function resolvePath(relativePath: string) {
 	if (relativePath?.startsWith('~/')) {
 		return path.join(hdbUtils.getHomeDir(), relativePath.slice(1));
@@ -96,7 +104,7 @@ const RENAME_RETRY_MAX_DELAY_MS = 500;
 // Never notified; exists only so Atomics.wait can time out (a synchronous, CPU-idle sleep).
 const renameRetrySleepBuffer = new Int32Array(new SharedArrayBuffer(4));
 
-function atomicWriteFile(
+export function atomicWriteFile(
 	filePath,
 	content,
 	{
@@ -193,8 +201,7 @@ export function createConfigFile(args, skipFsValidation = false) {
 	// Validates config doc and if required sets default values for some parameters.
 	validateConfig(configDoc, skipFsValidation);
 
-	const configObj = configDoc.toJSON();
-	flatConfigObj = flattenConfig(configObj);
+	flatConfigObj = setActiveConfig(configDoc.toJSON());
 
 	// Create new config file and write config doc to it.
 	const hdbRoot = configDoc.getIn(['rootPath']) as string;
@@ -500,9 +507,9 @@ export function initConfig(force = false) {
 
 		// Validates config doc and if required sets default values for some parameters.
 		validateConfig(configDoc);
-		const configObj = configDoc.toJSON();
-		(server as any).config = configObj;
-		flatConfigObj = flattenConfig(configObj);
+		const parsedConfig = configDoc.toJSON();
+		(server as any).config = parsedConfig;
+		flatConfigObj = setActiveConfig(parsedConfig);
 
 		// If config has old version of logrotate enabled let user know it has been deprecated.
 		if (flatConfigObj['logging_rotation_rotate']) {
@@ -690,6 +697,41 @@ export function updateConfigObject(param: string, value: any) {
 	}
 
 	flatConfigObj[configObjKey.toLowerCase()] = value;
+
+	// Keep the nested config tree in sync too: componentLoader (root components) and other
+	// nested-path readers derive behavior — e.g. a component's network port/protocol — from
+	// getConfigObj()'s tree, not the flattened map, so an override that only touched flatConfigObj
+	// was invisible to them. configObj is unset only before a live config has ever been installed
+	// (setActiveConfig), i.e. during install; don't auto-vivify it, since getConfigObj() reads its
+	// falsiness as "not yet initialized" and an empty object would short-circuit that lazy init
+	// permanently.
+	if (configObj != null && !NON_NESTED_CONFIG_PARAMS.has(configObjKey)) {
+		const pathSegments = configObjKey.split('_');
+		let node = configObj;
+		for (let i = 0; i < pathSegments.length - 1; i++) {
+			const segment = pathSegments[i];
+			if (node[segment] === undefined) {
+				// Auto-vivifying ancestors only to delete the leaf would leave empty sections behind,
+				// which componentLoader.ts would treat as components to load.
+				if (value === undefined) return;
+				node[segment] = {};
+			} else if (typeof node[segment] !== 'object' || node[segment] === null) {
+				// A legacy scalar shorthand for this key (e.g. `threads: 4`); descending would replace
+				// it with {} out from under other readers. Leaves the two views disagreeing on this
+				// param, so say so rather than diverging silently.
+				logger.trace(
+					`Config param '${configObjKey}' not mirrored into the nested config: '${segment}' holds a scalar value`
+				);
+				return;
+			}
+			node = node[segment];
+		}
+		// squashObj never writes an undefined value as an enumerable key either; root-config readers
+		// (e.g. bin/run.ts) iterate configObj's own keys and assume every one holds a real value.
+		const leaf = pathSegments[pathSegments.length - 1];
+		if (value === undefined) delete node[leaf];
+		else node[leaf] = value;
+	}
 }
 
 /**
@@ -855,7 +897,7 @@ export function updateConfigValue(
 	}
 	atomicWriteFile(configFileLocation, String(configDoc));
 	if (update_config_obj) {
-		flatConfigObj = flattenConfig(configDoc.toJSON());
+		flatConfigObj = setActiveConfig(configDoc.toJSON());
 	}
 	logger.trace(`Config parameter: ${param} updated with value: ${value}`);
 }
@@ -874,9 +916,25 @@ function backupConfigFile(configPath, hdbRoot) {
 	}
 }
 
+/**
+ * Flattens `obj` and installs it as the live config — the nested tree getConfigObj() hands out and
+ * updateConfigObject() mirrors overrides into. Only callers that own the live config may do this:
+ * flattenConfig() also runs on docs that are discarded moments later (the defaults doc, the
+ * user-supplied doc installer.ts reads for HDB_CONFIG), and no reader of configObj can tell one of
+ * those from the real thing.
+ * @param obj
+ * @returns the flattened config
+ */
+function setActiveConfig(obj) {
+	const flatObj = flattenConfig(obj);
+	configObj = obj;
+	return flatObj;
+}
+
 const PRESERVED_PROPERTIES = ['databases'];
 /**
  * Flattens the JSON version of Harper config with underscores separating each parent/child key.
+ * Does NOT install `obj` as the live config — see setActiveConfig() for that.
  * @param obj
  * @returns {null}
  */
@@ -885,10 +943,7 @@ export function flattenConfig(obj) {
 	if (obj?.operationsApi?.network) obj.operationsApi.network = { ...obj.http, ...obj.operationsApi.network };
 	if (obj?.operationsApi) obj.operationsApi.tls = { ...obj.tls, ...obj.operationsApi.tls };
 
-	configObj = obj;
-	const flatObj = squashObj(obj);
-
-	return flatObj;
+	return squashObj(obj);
 
 	function squashObj(obj) {
 		let result = {};

package/config/harperConfigEnvVars.ts

@@ -28,6 +28,7 @@ import * as path from 'node:path';
 import * as crypto from 'node:crypto';
 import { cloneDeep } from 'lodash';
 import { getBackupDirPath } from './configHelpers.ts';
+import { atomicWriteFile } from './configUtils.ts';
 import * as hdbTerms from '../utility/hdbTerms.ts';
 
 const STATE_FILE_NAME = '.harper-config-state.json';
@@ -78,6 +79,10 @@ interface ConfigState {
 	version: string;
 	sources: Record<string, ConfigSource>; // Maps config path to the source that set it
 	originalValues: Record<string, any>; // Original values before env var override (for restoration)
+	// Paths the config file declared as empty objects before an env layer first populated
+	// them (#1618/#1726). Kept separate from originalValues so a marker can never mask, or
+	// be consumed as, a real leaf original at the same path.
+	emptyScopeOriginals: Record<string, true>;
 	snapshots: {
 		// Snapshots of what each env var currently specifies (for detecting changes)
 		HARPER_DEFAULT_CONFIG?: { hash: string; config: ConfigObject };
@@ -410,21 +415,93 @@ function setNestedValue(obj: ConfigObject, path: string, value: any): void {
 }
 
 /**
- * Delete nested value by dot-notation path
+ * Delete nested value by dot-notation path, pruning ancestor objects the deletion
+ * emptied. Removal operates leaf-by-leaf on flattened paths, so deleting the last
+ * leaf under an entry would otherwise leave an `entry: {}` husk in the config file —
+ * invalid wherever validation requires fields, and sticky once persisted (#2067).
+ * Prunes only what this deletion emptied: an absent leaf deletes nothing and prunes
+ * nothing, so a deliberate empty scope (#1618/#1726) is never eaten by a no-op
+ * removal. Returns the pruned ancestor paths, deepest first.
  */
-function deleteNestedValue(obj: ConfigObject, path: string): void {
+function deleteNestedValue(obj: ConfigObject, path: string): string[] {
 	const keys = path.split('.');
+	const ancestors: ConfigObject[] = [];
 	let current = obj;
 
 	for (let i = 0; i < keys.length - 1; i++) {
 		const key = keys[i];
 		if (!isPlainObject(current[key])) {
-			return; // Path doesn't exist
+			return []; // Path doesn't exist
 		}
+		ancestors.push(current);
 		current = current[key];
 	}
 
-	delete current[keys[keys.length - 1]];
+	const leafKey = keys[keys.length - 1];
+	// Own-property check: `in` walks the prototype chain, so a leaf named like an
+	// Object.prototype member (`constructor`, `toString`) would pass, no-op the
+	// delete, and let the prune loop eat a deliberate empty scope.
+	if (!Object.prototype.hasOwnProperty.call(current, leafKey)) {
+		return [];
+	}
+	delete current[leafKey];
+
+	const prunedPaths: string[] = [];
+	for (let i = ancestors.length - 1; i >= 0 && Object.keys(current).length === 0; i--) {
+		delete ancestors[i][keys[i]];
+		prunedPaths.push(keys.slice(0, i + 1).join('.'));
+		current = ancestors[i];
+	}
+	return prunedPaths;
+}
+
+/**
+ * If the deepest existing ancestor of `path` is an empty plain object, record it in
+ * emptyScopeOriginals before a layer populates it: a bare `name: {}` in the config
+ * file is user content (#1618/#1726) even while an env var temporarily fills it. At
+ * most one ancestor can be both existing and empty, so one marker suffices.
+ */
+function recordEmptyAncestorOriginal(fileConfig: ConfigObject, state: ConfigState, path: string): void {
+	const keys = path.split('.');
+	let current = fileConfig;
+	for (let i = 0; i < keys.length - 1; i++) {
+		current = Object.prototype.hasOwnProperty.call(current, keys[i]) ? current[keys[i]] : undefined;
+		if (current === undefined) {
+			// An absent prefix cannot be a live file-declared empty scope, so markers at
+			// or under it are stale (the user deleted the scope while the env var held
+			// it) and must not resurrect it later. Only true absence qualifies: a scalar
+			// here is a higher layer occluding the scope, not the user deleting it, and
+			// emptiness is what an apply's first leaf creates right after recording.
+			const prefix = keys.slice(0, i + 1).join('.');
+			for (const markerPath of Object.keys(state.emptyScopeOriginals)) {
+				if (markerPath === prefix || markerPath.startsWith(prefix + '.')) {
+					delete state.emptyScopeOriginals[markerPath];
+				}
+			}
+			return;
+		}
+		if (!isPlainObject(current)) return;
+		if (Object.keys(current).length === 0) {
+			state.emptyScopeOriginals[keys.slice(0, i + 1).join('.')] = true;
+			return;
+		}
+	}
+}
+
+/**
+ * Counterpart to recordEmptyAncestorOriginal: when a prune removed an ancestor the
+ * file originally declared as `{}`, put the empty scope back. Only paths the deletion
+ * actually pruned are candidates, so a scalar overwrite or an absent-leaf no-op can
+ * never resurrect a scope over live env-layer content.
+ */
+function restorePrunedEmptyAncestor(fileConfig: ConfigObject, state: ConfigState, prunedPaths: string[]): void {
+	for (const prunedPath of prunedPaths) {
+		if (Object.prototype.hasOwnProperty.call(state.emptyScopeOriginals, prunedPath)) {
+			setNestedValue(fileConfig, prunedPath, {});
+			delete state.emptyScopeOriginals[prunedPath];
+			return;
+		}
+	}
 }
 
 /**
@@ -474,16 +551,23 @@ function loadConfigState(rootPath: string): ConfigState {
 			version: '1.0',
 			sources: {},
 			originalValues: {},
+			emptyScopeOriginals: {},
 			snapshots: {},
 		};
 	}
 
 	try {
 		const state = fs.readJsonSync(statePath) as ConfigState;
-		// Ensure originalValues exists (for backwards compatibility with old state files)
+		// Ensure newer fields exist (for backwards compatibility with old state files)
 		if (!state.originalValues) {
 			state.originalValues = {};
 		}
+		if (!state.emptyScopeOriginals) {
+			// Only the field is recoverable, not the information: a scope an env layer
+			// populated before this field existed has no marker and will not be restored
+			// on vacate (see DESIGN.md, env-config empty objects)
+			state.emptyScopeOriginals = {};
+		}
 		return state;
 	} catch (error) {
 		// If state file is corrupted, start fresh
@@ -493,6 +577,7 @@ function loadConfigState(rootPath: string): ConfigState {
 			version: '1.0',
 			sources: {},
 			originalValues: {},
+			emptyScopeOriginals: {},
 			snapshots: {},
 		};
 	}
@@ -508,7 +593,9 @@ function saveConfigState(rootPath: string, state: ConfigState): void {
 	// Ensure backup directory exists
 	fs.ensureDirSync(backupDir);
 
-	fs.writeJsonSync(statePath, state, { spaces: 2 });
+	// Atomic write: a torn state file resets to fresh on the next load, losing every
+	// restoration record — the blast radius is user config-file content
+	atomicWriteFile(statePath, JSON.stringify(state, null, 2) + '\n');
 }
 
 /**
@@ -553,7 +640,7 @@ function applyConfigLayer(
 	const flatEnvConfig = flattenObject(envConfig);
 
 	for (const [path, value] of Object.entries(flatEnvConfig)) {
-		const currentSource = state.sources[path];
+		const currentSource = Object.prototype.hasOwnProperty.call(state.sources, path) ? state.sources[path] : undefined;
 		const currentValue = getNestedValue(fileConfig, path);
 
 		// Skip if this path has a source we should respect
@@ -562,9 +649,14 @@ function applyConfigLayer(
 		}
 
 		// Store original value if requested and this is first time overriding
-		if (storeOriginals && !currentSource && currentValue !== undefined && currentValue !== null) {
-			if (!(path in state.originalValues)) {
-				state.originalValues[path] = currentValue;
+		if (storeOriginals) {
+			if (!currentSource && currentValue != null) {
+				if (!Object.prototype.hasOwnProperty.call(state.originalValues, path)) {
+					state.originalValues[path] = currentValue;
+				}
+			} else if (currentValue == null) {
+				// runs on re-assert too, so a hand-deleted scope clears its stale marker
+				recordEmptyAncestorOriginal(fileConfig, state, path);
 			}
 		}
 
@@ -599,13 +691,13 @@ function handleDeletions(
 				(sourceName === 'HARPER_DEFAULT_CONFIG' ||
 					sourceName === 'HARPER_CONFIG' ||
 					sourceName === 'HARPER_SET_CONFIG') &&
-				path in state.originalValues
+				Object.prototype.hasOwnProperty.call(state.originalValues, path)
 			) {
 				setNestedValue(fileConfig, path, state.originalValues[path]);
 				delete state.originalValues[path];
 			} else {
 				// For other sources or if no original value, delete
-				deleteNestedValue(fileConfig, path);
+				restorePrunedEmptyAncestor(fileConfig, state, deleteNestedValue(fileConfig, path));
 			}
 			delete state.sources[path];
 		}
@@ -619,7 +711,7 @@ function removeValuesWithSource(fileConfig: ConfigObject, state: ConfigState, so
 	const pathsToRemove = Object.keys(state.sources).filter((path) => state.sources[path] === sourceName);
 
 	for (const path of pathsToRemove) {
-		deleteNestedValue(fileConfig, path);
+		restorePrunedEmptyAncestor(fileConfig, state, deleteNestedValue(fileConfig, path));
 		delete state.sources[path];
 	}
 }
@@ -690,7 +782,9 @@ function processEnvVar(
 			// Runtime: Only update values we previously set
 			const flatEnvConfig = flattenObject(parsedConfig);
 			for (const [path, value] of Object.entries(flatEnvConfig)) {
-				const currentSource = state.sources[path];
+				const currentSource = Object.prototype.hasOwnProperty.call(state.sources, path)
+					? state.sources[path]
+					: undefined;
 				const currentValue = getNestedValue(fileConfig, path);
 
 				// Skip if path has a tracked source that's not HARPER_DEFAULT_CONFIG
@@ -702,12 +796,15 @@ function processEnvVar(
 				if (!currentSource) {
 					if (currentValue !== undefined && currentValue !== null) {
 						// Value exists but we never set it - store as original but don't override
-						if (!(path in state.originalValues)) {
+						if (!Object.prototype.hasOwnProperty.call(state.originalValues, path)) {
 							state.originalValues[path] = currentValue;
 						}
 						continue;
 					}
 				}
+				if (currentValue == null) {
+					recordEmptyAncestorOriginal(fileConfig, state, path);
+				}
 
 				// Set the value and track the source (directive leaves compose against current)
 				setNestedValue(fileConfig, path, resolveLeafValue(currentValue, value, path));
@@ -749,13 +846,13 @@ function cleanupRemovedEnvVar(
 	if (sourceName === 'HARPER_DEFAULT_CONFIG' || sourceName === 'HARPER_CONFIG' || sourceName === 'HARPER_SET_CONFIG') {
 		const pathsToCleanup = Object.keys(state.sources).filter((path) => state.sources[path] === sourceName);
 		for (const path of pathsToCleanup) {
-			if (path in state.originalValues) {
+			if (Object.prototype.hasOwnProperty.call(state.originalValues, path)) {
 				// Restore original value
 				setNestedValue(fileConfig, path, state.originalValues[path]);
 				delete state.originalValues[path];
 			} else {
 				// No original, just delete
-				deleteNestedValue(fileConfig, path);
+				restorePrunedEmptyAncestor(fileConfig, state, deleteNestedValue(fileConfig, path));
 			}
 			delete state.sources[path];
 		}