@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

package/config/configReadRetry.ts

@@ -0,0 +1,62 @@
+// Why a lock that outlives the reader's budget is retried from a timer, and why both the bound and
+// the backoff are wall clock rather than an attempt count: see "Root config watchers must read
+// synchronously" in DESIGN.md (harper#2191).
+const RETRY_BUDGET_MS = 3_100;
+const INITIAL_DELAY_MS = 100;
+const MAX_DELAY_MS = 1_600;
+
+let retryBudgetMs = RETRY_BUDGET_MS;
+
+// Test-only, the read-side twin of `atomicWriteFile`'s `retryBudgetMs` option. A case that only
+// needs the ladder *spent* has no way to get there but wall clock, so without this each one sits
+// out the shipped 3.1 s; a dozen of them cost the unit job 40 s. Called with no argument it
+// restores the shipped value, and `configReadRetry.test.js` still spends that value in full, so
+// shortening it elsewhere cannot hide a change to what ships.
+export function _setRetryBudgetForTests(ms: number = RETRY_BUDGET_MS): void {
+	retryBudgetMs = ms;
+}
+
+export class ConfigReadRetry {
+	#timer?: NodeJS.Timeout;
+	#deadline?: number;
+	// Captured with the deadline: the backoff is a fraction of the budget the deadline came from,
+	// so reading the module value again mid-ladder would mis-scale it if a test had changed it.
+	#budgetMs: number = retryBudgetMs;
+
+	// `holdEventLoop` is for a caller whose boot barrier this ladder is the only thing left to
+	// settle: an unref'd timer would let the thread drain and exit mid-boot instead.
+	schedule(retry: () => void, holdEventLoop: boolean = false): boolean {
+		this.cancel();
+		const now = performance.now();
+		if (this.#deadline === undefined) {
+			this.#budgetMs = retryBudgetMs;
+			this.#deadline = now + this.#budgetMs;
+		}
+		const remainingMs = this.#deadline - now;
+		if (remainingMs <= 0) {
+			this.reset();
+			return false;
+		}
+		const elapsedMs = this.#budgetMs - remainingMs;
+		const delayMs = Math.min(Math.max(elapsedMs, INITIAL_DELAY_MS), MAX_DELAY_MS, remainingMs);
+		this.#timer = setTimeout(retry, delayMs);
+		if (!holdEventLoop) this.#timer.unref();
+		return true;
+	}
+
+	get pending(): boolean {
+		return this.#timer !== undefined;
+	}
+
+	reset(): void {
+		this.cancel();
+		this.#deadline = undefined;
+	}
+
+	cancel(): void {
+		if (this.#timer) {
+			clearTimeout(this.#timer);
+			this.#timer = undefined;
+		}
+	}
+}

package/config/configUtils.ts

@@ -92,22 +92,56 @@ export function getConfigPath(param: string) {
 	return path.resolve(rootPath, value);
 }
 
-// Write atomically via temp file + rename so readers don't observe a truncated/empty file.
-// Temp path includes randomness so two worker threads in the same process (same pid) writing
-// in the same millisecond can't collide on the temp name and then race the rename.
-//
+// Write atomically via a randomized temp file + rename so readers do not observe partial content
+// and concurrent workers, which share process.pid, do not collide on a temp path.
 // Windows has no POSIX-style "replace an open file" semantics: rename() fails with
-// EPERM/EACCES while another descriptor is open on the destination. The sleep below blocks the
-// calling thread, so this can only ride out a holder that releases without needing that
-// thread's event loop. A holder on the calling thread would live exactly as long as the budget,
-// which is why config readers must not keep a descriptor on this file open across an event-loop
-// turn (RootConfigWatcher.handleChange, OptionsWatcher#handleChange).
-const RENAME_RETRY_MAX_ATTEMPTS = 12;
+// EPERM/EACCES/EBUSY while another worker or AV holds the destination open. Root config watchers use
+// readConfigFileSync so this blocking retry cannot wait on a read owned by its own worker.
+// The budget is the wall-clock window of the 12-attempt schedule it replaced
+// (10+20+40+80+160+320+500*6), so it stays a deadline rather than an attempt count without
+// widening the stall: this loop blocks the calling worker's event loop, and `set_configuration`
+// reaches it from a live request thread.
+const RENAME_RETRY_BUDGET_MS = 3_630;
+// Secondary guard only: stops a degenerate zero-delay option set from spinning the whole budget.
+const RENAME_RETRY_MAX_ATTEMPTS = 25;
 const RENAME_RETRY_INITIAL_DELAY_MS = 10;
 const RENAME_RETRY_MAX_DELAY_MS = 500;
-// Never notified; exists only so Atomics.wait can time out (a synchronous, CPU-idle sleep).
+// Never notified; Atomics.wait uses this only as a CPU-idle synchronous sleep.
 const renameRetrySleepBuffer = new Int32Array(new SharedArrayBuffer(4));
 
+// Classified by code alone rather than gated to win32 like the read side: `process.platform` does
+// not answer whether this filesystem can replace an open file — WSL drvfs, CIFS/SMB and Docker
+// Desktop bind mounts all report `linux` and return these codes transiently.
+function isRetryableRenameError(code: string): boolean {
+	return code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
+}
+
+type RenameRetryOptions = {
+	retryBudgetMs?: number;
+	maxRetries?: number;
+	initialDelayMs?: number;
+	maxDelayMs?: number;
+};
+
+type AtomicWriteOptions = RenameRetryOptions & {
+	skipIfUnchanged?: boolean;
+};
+
+function validateRenameRetryOptions({ retryBudgetMs, maxRetries, initialDelayMs, maxDelayMs }: RenameRetryOptions) {
+	const invalidOption =
+		!Number.isFinite(retryBudgetMs) ||
+		retryBudgetMs < 0 ||
+		(!Number.isFinite(maxRetries) && maxRetries !== Infinity) ||
+		maxRetries < 0 ||
+		!Number.isFinite(initialDelayMs) ||
+		initialDelayMs < 0 ||
+		!Number.isFinite(maxDelayMs) ||
+		maxDelayMs < 0;
+	if (invalidOption) {
+		throw new RangeError('rename retry options must be non-negative numbers');
+	}
+}
+
 // Linux has no libuv mapping for EDQUOT, so a quota-exhausted write surfaces as
 // `Unknown system error -122` with an unusable `code`; the numeric errno is the portable signal
 // (EDQUOT is 122 on Linux, 69 on macOS).
@@ -142,12 +176,15 @@ export function atomicWriteFile(
 	filePath,
 	content,
 	{
+		retryBudgetMs = RENAME_RETRY_BUDGET_MS,
 		maxRetries = RENAME_RETRY_MAX_ATTEMPTS,
 		initialDelayMs = RENAME_RETRY_INITIAL_DELAY_MS,
 		maxDelayMs = RENAME_RETRY_MAX_DELAY_MS,
 		skipIfUnchanged = false,
-	} = {}
+	}: AtomicWriteOptions = {}
 ) {
+	// Before the temp write, so an option set that can never rename leaves no file behind.
+	validateRenameRetryOptions({ retryBudgetMs, maxRetries, initialDelayMs, maxDelayMs });
 	// Opt-in: skipping means no mtime bump, so no watcher event. Only callers that re-derive the
 	// same file every boot want that.
 	if (skipIfUnchanged && matchesFileContent(filePath, content)) return false;
@@ -160,8 +197,10 @@ export function atomicWriteFile(
 		throw err;
 	}
 	try {
-		renameWithRetry(tempPath, filePath, { maxRetries, initialDelayMs, maxDelayMs });
+		renameWithRetry(tempPath, filePath, { retryBudgetMs, maxRetries, initialDelayMs, maxDelayMs });
 	} catch (err) {
+		// The temp name carries fresh randomness on every call, so a spent budget would otherwise
+		// leave a file nothing else will ever collect.
 		removeTempFile(tempPath);
 		throw err;
 	}
@@ -172,35 +211,48 @@ export function renameWithRetry(
 	fromPath,
 	toPath,
 	{
+		retryBudgetMs = RENAME_RETRY_BUDGET_MS,
 		maxRetries = RENAME_RETRY_MAX_ATTEMPTS,
 		initialDelayMs = RENAME_RETRY_INITIAL_DELAY_MS,
 		maxDelayMs = RENAME_RETRY_MAX_DELAY_MS,
-	} = {}
+	}: RenameRetryOptions = {}
 ) {
+	validateRenameRetryOptions({ retryBudgetMs, maxRetries, initialDelayMs, maxDelayMs });
 	let retries = maxRetries;
 	let delayMs = initialDelayMs;
+	let retryDeadline;
+	let finalAttempt = false;
 	let attempts = 0;
-	const startedAt = Date.now();
+	const startedAt = performance.now();
 	while (true) {
 		try {
 			attempts++;
 			fs.renameSync(fromPath, toPath);
 			return;
 		} catch (err) {
-			if (retries > 0 && (err.code === 'EPERM' || err.code === 'EACCES')) {
+			if (!finalAttempt && retries > 0 && isRetryableRenameError(err.code)) {
 				retries--;
-				// Sleep synchronously (all call sites are sync) to allow the holder to close the
-				// file. Atomics.wait yields the thread to the OS instead of spinning the CPU,
-				// which is what makes a multi-second worst-case budget affordable.
-				if (delayMs > 0) Atomics.wait(renameRetrySleepBuffer, 0, 0, delayMs);
-				delayMs = Math.min(delayMs * 2, maxDelayMs);
-				continue;
+				if (retryDeadline === undefined) {
+					retryDeadline = performance.now() + retryBudgetMs;
+				}
+				const remainingBudgetMs = retryDeadline - performance.now();
+				if (remainingBudgetMs > 0) {
+					// Sleep synchronously (all call sites are sync) to allow the holder to close the
+					// file. Atomics.wait yields the thread to the OS instead of spinning the CPU,
+					// which is what makes a multi-second worst-case budget affordable.
+					const sleepMs = Math.min(delayMs, remainingBudgetMs);
+					finalAttempt = sleepMs === remainingBudgetMs;
+					if (sleepMs > 0) Atomics.wait(renameRetrySleepBuffer, 0, 0, sleepMs);
+					delayMs = Math.min(Math.max(delayMs * 2, RENAME_RETRY_INITIAL_DELAY_MS), maxDelayMs);
+					continue;
+				}
 			}
-			// Attempts and elapsed distinguish a holder that never released from one that lost
-			// a race, and neither survives on the rethrown error.
-			if (err.code === 'EPERM' || err.code === 'EACCES') {
+			// Whether the budget was spent or the code was never retryable is the difference
+			// between a holder that never released and a one-off failure, and neither survives on
+			// the rethrown error.
+			if (isRetryableRenameError(err.code)) {
 				logger.warn(
-					`Could not replace ${toPath}: ${err.code} after ${attempts} attempts over ${Date.now() - startedAt}ms`
+					`Could not replace ${toPath}: ${err.code} after ${attempts} attempts over ${Math.round(performance.now() - startedAt)}ms`
 				);
 			}
 			throw err;

package/config/parseConfigFile.ts

@@ -0,0 +1,34 @@
+import { parse } from 'yaml';
+
+// yaml's prettyErrors frames the offending source lines into `message`, and a config file holds
+// credentials. Neither the cause nor the original stack is carried for the same reason.
+export class ConfigParseError extends Error {
+	constructor(filePath: string, error: unknown) {
+		const { name, code, linePos } = (error ?? {}) as {
+			name?: string;
+			code?: string;
+			linePos?: { line: number; col: number }[];
+		};
+		const at = linePos?.[0] ? ` at line ${linePos[0].line}, column ${linePos[0].col}` : '';
+		super(`Unable to parse the Harper configuration file at ${filePath}: ${code ?? name ?? 'parse failure'}${at}`);
+		this.name = 'ConfigParseError';
+	}
+}
+
+export function parseConfigFile(contents: string, filePath: string): any {
+	try {
+		// yaml routes warnings through `process.emitWarning` rather than a throw, so a framed
+		// warning would reach stderr with the config's own source lines in it, around this scrub.
+		return parse(contents, { logLevel: 'error' });
+	} catch (error) {
+		// Only yaml's own parse errors frame the source into `message`. Anything else is a fault in
+		// the parser, where the message is the whole of the debugging context.
+		if (!isYamlParseError(error)) throw error;
+		throw new ConfigParseError(filePath, error);
+	}
+}
+
+function isYamlParseError(error: unknown): boolean {
+	const { name, linePos } = (error ?? {}) as { name?: string; linePos?: unknown };
+	return linePos !== undefined || name === 'YAMLParseError';
+}

package/config/readConfigFileSync.ts

@@ -0,0 +1,44 @@
+import { readFileSync } from 'node:fs';
+
+// Why root config reads must not be async, and why this budget is shared rather than per call:
+// see "Root config watchers must read synchronously" in DESIGN.md (harper#2191).
+const READ_RETRY_BUDGET_MS = 500;
+const READ_RETRY_INITIAL_DELAY_MS = 10;
+const READ_RETRY_MAX_DELAY_MS = 100;
+const readRetrySleepBuffer = new Int32Array(new SharedArrayBuffer(4));
+const retryDeadlines = new Map<string, number>();
+
+// A caller that already owns a retry ladder passes `waitForLock` false, so one lock costs one
+// blocking window in total rather than one per rung.
+export function readConfigFileSync(filePath: string, waitForLock: boolean = true): string {
+	let delayMs = READ_RETRY_INITIAL_DELAY_MS;
+	while (true) {
+		try {
+			const contents = readFileSync(filePath, 'utf-8');
+			retryDeadlines.delete(filePath);
+			return contents;
+		} catch (error) {
+			const remainingBudgetMs = waitForLock && isSharingViolation(error) ? remainingRetryBudgetMs(filePath) : 0;
+			if (remainingBudgetMs <= 0) throw error;
+			Atomics.wait(readRetrySleepBuffer, 0, 0, Math.min(delayMs, remainingBudgetMs));
+			delayMs = Math.min(delayMs * 2, READ_RETRY_MAX_DELAY_MS);
+		}
+	}
+}
+
+function isSharingViolation(error: unknown): boolean {
+	if (process.platform !== 'win32') return false;
+	const code = (error as { code?: string } | null)?.code;
+	return code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
+}
+
+function remainingRetryBudgetMs(filePath: string): number {
+	const now = performance.now();
+	let deadline = retryDeadlines.get(filePath);
+	// A deadline more than one budget past its expiry belongs to an earlier burst, not this one.
+	if (deadline === undefined || now - deadline > READ_RETRY_BUDGET_MS) {
+		deadline = now + READ_RETRY_BUDGET_MS;
+		retryDeadlines.set(filePath, deadline);
+	}
+	return deadline - now;
+}

package/config/watcherArming.ts

@@ -0,0 +1,59 @@
+// See DESIGN.md, "`ready` means the watcher is armed". Platform is a parameter so a test on any
+// host can pin the darwin value, rather than only observing its own.
+export function armGraceMs(platform: string = process.platform): number {
+	return platform === 'darwin' ? 20 : 0;
+}
+
+const ARM_GRACE_MS = armGraceMs();
+
+/**
+ * The gate a config watcher opens when its chokidar watcher is really watching: the initial scan
+ * has finished *and* the platform's kernel-side warm-up has had its grace. Shared by
+ * `RootConfigWatcher` and `OptionsWatcher` because both read the config synchronously, which is
+ * what exposes the unarmed window in the first place.
+ */
+export class ArmGate {
+	#armed: boolean = false;
+	#timer: NodeJS.Timeout | undefined;
+	#graceMs: number;
+
+	// Taken as an argument, not read from the module constant, so the timer branch below is
+	// reachable from a host whose own platform has no grace — unit tests are ubuntu-only.
+	constructor(graceMs: number = ARM_GRACE_MS) {
+		this.#graceMs = graceMs;
+	}
+
+	get armed(): boolean {
+		return this.#armed;
+	}
+
+	// Synchronous where the platform needs no grace, so a watcher with no warm-up to wait out
+	// keeps reading inside chokidar's own dispatch.
+	arm(onArmed: () => void): void {
+		if (this.#armed || this.#timer) return;
+		if (!this.#graceMs) {
+			this.#armed = true;
+			onArmed();
+			return;
+		}
+		this.#timer = setTimeout(() => {
+			this.#timer = undefined;
+			this.#armed = true;
+			onArmed();
+		}, this.#graceMs);
+	}
+
+	// Drops a grace still counting down, for a watcher that is closing.
+	cancel(): void {
+		clearTimeout(this.#timer);
+		this.#timer = undefined;
+	}
+
+	// A replacement watcher has its own scan and its own unarmed window, so the gate arms again
+	// with it: for a file that is *absent* when the replacement scans, chokidar reports `ready` and
+	// nothing else, and the arming re-read is the only thing that would notice.
+	reset(): void {
+		this.cancel();
+		this.#armed = false;
+	}
+}

package/config-root.schema.json

@@ -172,6 +172,10 @@
 					]
 				},
 				"maxHeapMemory": { "type": "number", "description": "Heap memory limit per thread (MB)." },
+				"maxIsolated": {
+					"type": "integer",
+					"description": "Most applications marked isolated that get a dedicated worker thread; the rest fail to load. Default 8."
+				},
 				"heapSnapshotNearLimit": { "type": "boolean", "description": "Take a heap snapshot when near the heap limit." },
 				"preload": {
 					"oneOf": [{ "type": "string" }, { "type": "array", "items": { "type": "string" } }],

package/dataLayer/DESIGN.md

@@ -0,0 +1,179 @@
+# dataLayer/ — Design notes
+
+Backup and restore, version gating, system-table bootstrap and storage migration.
+
+**Read this when:** touching `rocksdbBackup.ts`, `restoreMarker.ts`, `blobBackup.ts`, `hdbInfoController.ts`, `bin/copyDb.ts` or `json/systemSchema.json`.
+
+Index of every design note: [DESIGN.md](../DESIGN.md).
+
+---
+
+## Version gate at startup: downgrades prompt, and only the minor direction is confirmable
+
+`getVersionUpdateInfo()` (`dataLayer/hdbInfoController.ts`) compares the store's `data_version_num` (latest `system.hdb_info` record) against the binary's `packageJson.version` on every start. Data newer than binary by a **major** version → hard refusal. Newer by a **minor** version → `forceDowngradePrompt()` asks for confirmation; answering yes records the data version back down to the binary's version and boots (upgrade directives are deliberately additive/downgrade-compatible — see the struct-mode section above and `patchHdbSecretIsHashAttribute` in `upgrade/directives/5-2-0.ts`).
+
+- The prompt's answer can be supplied non-interactively via `CONFIRM_DOWNGRADE` — env var or `--CONFIRM_DOWNGRADE` CLI arg; argv wins (`assignCMDENVVariables`). With no override and no TTY on stdin, the prompt throws instead of blocking on stdin forever (#2046 — services/CI hung with nothing in the log; the mismatch is also logged to hdb.log now).
+- Upgrades never prompt (see the rationale comment in `bin/upgrade.js`); only the downgrade direction confirms. `upgradeCertsPrompt()` on the 4.x upgrade path still has the block-on-stdin hazard.
+- Test-suite gotcha: a suite that supplies the override via `process.argv` affects every later test file in the same mocha process — save and restore `process.argv` in `before`/`after` (see `unitTests/dataLayer/hdbInfoController.test.js`).
+
+## Opening a source LMDB DBI for migration must thread through `compression`
+
+When `migrateOnStart` opens a source LMDB primary store to read records out for the RocksDB copy, it constructs an `OpenDBIObject` and calls `sourceRootStore.openDB(key, dbiInit)`. Critically, the per-attribute `compression` setting from the corresponding `__dbis__` entry must be assigned onto `dbiInit` before that call — `dbiInit.compression = attribute.compression`. Without it, lmdb-js doesn't install its decompression layer; every read on the DBI returns raw compressed bytes. msgpackr then misreads bytes in the `0x40–0x7F` range as shared-structure refs, calls `loadStructures` → decodes the (also compressed) structures buffer → finds more bytes in that range → recurses → stack overflow.
+
+Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match.
+
+The persisted `compression` value itself is LMDB-era and loosely shaped: `getDefaultCompression()` historically stored whatever falsy value the config resolved to (`''`, `false`, `null`) when `storage.compression` was disabled, and `{ startingOffset, threshold, dictionary? }` when enabled. lmdb-js interprets falsy as "no compression", but rocksdb-js >= 2.6 validates the option strictly (`''`/booleans throw `Unsupported compression algorithm`) and treats UNSET as "use the build default (lz4)" — the inverse default of lmdb. Every RocksDB open must therefore route through `toRocksCompression()` in `resources/databases.ts` (applied inside `openRocksDatabase`, the single chokepoint), which maps defined-falsy → `'none'` and enabled-without-an-algorithm → an explicit lz4 request when available. Don't pass persisted attribute compression to a RocksDB open directly.
+
+`bin/copyDb.ts`'s `openRocksDb` is part of that chokepoint, not an exception to it. This is about the bytes migration writes, not about a later failure: `copyDbToRocks()` closes every target handle before the staging directory is renamed, and rocksdb-js permits an explicit codec change across a close/reopen, so the runtime would open the migrated database fine either way. But a migration that ignores the configured codec writes the entire dataset uncompressed, and those SST/blob files then keep their original codec until write traffic rewrites them — a full LMDB→RocksDB migration is the one moment the whole dataset is written at once, so it is exactly when the deployment's codec should apply.
+
+## System table bootstrap: `systemSchema.json` + upgrade directive
+
+Adding a new system table (e.g. `hdb_deployment` in #641 Slice A) requires three changes:
+
+1. **`json/systemSchema.json`** — the table entry. Fresh installs auto-create it via `utility/mount_hdb.ts:createTables()`, which iterates `Object.keys(systemSchema)` on first boot.
+2. **`utility/hdbTerms.ts`** — add the table name to `SYSTEM_TABLE_NAMES`.
+3. **`upgrade/directives/<version>.ts`** — provisions the table on existing installs that already have a system schema. Registered in `upgrade/directives/directivesController.ts` (which is otherwise empty — its `versions` Map gets populated by these imports). The directive shape is `{ version, sync_functions, async_functions }`; copy `5-1-0.ts` for the canonical pattern (uses `bridge.createTable` to match what `mount_hdb` does on a fresh install).
+
+   **Version the directive to the first release that ships the dependent code, not a later one.** Directives only run when `current_version < directive_version <= upgrade_version` (`directivesController.getVersionsForUpgrade`). The `hdb_deployment` directive was originally mis-tagged `5.2.0` while the deployment-recorder code shipped in `5.1.0`, so on every `5.0.x -> 5.1.x` upgrade the directive was filtered out (`5.2.0 > 5.1.x`) and the table never got created — breaking replicated `deploy_component` on peer nodes for the entire existing customer base. Caveat: `utility/common_utils.ts:compareVersions` strips trailing `.0` and therefore sorts a pre-release (`5.1.0-beta.1`) _above_ its GA (`5.1.0`), so an install already on a `5.1.0-beta.x` data version will not pick up a `5.1.0` directive when upgrading to GA; those pre-release installs need the table created by other means.
+
+System tables replicate by default. To opt out, add the name to `NON_REPLICATING_SYSTEM_TABLES` in `resources/databases.ts`. The check happens after table init and sets `table.replicate = false` per-node.
+
+If the table needs `audit: true`, set it both in the schema (for fresh installs) **and** on the `CreateTableObject` instance in the directive (for upgrades) — otherwise the two paths diverge.
+
+## RocksDB backup/restore: the restore lock + marker protocol (`dataLayer/restoreMarker.ts`, `dataLayer/rocksdbBackup.ts`)
+
+The `restore_backup` operation restores a user database on a live server by closing it across all
+worker threads, purging its directory (`backups.restore` with `purgeAllFiles`), and reloading it.
+Three non-obvious mechanics keep that safe:
+
+- **Two files in an isolated `` `restore` `` directory beside (never inside) the database directory**,
+  each keyed by `sha256(basename(dbPath)).slice(0,32)`: `<key>.lock`, an OS-level exclusive flock
+  (rocksdb-js `tryFileLock`, auto-released on process death), serializes restores; `<key>.restoring`,
+  a marker written+fsynced (file _and_ the metadata directory) after the lock and before any
+  destructive step, means "a restore started and has not finished" (its first line records the
+  database directory name so the scan can map a marker back without decoding the key). The metadata
+  is hashed into a sibling directory rather than suffixed onto the database name (`<db>.restoring`)
+  for two reasons: a legal database literally named `orders.restoring` would otherwise be mistaken
+  for the restore marker of `orders`, and a 250-character name (the legal max) plus a `.restore.lock`
+  suffix exceeds `NAME_MAX` (255) on most filesystems. The directory name deliberately contains a
+  backtick — `schemaRegex` (the database-name validator) forbids only `/` and a backtick among
+  filesystem-legal characters — so it can never collide with a legal database name, including a
+  database literally named `.restore` (which _is_ a legal name; a plain `.restore/` directory would
+  be exactly that database's directory). Because the startup scan opens any `CURRENT`+`MANIFEST-`
+  directory without re-applying `schemaRegex`, it also explicitly skips the reserved `` `restore` ``
+  entry so an out-of-band directory at that name is never loaded as a database. Startup/rescan
+  detection (`databasesBlockedByRestore` → `scanBlockedRestores` in `dataLayer/restoreMarker.ts`)
+  reads the metadata directory and checks the **marker first**, only probing the lock when the marker exists —
+  probes take the flock and are mutually exclusive across threads, so probing the (persistent) lock
+  file of every long-ago-restored database on every rescan would make concurrent rescans misclassify
+  healthy databases as in-progress. Marker-present + lock-held = restore in progress (don't load);
+  marker-present + lock-free = crashed mid-restore (don't load; rerun the restore to recover).
+- **A recovery restore must not clear a pre-existing marker on a pre-destruction failure.**
+  `beginRestore` returns `preexisting: true` when a `.restoring` marker was already present (this run
+  is a recovery over a possibly half-purged directory). If such a run fails _before_ any destruction
+  (e.g. `verifyDatabaseClosed` finds a leaked handle), it must leave the marker in place — clearing
+  it and broadcasting a reload would surface the earlier attempt's partial/corrupt directory as
+  healthy. Only a _fresh_ marker on a _previously healthy_ database that failed before destruction is
+  safe to clear.
+- **The ITC close broadcast is best-effort, so closure is verified before the purge.** The SCHEMA
+  broadcast (`signalSchemaChange`) resolves after remote handlers complete but times out at 30s
+  "best-effort", swallows errors, and never reaches job-worker threads at all (their ports are
+  excluded from broadcasts to avoid re-entrant deadlocks). A destructive purge cannot trust it:
+  `restoreBackup` polls rocksdb-js `registryStatus()` (process-global across worker threads) until
+  the database path has no open instance, and aborts with a 409 — _cleaning up the marker, since
+  nothing was destroyed_ — if handles remain.
+- **Online restore is impossible for a database a component holds open — and that failure is
+  correct.** rocksdb-js's registry is process-global but records only a per-path refCount, with no
+  attribution to a thread or component; Harper keeps no component→database ownership map. So when a
+  loaded component (or the `system` database, which Harper itself never stops while running) holds
+  its own handle on the target database, `registryStatus()` stays non-zero, Harper can neither
+  identify nor force-close that handle, and an in-place purge would corrupt a live instance.
+  `verifyDatabaseClosed` therefore waits only a short grace period (`DATABASE_CLOSE_WAIT_MS`, for a
+  just-finished job worker's own close to drain) and then fails fast with a 409 that points at
+  running the operation offline (`harper restore_backup` with the server stopped, where no
+  components are loaded and nothing holds the database open). Offline restore is the supported path
+  for component-held and `system` databases; online restore serves databases not actively held by a
+  component. The CLI exposes each backup operation under its operation name only (`create_backup`,
+  `restore_backup`, …) — no hyphenated alias — and `bin/backup.ts` routes it to a reachable server
+  or, when the local server is stopped, to the equivalent offline function.
+- **Job workers must release their RocksDB handles on exit, or the closure check can never pass.**
+  rocksdb-js's registry is process-global across worker threads, and a thread that exits WITHOUT
+  closing leaks its handles (the refCount never drops); the only alternative, `shutdown()`, tears
+  down rocksdb for the _entire_ process. A job worker (`server/jobs/jobProcess.ts`) opens the whole
+  database graph via `getDatabases()` and exits when the job finishes — and `create_backup` is
+  itself a job, so before any `restore_backup` there is always at least one exited job worker that
+  touched the database. Without cleanup those leaked handles keep `registryStatus()` non-zero and
+  would fail the closure check even when no component holds the database. `jobProcess` therefore
+  calls `closeLoadedDatabases()` (`resources/databases.ts`) in its `finally`, closing every loaded
+  user database on that thread (the non-enumerable `system` DB is intentionally skipped), so an
+  exited job worker leaves no residual handle to be mistaken for a live holder.
+- **`dropDatabase` and `restore_backup` serialize on the same lock, not a check-then-act probe.**
+  A drop's `destroy()` interleaving with a restore's purge-and-copy on the same directory would gut
+  a "successful" restore (or vice versa). `dropDatabase` therefore _acquires_ the restore lock
+  (`acquireRestoreLock`, marker-less) for each RocksDB root store and holds it across the whole drop,
+  releasing in a `finally`; a restore in progress makes the acquire fail with 409, and a leftover
+  incomplete-restore marker (lock free, detected via `restoreMarkerPresent`, which — unlike
+  `checkRestoreState` — is safe while this thread holds the lock) is refused rather than dropped over.
+  `database()`'s on-demand open still uses the read-only `throwIfBlockedByRestore` (a
+  `create_table`/`create_schema` must not resurrect a half-purged directory as a fresh empty DB), but
+  the destructive drop path now uses the exclusive lock so the race is closed, not merely narrowed.
+- **The offline restore probes RocksDB's own `LOCK` file, and fails closed.** The offline path runs
+  only when the CLI sees no server (a PID heuristic; the PID file is briefly absent mid-`harper
+restart`), and `backups.restore`'s `purgeAllFiles` never takes RocksDB's lock — so before purging,
+  `restoreBackupOffline` opens the database to probe. It now takes the restore lock+marker _before_
+  probing (so a server that starts afterward sees the marker and refuses to load), and recognizes the
+  pinned rocksdb-js 2.5.0 lock error — a plain `Error` with no `code` and message
+  `IO error: While lock file: <db>/LOCK: Resource temporarily unavailable` (`isRocksDbLockError`) —
+  aborting with a 409 rather than purging a database another process holds open. Any _other_ open
+  failure (corrupt/half-restored) is exactly what restore recovers, so only a lock conflict aborts.
+
+Known limitation: the flock is process-owned; if the restore job's worker _thread_ dies without
+the process exiting, the lock stays held (restores 409) until Harper restarts. There is no typed
+native lock signal in rocksdb-js 2.5.0, so the offline probe relies on message matching; a native
+lock primitive is a rocksdb-js follow-on.
+
+## RocksDB managed backups: blob snapshots (`dataLayer/blobBackup.ts`)
+
+A database's file-backed blobs live in one or more roots _outside_ the RocksDB directory
+(`getBlobPathsForDatabaseName` in `resources/blob.ts` — one per configured `storage.blobPaths`, else
+`<hdb_root>/blobs/<database>`), so the engine's backup does not capture them. `create_backup`,
+`restore_backup`, `delete_backup`, `purge_backups`, and the streaming `get_backup` therefore handle
+blobs alongside the engine data (the `exclude_blobs` request option — default false — opts out for an
+engine-only backup):
+
+- **Managed backups** snapshot the blob roots to `<backupDir>/blobs/<backupId>/<rootIndex>/<relpath>`
+  — a full, non-incremental copy per backup, mirroring the binding's `transaction_logs/<id>/` layout.
+  Each enumerated entry is classified before capture: complete blobs and existing abort markers are
+  hard-linked when possible (copied across filesystems), `.repair` temporaries are omitted, and an
+  incomplete blob is replaced by a retryable PENDING (`0xfe`) marker. If a classified blob vanishes
+  before capture, a terminal ERROR (`0xff`) marker preserves its file id. A file reclaimed before its
+  parent directory is read is outside the snapshot. This keeps a snapshot inode from changing as a
+  live write finishes, while complete blobs remain safe to hard-link because published blob paths are
+  write-once. The snapshot is built in a `.tmp-<id>` sibling and atomically renamed so a failed create
+  leaves no partial snapshot. `restore_backup` purges each blob root and rewrites it from the snapshot;
+  `delete_backup` / `purge_backups` remove the corresponding snapshot directories.
+- **`get_backup`** appends the blob files to the same tar under `blobs/<rootIndex>/<relpath>`. The
+  binding's streaming backup finalizes its tar with exactly a 1024-byte (two-block) end-of-archive
+  marker; `createBackupStream` streams the native _plain_ tar while withholding that trailer
+  (verifying it is all-zero), appends the blob entries via `tar-stream` (whose `finalize` writes the
+  one real trailer), and gzips the combined stream itself when requested — so the binding is always
+  asked for a plain tar and compression happens after the append. No scratch disk. The same blob
+  classification rule applies: complete blobs are streamed, incomplete or post-enumeration missing
+  blobs become PENDING/ERROR marker entries, and repair temporaries are omitted.
+
+**Completion manifest (`dataLayer/backupManifest.ts`).** `create_backup` is two-phase: the engine
+backup (`rootStore.backup()`) resolves — and is immediately visible to `list_backups`/`verify_backup`/
+`restore_backup` — before the blob snapshot is copied. Without a completion record, a blob-snapshot
+failure (or a crash between the phases) would leave an engine backup that lists and verifies as
+healthy while silently missing its blobs, and a concurrent restore could pick a backup id whose
+snapshot is still being written and treat it as intentionally engine-only. So a manifest at
+`<backupDir>/manifests/<backupId>.json` — recording the blob-inclusion policy — is written
+(atomically, temp + rename) only after _both_ phases are durable, and a graceful blob-snapshot
+failure rolls back the just-created engine backup + partial snapshot. Consumers treat a backup id
+with no manifest as incomplete: `list_backups` hides it, `verify_backup`/`restore_backup` reject it
+(409 for a specific id, "no complete backups" for `latest`), and restore uses the manifest's `blobs`
+flag — not the mere presence of a snapshot dir — to decide whether to restore blobs (so an engine-only
+backup leaves live blobs untouched, and a manifest that claims blobs but has no snapshot is flagged
+corrupt by verify). This closes the "healthy-looking but incomplete" and concurrent-restore races;
+the remaining engine/blob point-in-time skew (a blob unlinked between the engine cut and the blob
+walk) is the documented best-effort limitation above.

package/dataLayer/harperBridge/ResourceBridge.ts

@@ -25,6 +25,7 @@ import type {
 import { collapseData } from '../../resources/tracked.ts';
 import { errorToString } from '../../utility/logging/harper_logger.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
+import { boundedAuditPruneEnd, raiseAuditFloor } from '../../resources/auditStore.ts';
 import { BridgeMethods } from './BridgeMethods.ts';
 import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.js';
 import { createBackupStream, resolveSingleRootStore } from '../rocksdbBackup.ts';
@@ -520,8 +521,26 @@ export class ResourceBridge extends BridgeMethods {
 			deleteObj.timestamp instanceof Date
 				? deleteObj.timestamp.getTime()
 				: typeof deleteObj.timestamp === 'string'
-					? Number.parseInt(deleteObj.timestamp)
+					? // Number, not Number.parseInt: parseInt takes a numeric PREFIX ('9999999999999oops' is a
+						// year-2286 bound to it, '1e3' is 1) where Number rejects both as NaN. Empty/whitespace is
+						// explicit because Number('') and Number('   ') are 0.
+						deleteObj.timestamp.trim() === ''
+						? Number.NaN
+						: Number(deleteObj.timestamp)
 					: deleteObj.timestamp;
+		// Audit keys are raw float64: NaN and negatives sort above every real timestamp (a range ending
+		// there spans the whole log), and Infinity records the unknown sentinel raiseAuditFloor never lifts.
+		// Reject anything but a finite non-negative number here, as the 400 it is rather than a server
+		// fault. `Number.isFinite` never coerces, so it is the type check too.
+		if (!Number.isFinite(before) || before < 0 || Object.is(before, -0))
+			throw handleHDBError(
+				new Error(),
+				`'timestamp' must be a non-negative epoch time or Date, received: ${String(deleteObj.timestamp)}`,
+				400,
+				undefined,
+				undefined,
+				true
+			);
 		const databaseName = deleteObj.database || deleteObj.schema || DEFAULT_DATABASE;
 		const table = getTable(deleteObj);
 		// A nonexistent table must not fall through to the no-table branch below — on RocksDB that
@@ -544,7 +563,14 @@ export class ResourceBridge extends BridgeMethods {
 			if (tables) {
 				for (const table of Object.values(tables)) {
 					if (table.primaryStore instanceof RocksDatabase) {
-						const deleted = table.primaryStore.purgeLogs({ before, includeEntryCounts: true });
+						// Clamp before recording: an operator-supplied bound has no ceiling of its own, and a
+						// floor above everything reachable never comes down (`raiseAuditFloor` only raises and
+						// `establishAuditFloor` skips a store that has a record), so a far-future bound would
+						// retire the floor for every table in this database. The purge takes the same clamped
+						// bound, so it cannot remove an entry the floor does not cover.
+						const pruneEnd = boundedAuditPruneEnd(table.auditStore, before);
+						raiseAuditFloor(table.auditStore, pruneEnd);
+						const deleted = table.primaryStore.purgeLogs({ before: pruneEnd, includeEntryCounts: true });
 						totalResults.log_files_deleted += deleted.length;
 						totalResults.entries_deleted += deleted.reduce((acc, file) => acc + file.entries, 0);
 						break;