@harperfast/harper 5.2.0-beta.4 → 5.2.0

package/resources/databases.ts

@@ -41,6 +41,15 @@ import { RocksIndexStore } from './RocksIndexStore.ts';
 import { when } from '../utility/when.ts';
 import { resolveRocksMemoryConfig } from '../utility/rocksMemoryConfig.ts';
 import { isProcessRunning } from '../utility/processManagement/processManagement.js';
+import {
+	acquireRestoreLock,
+	checkRestoreState,
+	releaseRestoreLock,
+	restoreMarkerPresent,
+	scanBlockedRestores,
+	RESTORE_META_DIR,
+	type RestoreLock,
+} from '../dataLayer/restoreMarker.ts';
 
 /**
  * Check if Harper is running in read-only mode.
@@ -167,8 +176,23 @@ export const databaseEventsEmitter = new EventEmitter<DatabaseWatcherEventMap>()
 export const tables: Tables = Object.create(null);
 export const databases: Databases = Object.create(null);
 
+/**
+ * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata
+ * carries values where a defined falsy value (false, '') means compression was explicitly
+ * disabled and true/{ threshold, ... } mean enabled with defaults. rocksdb-js >= 2.6
+ * validates `compression` strictly and treats UNSET as "use the build default (lz4)", so
+ * disabled must be mapped to 'none' explicitly, and true to unset.
+ */
+export function toRocksCompression(compression: unknown): unknown {
+	if (compression === true) return undefined;
+	if (compression !== undefined && !compression) return 'none';
+	return compression;
+}
+
 function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) {
 	options.disableWAL ??= true;
+	const legacyOptions = options as { compression?: unknown };
+	legacyOptions.compression = toRocksCompression(legacyOptions.compression);
 	// Apply read-only mode if enabled
 	if (isReadOnlyMode()) {
 		options.readOnly = true;
@@ -315,11 +339,18 @@ export function getDatabases(): Databases {
 	if (databasePath && existsSync(databasePath)) {
 		// First load all the databases from our main database folder
 		// TODO: Load any databases defined with explicit storage paths from the config
-		for (const databaseEntry of readdirSync(databasePath, { withFileTypes: true })) {
+		const entries = readdirSync(databasePath, { withFileTypes: true });
+		const blockedByRestore = databasesBlockedByRestore(databasePath);
+		for (const databaseEntry of entries) {
 			// in-progress migration staging dirs are not databases until atomically renamed into place
 			if (databaseEntry.name.endsWith(MIGRATING_DIR_SUFFIX)) continue;
+			// the restore-metadata directory is reserved: never load it as a database, even if a
+			// (out-of-band) RocksDB directory happens to occupy that reserved name — the API can't
+			// create it (schemaRegex forbids the backtick), but the scan opens any CURRENT+MANIFEST dir
+			if (databaseEntry.name === RESTORE_META_DIR) continue;
 			const dbName = basename(databaseEntry.name, '.mdb');
 			const dbPath = join(databasePath, databaseEntry.name);
+			if (blockedByRestore.has(dbName)) continue;
 
 			if (
 				databaseEntry.isFile() &&
@@ -376,8 +407,12 @@ export function getDatabases(): Databases {
 			const schemaConfig = schemaConfigs[dbName];
 			const databasePath = schemaConfig.path;
 			if (existsSync(databasePath)) {
-				for (const databaseEntry of readdirSync(databasePath, { withFileTypes: true })) {
-					if (databaseEntry.name.endsWith(MIGRATING_DIR_SUFFIX)) continue;
+				const entries = readdirSync(databasePath, { withFileTypes: true });
+				const blockedByRestore = databasesBlockedByRestore(databasePath);
+				for (const databaseEntry of entries) {
+					if (databaseEntry.name.endsWith(MIGRATING_DIR_SUFFIX)) continue; // migration staging dir
+					if (databaseEntry.name === RESTORE_META_DIR) continue; // reserved restore-metadata dir
+					if (blockedByRestore.has(basename(databaseEntry.name, '.mdb'))) continue;
 					if (databaseEntry.isFile() && extname(databaseEntry.name).toLowerCase() === '.mdb') {
 						readMetaDb(join(databasePath, databaseEntry.name), basename(databaseEntry.name, '.mdb'), dbName);
 					} else {
@@ -452,6 +487,29 @@ export function getDatabases(): Databases {
 	return databases;
 }
 
+/**
+ * Scan a databases directory's entries for restore lock/marker files and return the names of
+ * databases that must not be loaded: a held restore lock means a restore is in progress in some
+ * process; an unheld lock with a surviving `.restoring` marker means a restore was interrupted
+ * mid-purge (the directory may be partial garbage) and must be rerun. The files live *next to*
+ * the database directory, so this also covers a database whose directory is missing or empty.
+ */
+function databasesBlockedByRestore(databasePath: string): Set<string> {
+	const blocked = new Set<string>();
+	for (const [dbName, state] of scanBlockedRestores(databasePath)) {
+		if (state === 'in-progress') {
+			logger.warn(`A restore of database '${dbName}' is in progress; not loading it`);
+			blocked.add(dbName);
+		} else if (state === 'incomplete') {
+			logger.error(
+				`Incomplete restore of database '${dbName}' detected (a restore started but did not finish); not loading it — rerun the restore to recover`
+			);
+			blocked.add(dbName);
+		}
+	}
+	return blocked;
+}
+
 /**
  * This is responsible for reading the internal dbi of a single database file to get a list of all the tables and
  * their indexed or registered attributes
@@ -916,18 +974,12 @@ function setTable(tables, tableName, Table) {
 	return Table;
 }
 /**
- * Get root store for a database
- * @param options
- * @returns
+ * Resolve the directory that holds (or would hold) a database's storage, from the databases
+ * config, storage path config/env, or the hdb root — without opening anything. This is the
+ * parent directory selection used by `database()`; a RocksDB database lives at
+ * `join(resolveDatabaseStorageRoot(...), databaseName)`.
  */
-export function database({ database: databaseName, table: tableName }) {
-	if (!databaseName) databaseName = DEFAULT_DATABASE_NAME;
-	getDatabases();
-	ensureDB(databaseName);
-	const definedDatabase = definedDatabases.get(databaseName);
-	if ((definedDatabase as any)?.rootStore) {
-		return (definedDatabase as any).rootStore;
-	}
+export function resolveDatabaseStorageRoot(databaseName: string, tableName?: string): string {
 	const databaseConfig = envGet(CONFIG_PARAMS.DATABASES) || {};
 	if (process.env.SCHEMAS_DATA_PATH) {
 		databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
@@ -952,6 +1004,35 @@ export function database({ database: databaseName, table: tableName }) {
 			`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`
 		);
 	}
+	return databasePath;
+}
+
+/**
+ * Resolve the directory path of a RocksDB database (whether or not it exists or is loaded).
+ */
+export function resolveDatabasePath(databaseName: string): string {
+	return join(resolveDatabaseStorageRoot(databaseName), databaseName);
+}
+
+/**
+ * Get root store for a database
+ * @param options
+ * @returns
+ */
+export function database({ database: databaseName, table: tableName }) {
+	if (!databaseName) databaseName = DEFAULT_DATABASE_NAME;
+	getDatabases();
+	ensureDB(databaseName);
+	const definedDatabase = definedDatabases.get(databaseName);
+	if ((definedDatabase as any)?.rootStore) {
+		return (definedDatabase as any).rootStore;
+	}
+	const databaseConfig = envGet(CONFIG_PARAMS.DATABASES) || {};
+	if (process.env.SCHEMAS_DATA_PATH) {
+		databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
+	}
+	const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
+	const databasePath = resolveDatabaseStorageRoot(databaseName, tableName);
 
 	let rootStore: RootDatabaseKind;
 	const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || envGet(CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
@@ -959,6 +1040,10 @@ export function database({ database: databaseName, table: tableName }) {
 		const path = join(databasePath, tablePath ? tableName : databaseName);
 		rootStore = rocksdbDatabaseEnvs.get(path);
 		if (!rootStore || rootStore.status === 'closed') {
+			// this on-demand open (create_table/create_database and friends) must not resurrect a
+			// database that a restore is rewriting (or left half-purged) — the scan-time restore
+			// checks don't cover this path
+			throwIfBlockedByRestore(path, databaseName);
 			rootStore = openRocksDatabase(path, {
 				disableWAL: false,
 				enableStats: true,
@@ -981,6 +1066,51 @@ export function database({ database: databaseName, table: tableName }) {
 	if (definedDatabase) (definedDatabase as any).rootStore = rootStore;
 	return rootStore;
 }
+function throwIfBlockedByRestore(dbPath: string, databaseName: string): void {
+	const restoreState = checkRestoreState(dbPath);
+	if (restoreState !== 'clear') {
+		const error: any = new Error(
+			restoreState === 'in-progress'
+				? `Database '${databaseName}' is being restored; retry when the restore completes`
+				: `Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`
+		);
+		error.statusCode = 409;
+		throw error;
+	}
+}
+
+/**
+ * Take the per-database restore lock for a drop, refusing (409) if a restore holds it (in-progress)
+ * or a crashed restore left a marker (incomplete). Pushes the acquired lock onto `held` so the
+ * caller releases it after the drop. On refusal, releases anything already held and throws.
+ *
+ * The lock is not reentrant within a process, so a path already in `held` must be skipped — every
+ * table in a RocksDB database shares one root store (and one lock path), and re-acquiring it in the
+ * same drop would spuriously 409 on the second table.
+ */
+function lockDatabaseForDrop(dbPath: string, databaseName: string, held: RestoreLock[]): void {
+	if (held.some((h) => h.dbPath === dbPath)) return;
+	let lock: RestoreLock;
+	try {
+		lock = acquireRestoreLock(dbPath);
+	} catch (error) {
+		for (const h of held) releaseRestoreLock(h);
+		throw error; // 409: a restore is in progress and holds the lock
+	}
+	// We now hold the lock, so no restore is active. A surviving marker is therefore debris from a
+	// crashed restore (incomplete) — refuse rather than delete a directory that still needs recovery.
+	if (restoreMarkerPresent(dbPath)) {
+		releaseRestoreLock(lock);
+		for (const h of held) releaseRestoreLock(h);
+		const error: any = new Error(
+			`Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`
+		);
+		error.statusCode = 409;
+		throw error;
+	}
+	held.push(lock);
+}
+
 /**
  * Delete the database
  * @param databaseName
@@ -990,17 +1120,105 @@ export async function dropDatabase(databaseName) {
 	const dbTables = databases[databaseName];
 	let rootStore;
 
-	for (const tableName in dbTables) {
-		const table = dbTables[tableName];
-		rootStore = table.primaryStore.rootStore;
-		lmdbDatabaseEnvs.delete(rootStore.path);
-		rocksdbDatabaseEnvs.delete(rootStore.path);
+	// Hold the per-database restore lock across the entire drop so its file deletion can never
+	// interleave with a restore's purge-and-copy on the same directory — a destroy landing after a
+	// restore's copy would gut a "successful" restore, and vice versa. Restore takes the same lock
+	// (before writing its marker), so both operations serialize on this one primitive rather than on
+	// a check-then-act marker probe. Released in the finally below.
+	const restoreLocks: RestoreLock[] = [];
+	try {
+		for (const tableName in dbTables) {
+			const table = dbTables[tableName];
+			rootStore = table.primaryStore.rootStore;
+			if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+			lmdbDatabaseEnvs.delete(rootStore.path);
+			rocksdbDatabaseEnvs.delete(rootStore.path);
+		}
+
+		for (const tableName in dbTables) {
+			databaseEventsEmitter.emit('dropTable', tableName, databaseName);
+		}
+
+		if (databaseName === 'data') {
+			for (const tableName in tables) {
+				delete tables[tableName];
+			}
+			delete tables[DEFINED_TABLES];
+		}
+		delete databases[databaseName];
+
+		databaseEventsEmitter.emit('dropDatabase', databaseName);
+
+		if (rootStore) {
+			if (rootStore.status === 'open') {
+				if (rootStore instanceof RocksDatabase) {
+					rootStore.close();
+					rootStore.destroy();
+				} else {
+					await rootStore.close();
+					await unlink(rootStore.path);
+				}
+			}
+		} else {
+			rootStore = database({ database: databaseName, table: null });
+			// a tableless database resolves its root store here rather than in the loop above, so take
+			// the drop lock now (still before any destructive step)
+			if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+			if (rootStore instanceof RocksDatabase) {
+				rootStore.close();
+				rootStore.destroy();
+			} else if (rootStore.status === 'open') {
+				await rootStore.close();
+				await unlink(rootStore.path);
+			}
+		}
+
+		await deleteRootBlobPathsForDB(rootStore);
+	} finally {
+		for (const lock of restoreLocks) releaseRestoreLock(lock);
 	}
+}
 
+/**
+ * Close a RocksDB database's store handles on the current thread and unregister it, without
+ * touching its files. Used by the restore_backup flow: every thread must release its handles so
+ * `backups.restore()` can purge and rewrite the (fully closed) database directory. A subsequent
+ * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in
+ * progress, per the restore marker checks in the scan).
+ */
+export function closeDatabase(databaseName: string): boolean {
+	const dbTables = databases[databaseName];
+	if (!dbTables) return false;
+	const rootStores = new Set<any>();
+	const closeStore = (store: any, description: string) => {
+		try {
+			store?.close?.();
+		} catch (error) {
+			logger.warn(`Error closing ${description} while closing database ${databaseName}:`, error);
+		}
+	};
 	for (const tableName in dbTables) {
-		databaseEventsEmitter.emit('dropTable', tableName, databaseName);
+		const table: any = dbTables[tableName];
+		if (!table?.primaryStore) continue;
+		if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore);
+		for (const indexName in table.indices || {}) {
+			closeStore(table.indices[indexName], `index ${tableName}.${indexName}`);
+		}
+		closeStore(table.primaryStore, `table ${tableName}`);
 	}
-
+	// a database with no tables (an empty schema, or one whose tables were all dropped) still holds
+	// an open root store, tracked only on the defined-database entry rather than any table — include
+	// it so its handles are released too (the Set dedupes it against the per-table root stores above)
+	const definedRoot = (definedDatabases?.get(databaseName) as any)?.rootStore;
+	if (definedRoot) rootStores.add(definedRoot);
+	for (const rootStore of rootStores) {
+		closeStore(rootStore.dbisDb, 'attributes store');
+		closeStore(rootStore, 'root store');
+		lmdbDatabaseEnvs.delete(rootStore.path);
+		rocksdbDatabaseEnvs.delete(rootStore.path);
+	}
+	const definedDatabase = definedDatabases?.get(databaseName);
+	if (definedDatabase) (definedDatabase as any).rootStore = undefined;
 	if (databaseName === 'data') {
 		for (const tableName in tables) {
 			delete tables[tableName];
@@ -1008,31 +1226,41 @@ export async function dropDatabase(databaseName) {
 		delete tables[DEFINED_TABLES];
 	}
 	delete databases[databaseName];
+	return true;
+}
 
-	databaseEventsEmitter.emit('dropDatabase', databaseName);
-
-	if (rootStore) {
-		if (rootStore.status === 'open') {
-			if (rootStore instanceof RocksDatabase) {
-				rootStore.close();
-				rootStore.destroy();
-			} else {
-				await rootStore.close();
-				await unlink(rootStore.path);
+/**
+ * Close every RocksDB (user) database this thread has open, releasing its native handles.
+ *
+ * rocksdb-js's registry is process-global across worker threads, and a thread that exits WITHOUT
+ * closing leaks its handles (the process-global refCount never drops), while the only alternative,
+ * `shutdown()`, tears down rocksdb for the entire process. So a worker thread that opens databases
+ * and then exits — notably a job worker (jobProcess), which opens the whole database graph via
+ * `getDatabases()` and exits when the job finishes — must close its handles explicitly, or those
+ * handles linger process-wide (and, e.g., block an online `restore_backup` from confirming the
+ * database is closed). The `system` database is intentionally left open: it is non-enumerable here
+ * (skipped by the loop), is never restored online, and the exiting worker may still touch the job
+ * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`.
+ */
+export function closeLoadedDatabases(): void {
+	// snapshot the names first: closeDatabase() deletes from `databases` as it goes
+	for (const databaseName of Object.keys(databases)) {
+		const dbTables = databases[databaseName];
+		if (!dbTables) continue;
+		let isRocks = false;
+		for (const tableName in dbTables) {
+			if (dbTables[tableName]?.primaryStore?.rootStore instanceof RocksDatabase) {
+				isRocks = true;
+				break;
 			}
 		}
-	} else {
-		rootStore = database({ database: databaseName, table: null });
-		if (rootStore instanceof RocksDatabase) {
-			rootStore.close();
-			rootStore.destroy();
-		} else if (rootStore.status === 'open') {
-			await rootStore.close();
-			await unlink(rootStore.path);
+		// a tableless database exposes no table root store, so also check the defined-database
+		// entry — otherwise its open root store would leak on worker exit
+		if (!isRocks && (definedDatabases?.get(databaseName) as any)?.rootStore instanceof RocksDatabase) {
+			isRocks = true;
 		}
+		if (isRocks) closeDatabase(databaseName);
 	}
-
-	await deleteRootBlobPathsForDB(rootStore);
 }
 // HNSW_NO_AUTOVERSION kill-switch: when set, a NEW index initializes as legacy rather than
 // versioned. process.env values are strings, so a bare truthiness check would treat "0"/"false"
@@ -2052,7 +2280,9 @@ export function getDefaultCompression() {
 	if (STORAGE_COMPRESSION_DICTIONARY)
 		LMDB_COMPRESSION_OPTS['dictionary'] = readFileSync(STORAGE_COMPRESSION_DICTIONARY);
 	if (STORAGE_COMPRESSION_THRESHOLD) LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
-	return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
+	// normalize disabled to false so a falsy config value ('' or null) is never persisted
+	// into table metadata as-is (openRocksDatabase maps defined-falsy to 'none')
+	return LMDB_COMPRESSION ? LMDB_COMPRESSION_OPTS : false;
 }
 
 /**

package/resources/transaction.ts

@@ -42,9 +42,6 @@ export function transaction<T>(
 	if (context.replicatedConfirmation) transaction.replicatedConfirmation = context.replicatedConfirmation;
 	if (context.sourceApply) transaction.sourceApply = true;
 	transaction.setContext(context);
-
-	// create a resource cache so that multiple requests to the same resource return the same resource
-	if (!context.resourceCache) context.resourceCache = new Map();
 	let result;
 	try {
 		result =

package/server/DESIGN.md

@@ -119,6 +119,22 @@ Components register listeners with optional `before: 'name'` / `after: 'name'` o
 
 The default WebSocket upgrade handler is registered automatically inside `onWebSocket()` the first time it runs for a given port.
 
+### Application mounts (`host` / `urlPath` in the root config)
+
+An operator mounts an application by putting `host`/`urlPath` on its entry in the **root** config; `components/scopeMount.ts` models it and the loader threads it into every `Scope` for that application (both load paths — the root-config `package` recursion and the components-root directory scan).
+
+**The mount is applied at exactly one place: `Scope.routeFor()`, used by the `scope.server` proxy.** Do not push it anywhere else. In particular, do not compose it into the plugin config the entry pipeline reads: `entry.urlPath` is what `graphqlSchema` and `jsResource` derive **resource** paths from, and the router strips the mount _before_ REST resolves them. Composing it there registers a table at `/v1/Thing` while REST looks up `Thing`, and every mounted REST route 404s. A single-app `static` test will not catch it — static de-prefixes its own map keys, so it stays self-consistent either way.
+
+Consequences worth knowing:
+
+- Everything inside an application addresses itself **mount-relative**. Only two things need the absolute path: code that emits a URL back to the client (use `Scope.externalBasePath()` — static's redirect `Location`), and code that bypasses the routed chain (legacy fastify registers on the bare server, so its route prefix must be the full external path). `static.ts`'s mount-root redirect gates on the _external_ base path, not the plugin-local one — a root-level static plugin (`baseURLPath === '/'`) still needs the redirect when the application itself carries a mount, since the client-visible mount root is then `externalBaseURLPath`, not `/`.
+- A plugin registering per-mount state must key it on `Scope.routeFor()`'s resolved route, not on the parts it composes from — distinct `(mount, pluginUrlPath)` pairs can flatten to the same string (`/a`+`bc` and `/ab`+`c`). `REST.ts`'s `startedMounts` does this; it replaced a process-global `started` flag that silently 404'd the second mounted application's REST API. `handleApplication` also closes over `resources`/`httpOptions` per call rather than a module-level var, and skips deploy pre-flight validation scopes (`scope.isTransientValidation`) entirely — registering handlers from a throwaway validation scope would splice a validation run into the live request path and permanently mark that mount started, silently skipping the real scope's later registration.
+- A mount is routing, **not** isolation: exported resources stay instance-wide, and a `host` mount cannot constrain legacy fastify routes — `fastifyRoutes.ts` refuses to load (throws) rather than warn when a `host` mount is configured, since the fallback really is reachable on every host.
+- An invalid mount (unparseable `host`/`urlPath`) fails the application **closed**: `componentLoader.tryRootConfigMount` skips loading it entirely rather than falling back to unmounted access — loading unconstrained would silently drop the isolation the operator asked for, which is worse than not loading at all.
+- Two applications mounted at different routes can register same-named middleware (e.g. both enable `rest`) without colliding: `middlewareChain.resolveRoutedChains` resolves `before`/`after` name references against a registry scoped to that route's own group, falling back to a _global_ registry that only holds genuinely unmounted entries (e.g. `authentication`) — never another mounted route's entries.
+- `host` matching reads `request.host` (Harper's `Request.host` getter), not the raw `Host` header — HTTP/2 clients send `:authority`, never `Host`, so reading the header directly silently 404s every host-mounted app under h2 while h1 keeps working. `hostnameFromHeader` also strips a trailing dot (`api.example.com.`, the absolute-FQDN form some resolvers emit) since it names the same origin.
+- `scopeMount.normalizeMountHost` validates against the same grammar as the `deploy_component` operation's `host` field (bare DNS hostname or IPv6 literal) and throws otherwise, so a hand-typed root-config `host` with a port/scheme/path fails the application closed too, instead of loading it unreachably. `nestScopeMount` logs a warning when a child's `host` is discarded by the parent-authority rule, so that isn't silent.
+
 ---
 
 ## Resource ↔ HTTP boundary

package/server/REST.ts

@@ -23,7 +23,6 @@ import { entryMap } from '../resources/RecordEncoder.ts';
 const { errorToString, errorForLog } = harperLogger;
 const etagBytes = new Uint8Array(8);
 const etagFloat = new Float64Array(etagBytes.buffer, 0, 1);
-let httpOptions = {};
 
 const OPENAPI_DOMAIN = 'openapi';
 
@@ -118,7 +117,7 @@ async function findInactiveComponent(url: string): Promise<string | undefined> {
 	}
 }
 
-async function http(request: Request, nextHandler) {
+async function http(request: Request, nextHandler, resources: Resources, httpOptions: any) {
 	const headersObject = request.headers.asObject;
 	const isSse = headersObject.accept === 'text/event-stream';
 	const method = isSse ? 'CONNECT' : request.method;
@@ -395,24 +394,41 @@ async function http(request: Request, nextHandler) {
 	}
 }
 
-let started = false;
-let resources: Resources;
+// One registration per distinct route mount, not one per process. Two applications that both
+// enable `rest` under different root-config mounts each need their own chain — a single global
+// `started` flag registered only the first, silently 404ing the other application's REST API.
+const startedMounts = new Set<string>();
 let addedMetrics;
 let connectionCount = 0;
 
 export function handleApplication(scope: import('../components/Scope.ts').Scope) {
-	httpOptions = scope.options.getAll();
+	// A deploy pre-flight validation scope exists only to validate config, and may share a live
+	// component's identity — registering real HTTP/WS handlers from it would splice a validation
+	// run into the live request path, and permanently marking its mount as started below would
+	// leave the REAL scope's later registration silently skipped (review finding).
+	if (scope.isTransientValidation) return;
+
+	const httpOptions = scope.options.getAll();
 	if ((httpOptions as any).includeExpensiveRecordCountEstimates) {
 		// If they really want to enable expensive record count estimates
 		(Request.prototype as any).includeExpensiveRecordCountEstimates = true;
 	}
-	resources = scope.resources;
-	if (started) return;
-	started = true;
+	// Captured per-mount in this closure — not a shared module-level variable — so each mount's
+	// registered handler always reads ITS OWN resources/options, not whichever mount happened to
+	// register last (review finding: two mounts sharing mutable module globals).
+	const resources = scope.resources;
+	// Key on the route this registration will actually answer on, not on the parts it is composed
+	// from: two configurations that resolve to the same route are the same chain and must dedupe,
+	// while two that resolve to different routes must not collide. Concatenating the mount and the
+	// plugin's raw urlPath did collide (mount '/a' + 'bc' and mount '/ab' + 'c' both gave '/abc').
+	const route = scope.routeFor(httpOptions as any);
+	const mountKey = `${route.host ?? ''}|${route.urlPath ?? ''}`;
+	if (startedMounts.has(mountKey)) return;
+	startedMounts.add(mountKey);
 	scope.server.http(
 		async (request: any, nextHandler) => {
 			if (request.isWebSocket) return;
-			return http(request, nextHandler);
+			return http(request, nextHandler, resources, httpOptions);
 		},
 		{ after: 'authentication', ...(httpOptions as any) }
 	);

package/server/fastifyRoutes.ts

@@ -1,6 +1,7 @@
 import { dirname, basename } from 'path';
 import { existsSync } from 'fs';
 import { deriveRoutePrefix } from './fastifyRoutes/helpers/deriveRoutePrefix.ts';
+import { resolveBaseURLPath } from '../components/resolveBaseURLPath.ts';
 import fastify from 'fastify';
 import fastifyCors from '@fastify/cors';
 import requestTimePlugin from './serverHelpers/requestTimePlugin.js';
@@ -38,6 +39,19 @@ const routeFolders = new Set();
 export function handleApplication(scope: import('../components/Scope.ts').Scope) {
 	// if we have a secure port, need to use the secure HTTP server for fastify (it can be used for HTTP as well)
 	const isHttps = (scope.options.getAll() as { securePort?: number }).securePort > 0;
+	// Fastify is registered as a global fallback below (`server.http(built.server)` on the bare
+	// server, not the scoped one), so it never joins a routed middleware chain. A `urlPath` mount
+	// still works — it becomes the fastify route prefix, which is matched against the full external
+	// path — but a `host` mount cannot constrain it: these routes would stay reachable under every
+	// Host header, silently dropping the isolation the operator declared. Refuse to load rather than
+	// imply isolation the fallback doesn't provide (review finding) — contained to this component by
+	// the loader's per-component try/catch, so it does not take down the rest of the application.
+	const mountedHost = scope.mount?.host ?? (scope.options.getAll() as { host?: string })?.host;
+	if (mountedHost) {
+		throw new Error(
+			`Application '${basename(scope.appName)}' is mounted on host '${mountedHost}', but its fastifyRoutes cannot be constrained by host: legacy fastify routes are registered as a global fallback and would remain reachable on every host. Port them to server.http() to get host routing, or drop the host mount.`
+		);
+	}
 	scope.handleEntry(async (entry) => {
 		if (entry.eventType !== 'add') {
 			scope.requestRestart();
@@ -62,7 +76,12 @@ export function handleApplication(scope: import('../components/Scope.ts').Scope)
 		// ignores the deprecated `path` alias, so it can't express the app-name namespace.
 		// `appName` may be an absolute path (loaded via RUN_HDB_APP), hence basename().
 		const config = (scope.options.getAll() as { urlPath?: string; path?: string }) ?? {};
-		const prefix = deriveRoutePrefix(basename(scope.appName), config.urlPath ?? config.path);
+		// Fastify is registered on the bare server, so nothing strips the application's mount before
+		// it matches — unlike a routed chain, its prefix must be the full external path.
+		const prefix = deriveRoutePrefix(
+			basename(scope.appName),
+			scope.externalBasePath(resolveBaseURLPath(basename(scope.appName), config.urlPath ?? config.path))
+		);
 		if (!routeFolders.has(routeFolder)) {
 			routeFolders.add(routeFolder);
 			try {

package/server/itc/serverHandlers.js

@@ -12,7 +12,7 @@ const harperBridge =
 	require('../../dataLayer/harperBridge/harperBridge.ts');
 const process = require('process');
 const { isMainThread, workerData } = require('worker_threads');
-const { resetDatabases } = require('../../resources/databases.ts');
+const { resetDatabases, closeDatabase } = require('../../resources/databases.ts');
 
 /**
  * This object/functions are passed to the ITC client instance and dynamically added as event handlers.
@@ -46,6 +46,12 @@ async function schemaHandler(event) {
 	}
 
 	hdbLogger.trace(`ITC schemaHandler received schema event:`, event);
+	// restore_backup: this thread must release its store handles so the restore can purge and
+	// rewrite the database directory. The rescan below (resetDatabases) skips reloading it while
+	// the restoring marker is present, and reloads it on the completion signal (marker gone).
+	if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) {
+		closeDatabase(event.message.schema);
+	}
 	await cleanLmdbMap(event.message);
 	await syncSchemaMetadata(event.message);
 	for (let listener of schemaListeners) {

package/server/jobs/jobProcess.ts

@@ -77,7 +77,24 @@ const JOB_ID = JOB_NAME.substring(4);
 		jobObj.message = err.message ? err.message : err;
 		jobObj.end_datetime = moment().valueOf();
 	} finally {
-		await jobs.updateJob(jobObj);
+		// A rejected updateJob must not skip handle cleanup and exit scheduling below (that would
+		// leak this worker's process-global RocksDB handles and leave the worker hanging).
+		try {
+			await jobs.updateJob(jobObj);
+		} catch (updateErr) {
+			harperLogger.error('Error updating job record on job worker exit:', updateErr);
+		}
+		// Release this worker's RocksDB handles before it exits. A job worker opens the whole
+		// database graph via getDatabases(); rocksdb-js's registry is process-global and a thread
+		// that exits without closing leaks its handles process-wide, which (among other costs)
+		// blocks an online restore_backup from confirming the target database is closed. Best
+		// effort — never let cleanup mask the job result.
+		try {
+			const { closeLoadedDatabases } = await import('../../resources/databases.ts');
+			closeLoadedDatabases();
+		} catch (closeErr) {
+			harperLogger.warn('Error releasing database handles on job worker exit:', closeErr);
+		}
 		// On Bun 1.3.13, calling process.exit() in a worker thread with lmdb-js loaded
 		// while sibling workers are running causes a NAPI fatal error crash. Unref
 		// parentPort (which broadcastWithAcknowledgement may have ref'd during schema

package/server/jobs/jobRunner.ts

@@ -10,6 +10,7 @@ import log from '../../utility/logging/harper_logger.ts';
 import * as jobs from './jobs.ts';
 import * as hdbExport from '../../dataLayer/export.ts';
 import * as hdbDelete from '../../dataLayer/delete.ts';
+import * as rocksdbBackup from '../../dataLayer/rocksdbBackup.ts';
 import * as threadsStart from '../threads/manageThreads.js';
 import * as transactionLog from '../../utility/logging/transactionLog.ts';
 import * as restart from '../../bin/restart.ts';
@@ -79,6 +80,15 @@ async function parseMessage(runnerMessage: any) {
 		case hdbTerms.JOB_TYPE_ENUM.delete_transaction_logs_before:
 			await runJob(runnerMessage, transactionLog.deleteTransactionLogsBefore);
 			break;
+		case hdbTerms.JOB_TYPE_ENUM.create_backup:
+			await runJob(runnerMessage, rocksdbBackup.createBackup);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.verify_backup:
+			await runJob(runnerMessage, rocksdbBackup.verifyBackup);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.restore_backup:
+			await runJob(runnerMessage, rocksdbBackup.restoreBackup);
+			break;
 		case hdbTerms.JOB_TYPE_ENUM.restart_service:
 			await runJob(runnerMessage, restart.restartService);
 			return `Restarting ${runnerMessage.json.service}`;

package/server/jobs/jobs.ts

@@ -20,6 +20,7 @@ import * as hdbUtil from '../../utility/common_utils.ts';
 import { promisify } from 'util';
 import moment from 'moment';
 import * as fileLoadValidator from '../../validation/fileLoadValidator.ts';
+import * as rocksdbBackup from '../../dataLayer/rocksdbBackup.ts';
 import bulkDeleteValidator from '../../validation/bulkDeleteValidator.ts';
 import { deleteTransactionLogsBeforeValidator } from '../../validation/transactionLogValidator.ts';
 import { handleHDBError, ClientError } from '../../utility/errors/hdbError.ts';
@@ -122,6 +123,16 @@ export async function addJob(jsonBody: any) {
 				throw handleHDBError(new Error(), 'Invalid service', HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
 			}
 			break;
+		// backup job validators throw ClientError (with statusCode) directly on failure
+		case hdbTerms.OPERATIONS_ENUM.CREATE_BACKUP:
+			await rocksdbBackup.validateCreateBackup(jsonBody);
+			break;
+		case hdbTerms.OPERATIONS_ENUM.VERIFY_BACKUP:
+			await rocksdbBackup.validateVerifyBackup(jsonBody);
+			break;
+		case hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP:
+			await rocksdbBackup.validateRestoreBackup(jsonBody);
+			break;
 		default:
 			break;
 	}