@harperfast/harper 5.2.0-beta.4 → 5.2.1

package/resources/databases.ts

@@ -33,7 +33,7 @@ import { handleLocalTimeForGets } from './RecordEncoder.ts';
 import { deleteRootBlobPathsForDB } from './blob.ts';
 import { CUSTOM_INDEXES } from './indexes/customIndexes.ts';
 import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.ts';
-import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
+import { RocksDatabase, supportedCompression, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
 import { PrimaryRocksDatabase } from './PrimaryRocksDatabase.ts';
 import { replayLogs } from './replayLogs.ts';
 import { totalmem } from 'node:os';
@@ -41,6 +41,15 @@ import { RocksIndexStore } from './RocksIndexStore.ts';
 import { when } from '../utility/when.ts';
 import { resolveRocksMemoryConfig } from '../utility/rocksMemoryConfig.ts';
 import { isProcessRunning } from '../utility/processManagement/processManagement.js';
+import {
+	acquireRestoreLock,
+	checkRestoreState,
+	releaseRestoreLock,
+	restoreMarkerPresent,
+	scanBlockedRestores,
+	RESTORE_META_DIR,
+	type RestoreLock,
+} from '../dataLayer/restoreMarker.ts';
 
 /**
  * Check if Harper is running in read-only mode.
@@ -89,6 +98,80 @@ const DEFAULT_DATABASE_NAME = 'data';
 const DEFINED_TABLES = Symbol('defined-tables');
 const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
 initSync();
+/**
+ * The RocksDB block/blob codec for every column family this process opens (`storage.rocks.compression`),
+ * or `undefined` to leave rocksdb-js on its own default (lz4 wherever the native build has it).
+ *
+ * Resolved on the first open and then frozen, deliberately. RocksDB fixes a column family's codec
+ * for as long as it is open and rejects a reopen that disagrees, and Harper's worker threads share
+ * one process-wide column-family registry — so every open, in every thread, has to resolve the same
+ * value. Re-reading config per open does not guarantee that: on a fresh install the system families
+ * are created by `mountHdb()` before the config file exists, so the main thread would resolve
+ * nothing and the workers would resolve the configured codec, after which `__dbis__` cannot be
+ * reopened and Harper fails with "The system database failed to load". The installer stages this
+ * value before `mountHdb()` (see utility/install/installer.ts) so that first open already sees it.
+ *
+ * Unset is NOT "use the build default" for a family that already exists — see toRocksCompression.
+ */
+let resolvedRocksCompression: string | undefined;
+let rocksCompressionResolved = false;
+
+export function getRocksCompression(): string | undefined {
+	if (!rocksCompressionResolved) {
+		resolvedRocksCompression = readDatabaseCodec();
+		rocksCompressionResolved = true;
+	}
+	return resolvedRocksCompression;
+}
+
+/**
+ * Test-only: un-freezes the resolved codec. Production code never calls this — the freeze is the
+ * invariant (see getRocksCompression above) — but a test process runs many unrelated test files in
+ * one process, so whichever file happens to open a RocksDatabase first freezes this for everyone
+ * after it. Tests that need to exercise config changes call this to get back to the unresolved state.
+ */
+export function resetRocksCompression(): void {
+	resolvedRocksCompression = undefined;
+	rocksCompressionResolved = false;
+}
+
+/**
+ * The codec every column family in this process opens under.
+ *
+ * Compression is a deployment setting, not a per-table one. RocksDB opens all of a database's
+ * column families in one call, so the codec has to be decided before the first open — which is
+ * before Harper has read any table's metadata (that catalog is itself one of the families being
+ * opened). Resolving one codec from configuration and applying it to every family is what makes
+ * that possible; it is passed with `compressionForAllColumnFamilies` so families this process
+ * never names individually adopt it too, which is what lets a database created before the codec
+ * existed start compressing.
+ *
+ * `storage.rocks.compression` names a codec outright. Otherwise `storage.compression` (default
+ * true) decides enabled-or-not and the build default fills in the algorithm. Per-table metadata
+ * still records the LMDB-era boolean, but no longer selects: a table persisted as disabled inside
+ * a deployment that enables compression would need its own codec, and it cannot have one.
+ */
+function readDatabaseCodec(): string | undefined {
+	const explicit = readRocksCompressionConfig();
+	if (explicit) return explicit;
+	return toRocksCompression(getDefaultCompression()) as string | undefined;
+}
+
+function readRocksCompressionConfig(): string | undefined {
+	const configured = envGet(CONFIG_PARAMS.STORAGE_ROCKS_COMPRESSION);
+	if (configured === undefined || configured === null || configured === '') return undefined;
+	const requested = String(configured).trim().toLowerCase();
+	if (!requested) return undefined;
+	// Rejected here rather than at the open: an unsupported name throws inside RocksDatabase.open,
+	// which surfaces as the system database failing to load partway through startup.
+	if (!supportedCompression.includes(requested)) {
+		throw new Error(
+			`storage.rocks.compression="${requested}" is not available in this build of @harperfast/rocksdb-js. Supported: ${supportedCompression.join(', ')}`
+		);
+	}
+	return requested;
+}
+
 // I don't know if this is the best place for this, but somewhere we need to specify which tables
 // replicate by default:
 export const NON_REPLICATING_SYSTEM_TABLES = [
@@ -167,8 +250,52 @@ export const databaseEventsEmitter = new EventEmitter<DatabaseWatcherEventMap>()
 export const tables: Tables = Object.create(null);
 export const databases: Databases = Object.create(null);
 
+/**
+ * Codec used to honor an "enabled, unspecified" compression setting, or `undefined` where the
+ * native build cannot provide it (in which case the request degrades to the build default rather
+ * than throwing).
+ */
+const DEFAULT_ENABLED_CODEC = supportedCompression.includes('lz4') ? 'lz4' : undefined;
+
+/**
+ * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata carries
+ * values where a defined falsy value (false, '') means compression was explicitly disabled, and
+ * `true` / `{ threshold, ... }` mean enabled with defaults — `storage.compression` defaults to
+ * `true` (defaultConfig.yaml), so essentially every pre-existing table asked for compression.
+ *
+ * "Enabled" resolves to an explicit codec rather than to unset. Unset is not equivalent: RocksDB
+ * persists the codec per column family and a reopen that requests nothing inherits what the family
+ * already has, applying the build default only when the family does not yet exist. Leaving these
+ * unset therefore silently ignores the operator's request on every database created before the
+ * native build carried codecs — it keeps writing uncompressed forever, while a brand-new database
+ * gets lz4. Naming the codec makes the setting mean the same thing in both cases.
+ *
+ * This governs newly written files; existing SSTs keep their codec until write traffic rewrites
+ * them (`db.compact()` will not — see getRocksCompression above).
+ */
+export function toRocksCompression(compression: unknown): unknown {
+	if (compression === undefined) return undefined;
+	if (!compression) return 'none';
+	// An object carrying an explicit `algorithm` is already a rocksdb-js request; anything else
+	// (`true`, or an LMDB descriptor like { startingOffset, threshold }) is "enabled, unspecified".
+	if (compression === true || (typeof compression === 'object' && !(compression as { algorithm?: unknown }).algorithm))
+		return DEFAULT_ENABLED_CODEC;
+	return compression;
+}
+
 function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) {
 	options.disableWAL ??= true;
+	const legacyOptions = options as { compression?: unknown };
+	// A configured codec applies to every column family, overriding whatever per-table metadata
+	// carries — that metadata records the LMDB-era boolean, so without this there is no way to
+	// select a RocksDB codec for a deployment.
+	// One codec for every column family, and applied to every family this open touches — not just
+	// the one being named. RocksDB opens them all at once and a family's codec cannot change while
+	// it is open, so a family this process never names individually would otherwise stay on
+	// whatever it was created with, forever.
+	const databaseCodec = getRocksCompression();
+	legacyOptions.compression = databaseCodec;
+	if (databaseCodec) (options as { compressionForAllColumnFamilies?: boolean }).compressionForAllColumnFamilies = true;
 	// Apply read-only mode if enabled
 	if (isReadOnlyMode()) {
 		options.readOnly = true;
@@ -315,11 +442,18 @@ export function getDatabases(): Databases {
 	if (databasePath && existsSync(databasePath)) {
 		// First load all the databases from our main database folder
 		// TODO: Load any databases defined with explicit storage paths from the config
-		for (const databaseEntry of readdirSync(databasePath, { withFileTypes: true })) {
+		const entries = readdirSync(databasePath, { withFileTypes: true });
+		const blockedByRestore = databasesBlockedByRestore(databasePath);
+		for (const databaseEntry of entries) {
 			// in-progress migration staging dirs are not databases until atomically renamed into place
 			if (databaseEntry.name.endsWith(MIGRATING_DIR_SUFFIX)) continue;
+			// the restore-metadata directory is reserved: never load it as a database, even if a
+			// (out-of-band) RocksDB directory happens to occupy that reserved name — the API can't
+			// create it (schemaRegex forbids the backtick), but the scan opens any CURRENT+MANIFEST dir
+			if (databaseEntry.name === RESTORE_META_DIR) continue;
 			const dbName = basename(databaseEntry.name, '.mdb');
 			const dbPath = join(databasePath, databaseEntry.name);
+			if (blockedByRestore.has(dbName)) continue;
 
 			if (
 				databaseEntry.isFile() &&
@@ -376,8 +510,12 @@ export function getDatabases(): Databases {
 			const schemaConfig = schemaConfigs[dbName];
 			const databasePath = schemaConfig.path;
 			if (existsSync(databasePath)) {
-				for (const databaseEntry of readdirSync(databasePath, { withFileTypes: true })) {
-					if (databaseEntry.name.endsWith(MIGRATING_DIR_SUFFIX)) continue;
+				const entries = readdirSync(databasePath, { withFileTypes: true });
+				const blockedByRestore = databasesBlockedByRestore(databasePath);
+				for (const databaseEntry of entries) {
+					if (databaseEntry.name.endsWith(MIGRATING_DIR_SUFFIX)) continue; // migration staging dir
+					if (databaseEntry.name === RESTORE_META_DIR) continue; // reserved restore-metadata dir
+					if (blockedByRestore.has(basename(databaseEntry.name, '.mdb'))) continue;
 					if (databaseEntry.isFile() && extname(databaseEntry.name).toLowerCase() === '.mdb') {
 						readMetaDb(join(databasePath, databaseEntry.name), basename(databaseEntry.name, '.mdb'), dbName);
 					} else {
@@ -452,6 +590,29 @@ export function getDatabases(): Databases {
 	return databases;
 }
 
+/**
+ * Scan a databases directory's entries for restore lock/marker files and return the names of
+ * databases that must not be loaded: a held restore lock means a restore is in progress in some
+ * process; an unheld lock with a surviving `.restoring` marker means a restore was interrupted
+ * mid-purge (the directory may be partial garbage) and must be rerun. The files live *next to*
+ * the database directory, so this also covers a database whose directory is missing or empty.
+ */
+function databasesBlockedByRestore(databasePath: string): Set<string> {
+	const blocked = new Set<string>();
+	for (const [dbName, state] of scanBlockedRestores(databasePath)) {
+		if (state === 'in-progress') {
+			logger.warn(`A restore of database '${dbName}' is in progress; not loading it`);
+			blocked.add(dbName);
+		} else if (state === 'incomplete') {
+			logger.error(
+				`Incomplete restore of database '${dbName}' detected (a restore started but did not finish); not loading it — rerun the restore to recover`
+			);
+			blocked.add(dbName);
+		}
+	}
+	return blocked;
+}
+
 /**
  * This is responsible for reading the internal dbi of a single database file to get a list of all the tables and
  * their indexed or registered attributes
@@ -530,7 +691,7 @@ function initStores(
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
-			} as any) as RocksDatabaseEx;
+			} as any);
 		} else {
 			attributesDbi = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit as any);
 		}
@@ -916,18 +1077,12 @@ function setTable(tables, tableName, Table) {
 	return Table;
 }
 /**
- * Get root store for a database
- * @param options
- * @returns
+ * Resolve the directory that holds (or would hold) a database's storage, from the databases
+ * config, storage path config/env, or the hdb root — without opening anything. This is the
+ * parent directory selection used by `database()`; a RocksDB database lives at
+ * `join(resolveDatabaseStorageRoot(...), databaseName)`.
  */
-export function database({ database: databaseName, table: tableName }) {
-	if (!databaseName) databaseName = DEFAULT_DATABASE_NAME;
-	getDatabases();
-	ensureDB(databaseName);
-	const definedDatabase = definedDatabases.get(databaseName);
-	if ((definedDatabase as any)?.rootStore) {
-		return (definedDatabase as any).rootStore;
-	}
+export function resolveDatabaseStorageRoot(databaseName: string, tableName?: string): string {
 	const databaseConfig = envGet(CONFIG_PARAMS.DATABASES) || {};
 	if (process.env.SCHEMAS_DATA_PATH) {
 		databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
@@ -952,6 +1107,35 @@ export function database({ database: databaseName, table: tableName }) {
 			`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`
 		);
 	}
+	return databasePath;
+}
+
+/**
+ * Resolve the directory path of a RocksDB database (whether or not it exists or is loaded).
+ */
+export function resolveDatabasePath(databaseName: string): string {
+	return join(resolveDatabaseStorageRoot(databaseName), databaseName);
+}
+
+/**
+ * Get root store for a database
+ * @param options
+ * @returns
+ */
+export function database({ database: databaseName, table: tableName }) {
+	if (!databaseName) databaseName = DEFAULT_DATABASE_NAME;
+	getDatabases();
+	ensureDB(databaseName);
+	const definedDatabase = definedDatabases.get(databaseName);
+	if ((definedDatabase as any)?.rootStore) {
+		return (definedDatabase as any).rootStore;
+	}
+	const databaseConfig = envGet(CONFIG_PARAMS.DATABASES) || {};
+	if (process.env.SCHEMAS_DATA_PATH) {
+		databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
+	}
+	const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
+	const databasePath = resolveDatabaseStorageRoot(databaseName, tableName);
 
 	let rootStore: RootDatabaseKind;
 	const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || envGet(CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
@@ -959,6 +1143,10 @@ export function database({ database: databaseName, table: tableName }) {
 		const path = join(databasePath, tablePath ? tableName : databaseName);
 		rootStore = rocksdbDatabaseEnvs.get(path);
 		if (!rootStore || rootStore.status === 'closed') {
+			// this on-demand open (create_table/create_database and friends) must not resurrect a
+			// database that a restore is rewriting (or left half-purged) — the scan-time restore
+			// checks don't cover this path
+			throwIfBlockedByRestore(path, databaseName);
 			rootStore = openRocksDatabase(path, {
 				disableWAL: false,
 				enableStats: true,
@@ -981,6 +1169,51 @@ export function database({ database: databaseName, table: tableName }) {
 	if (definedDatabase) (definedDatabase as any).rootStore = rootStore;
 	return rootStore;
 }
+function throwIfBlockedByRestore(dbPath: string, databaseName: string): void {
+	const restoreState = checkRestoreState(dbPath);
+	if (restoreState !== 'clear') {
+		const error: any = new Error(
+			restoreState === 'in-progress'
+				? `Database '${databaseName}' is being restored; retry when the restore completes`
+				: `Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`
+		);
+		error.statusCode = 409;
+		throw error;
+	}
+}
+
+/**
+ * Take the per-database restore lock for a drop, refusing (409) if a restore holds it (in-progress)
+ * or a crashed restore left a marker (incomplete). Pushes the acquired lock onto `held` so the
+ * caller releases it after the drop. On refusal, releases anything already held and throws.
+ *
+ * The lock is not reentrant within a process, so a path already in `held` must be skipped — every
+ * table in a RocksDB database shares one root store (and one lock path), and re-acquiring it in the
+ * same drop would spuriously 409 on the second table.
+ */
+function lockDatabaseForDrop(dbPath: string, databaseName: string, held: RestoreLock[]): void {
+	if (held.some((h) => h.dbPath === dbPath)) return;
+	let lock: RestoreLock;
+	try {
+		lock = acquireRestoreLock(dbPath);
+	} catch (error) {
+		for (const h of held) releaseRestoreLock(h);
+		throw error; // 409: a restore is in progress and holds the lock
+	}
+	// We now hold the lock, so no restore is active. A surviving marker is therefore debris from a
+	// crashed restore (incomplete) — refuse rather than delete a directory that still needs recovery.
+	if (restoreMarkerPresent(dbPath)) {
+		releaseRestoreLock(lock);
+		for (const h of held) releaseRestoreLock(h);
+		const error: any = new Error(
+			`Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`
+		);
+		error.statusCode = 409;
+		throw error;
+	}
+	held.push(lock);
+}
+
 /**
  * Delete the database
  * @param databaseName
@@ -990,17 +1223,105 @@ export async function dropDatabase(databaseName) {
 	const dbTables = databases[databaseName];
 	let rootStore;
 
-	for (const tableName in dbTables) {
-		const table = dbTables[tableName];
-		rootStore = table.primaryStore.rootStore;
-		lmdbDatabaseEnvs.delete(rootStore.path);
-		rocksdbDatabaseEnvs.delete(rootStore.path);
+	// Hold the per-database restore lock across the entire drop so its file deletion can never
+	// interleave with a restore's purge-and-copy on the same directory — a destroy landing after a
+	// restore's copy would gut a "successful" restore, and vice versa. Restore takes the same lock
+	// (before writing its marker), so both operations serialize on this one primitive rather than on
+	// a check-then-act marker probe. Released in the finally below.
+	const restoreLocks: RestoreLock[] = [];
+	try {
+		for (const tableName in dbTables) {
+			const table = dbTables[tableName];
+			rootStore = table.primaryStore.rootStore;
+			if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+			lmdbDatabaseEnvs.delete(rootStore.path);
+			rocksdbDatabaseEnvs.delete(rootStore.path);
+		}
+
+		for (const tableName in dbTables) {
+			databaseEventsEmitter.emit('dropTable', tableName, databaseName);
+		}
+
+		if (databaseName === 'data') {
+			for (const tableName in tables) {
+				delete tables[tableName];
+			}
+			delete tables[DEFINED_TABLES];
+		}
+		delete databases[databaseName];
+
+		databaseEventsEmitter.emit('dropDatabase', databaseName);
+
+		if (rootStore) {
+			if (rootStore.status === 'open') {
+				if (rootStore instanceof RocksDatabase) {
+					rootStore.close();
+					rootStore.destroy();
+				} else {
+					await rootStore.close();
+					await unlink(rootStore.path);
+				}
+			}
+		} else {
+			rootStore = database({ database: databaseName, table: null });
+			// a tableless database resolves its root store here rather than in the loop above, so take
+			// the drop lock now (still before any destructive step)
+			if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+			if (rootStore instanceof RocksDatabase) {
+				rootStore.close();
+				rootStore.destroy();
+			} else if (rootStore.status === 'open') {
+				await rootStore.close();
+				await unlink(rootStore.path);
+			}
+		}
+
+		await deleteRootBlobPathsForDB(rootStore);
+	} finally {
+		for (const lock of restoreLocks) releaseRestoreLock(lock);
 	}
+}
 
+/**
+ * Close a RocksDB database's store handles on the current thread and unregister it, without
+ * touching its files. Used by the restore_backup flow: every thread must release its handles so
+ * `backups.restore()` can purge and rewrite the (fully closed) database directory. A subsequent
+ * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in
+ * progress, per the restore marker checks in the scan).
+ */
+export function closeDatabase(databaseName: string): boolean {
+	const dbTables = databases[databaseName];
+	if (!dbTables) return false;
+	const rootStores = new Set<any>();
+	const closeStore = (store: any, description: string) => {
+		try {
+			store?.close?.();
+		} catch (error) {
+			logger.warn(`Error closing ${description} while closing database ${databaseName}:`, error);
+		}
+	};
 	for (const tableName in dbTables) {
-		databaseEventsEmitter.emit('dropTable', tableName, databaseName);
+		const table: any = dbTables[tableName];
+		if (!table?.primaryStore) continue;
+		if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore);
+		for (const indexName in table.indices || {}) {
+			closeStore(table.indices[indexName], `index ${tableName}.${indexName}`);
+		}
+		closeStore(table.primaryStore, `table ${tableName}`);
 	}
-
+	// a database with no tables (an empty schema, or one whose tables were all dropped) still holds
+	// an open root store, tracked only on the defined-database entry rather than any table — include
+	// it so its handles are released too (the Set dedupes it against the per-table root stores above)
+	const definedRoot = (definedDatabases?.get(databaseName) as any)?.rootStore;
+	if (definedRoot) rootStores.add(definedRoot);
+	for (const rootStore of rootStores) {
+		closeStore(rootStore.dbisDb, 'attributes store');
+		closeStore(rootStore, 'root store');
+		lmdbDatabaseEnvs.delete(rootStore.path);
+		rocksdbDatabaseEnvs.delete(rootStore.path);
+	}
+	const definedDatabase = definedDatabases?.get(databaseName);
+	if (definedDatabase) (definedDatabase as any).rootStore = undefined;
 	if (databaseName === 'data') {
 		for (const tableName in tables) {
 			delete tables[tableName];
@@ -1008,31 +1329,41 @@ export async function dropDatabase(databaseName) {
 		delete tables[DEFINED_TABLES];
 	}
 	delete databases[databaseName];
+	return true;
+}
 
-	databaseEventsEmitter.emit('dropDatabase', databaseName);
-
-	if (rootStore) {
-		if (rootStore.status === 'open') {
-			if (rootStore instanceof RocksDatabase) {
-				rootStore.close();
-				rootStore.destroy();
-			} else {
-				await rootStore.close();
-				await unlink(rootStore.path);
+/**
+ * Close every RocksDB (user) database this thread has open, releasing its native handles.
+ *
+ * rocksdb-js's registry is process-global across worker threads, and a thread that exits WITHOUT
+ * closing leaks its handles (the process-global refCount never drops), while the only alternative,
+ * `shutdown()`, tears down rocksdb for the entire process. So a worker thread that opens databases
+ * and then exits — notably a job worker (jobProcess), which opens the whole database graph via
+ * `getDatabases()` and exits when the job finishes — must close its handles explicitly, or those
+ * handles linger process-wide (and, e.g., block an online `restore_backup` from confirming the
+ * database is closed). The `system` database is intentionally left open: it is non-enumerable here
+ * (skipped by the loop), is never restored online, and the exiting worker may still touch the job
+ * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`.
+ */
+export function closeLoadedDatabases(): void {
+	// snapshot the names first: closeDatabase() deletes from `databases` as it goes
+	for (const databaseName of Object.keys(databases)) {
+		const dbTables = databases[databaseName];
+		if (!dbTables) continue;
+		let isRocks = false;
+		for (const tableName in dbTables) {
+			if (dbTables[tableName]?.primaryStore?.rootStore instanceof RocksDatabase) {
+				isRocks = true;
+				break;
 			}
 		}
-	} else {
-		rootStore = database({ database: databaseName, table: null });
-		if (rootStore instanceof RocksDatabase) {
-			rootStore.close();
-			rootStore.destroy();
-		} else if (rootStore.status === 'open') {
-			await rootStore.close();
-			await unlink(rootStore.path);
+		// a tableless database exposes no table root store, so also check the defined-database
+		// entry — otherwise its open root store would leak on worker exit
+		if (!isRocks && (definedDatabases?.get(databaseName) as any)?.rootStore instanceof RocksDatabase) {
+			isRocks = true;
 		}
+		if (isRocks) closeDatabase(databaseName);
 	}
-
-	await deleteRootBlobPathsForDB(rootStore);
 }
 // HNSW_NO_AUTOVERSION kill-switch: when set, a NEW index initializes as legacy rather than
 // versioned. process.env values are strings, so a bare truthiness check would treat "0"/"false"
@@ -1122,7 +1453,11 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any)
 		// Enable cache (WeakLRUCache + VT) for all custom-object index stores so the VT is
 		// available before resolveIndexFormat decides the format. Versioned stores need the VT
 		// for cached traversal; legacy stores pay a small per-write cache.delete() overhead only.
-		dbi = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiKey, cache: isCustomObjectIndex } as any) as any;
+		dbi = openRocksDatabase(rootStore.path, {
+			...dbiInit,
+			name: dbiKey,
+			cache: isCustomObjectIndex,
+		} as any) as any;
 		(dbi as any).rootStore = rootStore;
 		// Custom-index object stores (e.g. HNSW) write graph nodes via plain put() with no staged
 		// transaction timestamp, so their values carry no version and the PrimaryRocksDatabase
@@ -1345,6 +1680,10 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 				clearInterruptedDropEntries(rootStore.path, tableName);
 			}
 			if (rootStore instanceof RocksDatabase) {
+				// Usually a genuinely new column family (existingTableMeta above found no catalog
+				// entry), but an interrupted drop just completed above can leave the physical CF
+				// behind under its old codec even though the catalog entry is gone — same fallback
+				// as the reconcile paths covers that remnant case too.
 				primaryStore = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiName, cache: true } as any);
 			} else {
 				primaryStore = (rootStore as any).openDB(dbiName, dbiInit as any);
@@ -1955,7 +2294,7 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string,
 	if (rootStore instanceof RocksDatabase) {
 		for (const columnName of (rootStore as any).columns) {
 			if (columnName.startsWith(tableName + '/')) {
-				const columnStore = openRocksDatabase(rootStore.path, { name: columnName } as any);
+				const columnStore = openRocksDatabase(rootStore.path, { name: columnName });
 				try {
 					columnStore.dropSync();
 				} catch (error) {
@@ -2052,7 +2391,9 @@ export function getDefaultCompression() {
 	if (STORAGE_COMPRESSION_DICTIONARY)
 		LMDB_COMPRESSION_OPTS['dictionary'] = readFileSync(STORAGE_COMPRESSION_DICTIONARY);
 	if (STORAGE_COMPRESSION_THRESHOLD) LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
-	return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
+	// normalize disabled to false so a falsy config value ('' or null) is never persisted
+	// into table metadata as-is (openRocksDatabase maps defined-falsy to 'none')
+	return LMDB_COMPRESSION ? LMDB_COMPRESSION_OPTS : false;
 }
 
 /**

package/resources/jsResource.ts

@@ -69,41 +69,13 @@ export class ResourceLoadError extends Error {
  *
  * Once a file has been loaded it cannot be unloaded without a restart.
  *
- * Thus, this plugin only handle files as they are added (`add` event). All other events result in a restart request.
+ * Thus, this plugin only handles files as they are added (`add` event). All other events result in a restart request.
  *
- * A redeploy tears down and reinstalls the component's files while this scope's watcher is paused
- * (see `Scope`/`EntryHandler` deploy lifecycle); on resume the fresh chokidar scan re-emits every
- * existing file as `'add'` — including ones whose contents just changed. Treating those as plain
- * adds would silently re-run against the stale module cache and never flag a restart (harper#1817).
- * So we track which files this scope has already loaded: a re-`add` of a known file is a redeploy of
- * loaded code we cannot hot-swap, and is handled like a `change` — request a restart. A first-time
- * `add` (initial load, or a genuinely new file added at runtime) still loads without a restart.
- *
- * A redeploy that *deletes* a loaded file is a different shape of the same problem: the fresh
- * chokidar scan only reports what's currently on disk, so a file that's gone produces no event at
- * all — no re-`add`, no `unlink` — and the modified-file handling above never sees it. Left
- * unhandled, the deleted resource stays registered and active in memory (harper#1817 follow-up). So
- * we also track which files the post-redeploy scan pass reports, and once that scan's `ready` fires,
- * diff it against everything this scope has ever loaded: anything missing was deleted, and is
- * handled the same way as a modified file — request a restart.
- *
- * That diff must only run for an actual redeploy rescan, not every time `EntryHandler` emits
- * `ready` — it also refires after each ordinary runtime add/change once that file's read settles
- * (its initial-scan-complete latch never resets outside a full rescan), and diffing against that
- * would falsely treat every other already-loaded file as deleted. So the diff window is gated by
- * the scope's own `deploy:start`/`deploy:end` bracket (see `Scope`): `deploy:start` pauses the
- * watcher and opens the window (and is where we reset the scan-file tracking, since no file events
- * can land while paused), and the first `ready` afterward — the resumed watcher's fresh scan
- * completing — closes it and runs the diff.
+ * EntryHandler preserves file identity across a deploy pause/resume and emits `change` or `unlink`
+ * for loaded files that changed or disappeared, so those events request the required restart.
  */
 export async function handleApplication(scope: Scope) {
-	const loadedResourceFiles = new Set<string>();
-	// Files reported as `add` since the most recent `deploy:start`, populated only while
-	// `awaitingPostRedeployScan` is true — see the gating note above.
-	let currentScanFiles = new Set<string>();
-	let awaitingPostRedeployScan = false;
-
-	const entryHandler = scope.handleEntry(async function handleResourceEntry(entryEvent) {
+	scope.handleEntry(async function handleResourceEntry(entryEvent) {
 		if (entryEvent.entryType !== 'file') {
 			scope.logger.warn(
 				`jsResource plugin cannot handle entry type ${entryEvent.entryType}. Modify the 'files' option in ${scope.configFilePath} to only include files.`
@@ -111,13 +83,7 @@ export async function handleApplication(scope: Scope) {
 			return;
 		}
 
-		if (awaitingPostRedeployScan && entryEvent.eventType === 'add') {
-			// Recorded unconditionally — before the loaded/re-add branch below — so the post-scan
-			// deletion diff sees every file this scan reported, whether newly loaded or already known.
-			currentScanFiles.add(entryEvent.absolutePath);
-		}
-
-		if (entryEvent.eventType !== 'add' || loadedResourceFiles.has(entryEvent.absolutePath)) {
+		if (entryEvent.eventType !== 'add') {
 			scope.requestRestart();
 			return;
 		}
@@ -133,9 +99,6 @@ export async function handleApplication(scope: Scope) {
 				scope.logger.debug?.(`Registered root resource: ${path}`);
 			}
 			recurseForResources(scope, resourceModule, root);
-			// Record the load so a later re-`add` of this same file (a redeploy re-scan) is treated
-			// as a change and requests a restart rather than silently re-serving stale cached code.
-			loadedResourceFiles.add(entryEvent.absolutePath);
 			// A JS resource that extends an exported @table is the one carrying author opt-ins
 			// (`static mcpTools`/`mcpPrompts`), and it registers here — after the schema-derived
 			// table class and after the MCP component's boot scan. Signal so listing surfaces
@@ -146,26 +109,6 @@ export async function handleApplication(scope: Scope) {
 			throw new ResourceLoadError(entryEvent.absolutePath, error);
 		}
 	});
-
-	// Optional chaining: a mock/test scope may not implement EventEmitter, and Scope#handleEntry
-	// itself can return undefined (e.g. MissingDefaultFilesOptionError). In real use `scope` is
-	// always an EventEmitter and `entryHandler` is always the EntryHandler backing this watcher.
-	scope.on?.('deploy:start', () => {
-		awaitingPostRedeployScan = true;
-		currentScanFiles = new Set();
-	});
-
-	entryHandler?.on?.('ready', () => {
-		if (!awaitingPostRedeployScan) return;
-		awaitingPostRedeployScan = false;
-		for (const loadedFile of loadedResourceFiles) {
-			if (!currentScanFiles.has(loadedFile)) {
-				// Known file that the just-completed scan never reported — deleted during the redeploy.
-				loadedResourceFiles.delete(loadedFile);
-				scope.requestRestart();
-			}
-		}
-	});
 }
 
 function recurseForResources(scope: Scope, resourceModule: any, prefix: string) {