@harperfast/harper 5.2.0-beta.4 → 5.2.1

package/dist/resources/databases.d.ts

@@ -4,6 +4,14 @@ import { makeTable } from './Table.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 export declare function isReadOnlyMode(): boolean;
 declare const DEFINED_TABLES: unique symbol;
+export declare function getRocksCompression(): string | undefined;
+/**
+ * Test-only: un-freezes the resolved codec. Production code never calls this — the freeze is the
+ * invariant (see getRocksCompression above) — but a test process runs many unrelated test files in
+ * one process, so whichever file happens to open a RocksDatabase first freezes this for everyone
+ * after it. Tests that need to exercise config changes call this to get back to the unresolved state.
+ */
+export declare function resetRocksCompression(): void;
 export declare const NON_REPLICATING_SYSTEM_TABLES: string[];
 export type Table = ReturnType<typeof makeTable> & {
     indexingOperation?: any;
@@ -63,6 +71,23 @@ export type DatabaseWatcherEventMap = {
 export declare const databaseEventsEmitter: EventEmitter<DatabaseWatcherEventMap>;
 export declare const tables: Tables;
 export declare const databases: Databases;
+/**
+ * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata carries
+ * values where a defined falsy value (false, '') means compression was explicitly disabled, and
+ * `true` / `{ threshold, ... }` mean enabled with defaults — `storage.compression` defaults to
+ * `true` (defaultConfig.yaml), so essentially every pre-existing table asked for compression.
+ *
+ * "Enabled" resolves to an explicit codec rather than to unset. Unset is not equivalent: RocksDB
+ * persists the codec per column family and a reopen that requests nothing inherits what the family
+ * already has, applying the build default only when the family does not yet exist. Leaving these
+ * unset therefore silently ignores the operator's request on every database created before the
+ * native build carried codecs — it keeps writing uncompressed forever, while a brand-new database
+ * gets lz4. Naming the codec makes the setting mean the same thing in both cases.
+ *
+ * This governs newly written files; existing SSTs keep their codec until write traffic rewrites
+ * them (`db.compact()` will not — see getRocksCompression above).
+ */
+export declare function toRocksCompression(compression: unknown): unknown;
 /**
  * This gets the set of tables from the default database ("data").
  */
@@ -107,6 +132,17 @@ interface TableDefinition {
     hidden?: boolean;
     cacheControl?: string | null;
 }
+/**
+ * Resolve the directory that holds (or would hold) a database's storage, from the databases
+ * config, storage path config/env, or the hdb root — without opening anything. This is the
+ * parent directory selection used by `database()`; a RocksDB database lives at
+ * `join(resolveDatabaseStorageRoot(...), databaseName)`.
+ */
+export declare function resolveDatabaseStorageRoot(databaseName: string, tableName?: string): string;
+/**
+ * Resolve the directory path of a RocksDB database (whether or not it exists or is loaded).
+ */
+export declare function resolveDatabasePath(databaseName: string): string;
 /**
  * Get root store for a database
  * @param options
@@ -121,6 +157,28 @@ export declare function database({ database: databaseName, table: tableName }: {
  * @param databaseName
  */
 export declare function dropDatabase(databaseName: any): Promise<void>;
+/**
+ * Close a RocksDB database's store handles on the current thread and unregister it, without
+ * touching its files. Used by the restore_backup flow: every thread must release its handles so
+ * `backups.restore()` can purge and rewrite the (fully closed) database directory. A subsequent
+ * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in
+ * progress, per the restore marker checks in the scan).
+ */
+export declare function closeDatabase(databaseName: string): boolean;
+/**
+ * Close every RocksDB (user) database this thread has open, releasing its native handles.
+ *
+ * rocksdb-js's registry is process-global across worker threads, and a thread that exits WITHOUT
+ * closing leaks its handles (the process-global refCount never drops), while the only alternative,
+ * `shutdown()`, tears down rocksdb for the entire process. So a worker thread that opens databases
+ * and then exits — notably a job worker (jobProcess), which opens the whole database graph via
+ * `getDatabases()` and exits when the job finishes — must close its handles explicitly, or those
+ * handles linger process-wide (and, e.g., block an online `restore_backup` from confirming the
+ * database is closed). The `system` database is intentionally left open: it is non-enumerable here
+ * (skipped by the loop), is never restored online, and the exiting worker may still touch the job
+ * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`.
+ */
+export declare function closeLoadedDatabases(): void;
 /**
  * This can be called to ensure that the specified table exists and if it does not exist, it should be created.
  * @param tableName
@@ -162,7 +220,7 @@ export declare function onRemovedTable(listener: (tableName: string, databaseNam
 export declare function onRemovedDB(listener: (databaseName: string) => void): {
     remove(): void;
 };
-export declare function getDefaultCompression(): {
+export declare function getDefaultCompression(): false | {
     startingOffset: number;
 };
 /**

package/dist/resources/databases.js

@@ -38,12 +38,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
 exports.isReadOnlyMode = isReadOnlyMode;
+exports.getRocksCompression = getRocksCompression;
+exports.resetRocksCompression = resetRocksCompression;
+exports.toRocksCompression = toRocksCompression;
 exports.getTables = getTables;
 exports.getDatabases = getDatabases;
 exports.readMetaDb = readMetaDb;
 exports.resetDatabases = resetDatabases;
+exports.resolveDatabaseStorageRoot = resolveDatabaseStorageRoot;
+exports.resolveDatabasePath = resolveDatabasePath;
 exports.database = database;
 exports.dropDatabase = dropDatabase;
+exports.closeDatabase = closeDatabase;
+exports.closeLoadedDatabases = closeLoadedDatabases;
 exports.table = table;
 exports.canonicalizeIndexOptions = canonicalizeIndexOptions;
 exports.dropTableMeta = dropTableMeta;
@@ -86,6 +93,7 @@ const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
 const when_ts_1 = require("../utility/when.js");
 const rocksMemoryConfig_ts_1 = require("../utility/rocksMemoryConfig.js");
 const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
+const restoreMarker_ts_1 = require("../dataLayer/restoreMarker.js");
 /**
  * Check if Harper is running in read-only mode.
  * Read-only mode can be enabled via:
@@ -133,6 +141,76 @@ const DEFAULT_DATABASE_NAME = 'data';
 const DEFINED_TABLES = Symbol('defined-tables');
 const DEFAULT_COMPRESSION_THRESHOLD = ((0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
 (0, environmentManager_ts_1.initSync)();
+/**
+ * The RocksDB block/blob codec for every column family this process opens (`storage.rocks.compression`),
+ * or `undefined` to leave rocksdb-js on its own default (lz4 wherever the native build has it).
+ *
+ * Resolved on the first open and then frozen, deliberately. RocksDB fixes a column family's codec
+ * for as long as it is open and rejects a reopen that disagrees, and Harper's worker threads share
+ * one process-wide column-family registry — so every open, in every thread, has to resolve the same
+ * value. Re-reading config per open does not guarantee that: on a fresh install the system families
+ * are created by `mountHdb()` before the config file exists, so the main thread would resolve
+ * nothing and the workers would resolve the configured codec, after which `__dbis__` cannot be
+ * reopened and Harper fails with "The system database failed to load". The installer stages this
+ * value before `mountHdb()` (see utility/install/installer.ts) so that first open already sees it.
+ *
+ * Unset is NOT "use the build default" for a family that already exists — see toRocksCompression.
+ */
+let resolvedRocksCompression;
+let rocksCompressionResolved = false;
+function getRocksCompression() {
+    if (!rocksCompressionResolved) {
+        resolvedRocksCompression = readDatabaseCodec();
+        rocksCompressionResolved = true;
+    }
+    return resolvedRocksCompression;
+}
+/**
+ * Test-only: un-freezes the resolved codec. Production code never calls this — the freeze is the
+ * invariant (see getRocksCompression above) — but a test process runs many unrelated test files in
+ * one process, so whichever file happens to open a RocksDatabase first freezes this for everyone
+ * after it. Tests that need to exercise config changes call this to get back to the unresolved state.
+ */
+function resetRocksCompression() {
+    resolvedRocksCompression = undefined;
+    rocksCompressionResolved = false;
+}
+/**
+ * The codec every column family in this process opens under.
+ *
+ * Compression is a deployment setting, not a per-table one. RocksDB opens all of a database's
+ * column families in one call, so the codec has to be decided before the first open — which is
+ * before Harper has read any table's metadata (that catalog is itself one of the families being
+ * opened). Resolving one codec from configuration and applying it to every family is what makes
+ * that possible; it is passed with `compressionForAllColumnFamilies` so families this process
+ * never names individually adopt it too, which is what lets a database created before the codec
+ * existed start compressing.
+ *
+ * `storage.rocks.compression` names a codec outright. Otherwise `storage.compression` (default
+ * true) decides enabled-or-not and the build default fills in the algorithm. Per-table metadata
+ * still records the LMDB-era boolean, but no longer selects: a table persisted as disabled inside
+ * a deployment that enables compression would need its own codec, and it cannot have one.
+ */
+function readDatabaseCodec() {
+    const explicit = readRocksCompressionConfig();
+    if (explicit)
+        return explicit;
+    return toRocksCompression(getDefaultCompression());
+}
+function readRocksCompressionConfig() {
+    const configured = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_COMPRESSION);
+    if (configured === undefined || configured === null || configured === '')
+        return undefined;
+    const requested = String(configured).trim().toLowerCase();
+    if (!requested)
+        return undefined;
+    // Rejected here rather than at the open: an unsupported name throws inside RocksDatabase.open,
+    // which surfaces as the system database failing to load partway through startup.
+    if (!rocksdb_js_1.supportedCompression.includes(requested)) {
+        throw new Error(`storage.rocks.compression="${requested}" is not available in this build of @harperfast/rocksdb-js. Supported: ${rocksdb_js_1.supportedCompression.join(', ')}`);
+    }
+    return requested;
+}
 // I don't know if this is the best place for this, but somewhere we need to specify which tables
 // replicate by default:
 exports.NON_REPLICATING_SYSTEM_TABLES = [
@@ -148,8 +226,53 @@ exports.NON_REPLICATING_SYSTEM_TABLES = [
 exports.databaseEventsEmitter = new node_events_1.EventEmitter();
 exports.tables = Object.create(null);
 exports.databases = Object.create(null);
+/**
+ * Codec used to honor an "enabled, unspecified" compression setting, or `undefined` where the
+ * native build cannot provide it (in which case the request degrades to the build default rather
+ * than throwing).
+ */
+const DEFAULT_ENABLED_CODEC = rocksdb_js_1.supportedCompression.includes('lz4') ? 'lz4' : undefined;
+/**
+ * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata carries
+ * values where a defined falsy value (false, '') means compression was explicitly disabled, and
+ * `true` / `{ threshold, ... }` mean enabled with defaults — `storage.compression` defaults to
+ * `true` (defaultConfig.yaml), so essentially every pre-existing table asked for compression.
+ *
+ * "Enabled" resolves to an explicit codec rather than to unset. Unset is not equivalent: RocksDB
+ * persists the codec per column family and a reopen that requests nothing inherits what the family
+ * already has, applying the build default only when the family does not yet exist. Leaving these
+ * unset therefore silently ignores the operator's request on every database created before the
+ * native build carried codecs — it keeps writing uncompressed forever, while a brand-new database
+ * gets lz4. Naming the codec makes the setting mean the same thing in both cases.
+ *
+ * This governs newly written files; existing SSTs keep their codec until write traffic rewrites
+ * them (`db.compact()` will not — see getRocksCompression above).
+ */
+function toRocksCompression(compression) {
+    if (compression === undefined)
+        return undefined;
+    if (!compression)
+        return 'none';
+    // An object carrying an explicit `algorithm` is already a rocksdb-js request; anything else
+    // (`true`, or an LMDB descriptor like { startingOffset, threshold }) is "enabled, unspecified".
+    if (compression === true || (typeof compression === 'object' && !compression.algorithm))
+        return DEFAULT_ENABLED_CODEC;
+    return compression;
+}
 function openRocksDatabase(path, options) {
     options.disableWAL ??= true;
+    const legacyOptions = options;
+    // A configured codec applies to every column family, overriding whatever per-table metadata
+    // carries — that metadata records the LMDB-era boolean, so without this there is no way to
+    // select a RocksDB codec for a deployment.
+    // One codec for every column family, and applied to every family this open touches — not just
+    // the one being named. RocksDB opens them all at once and a family's codec cannot change while
+    // it is open, so a family this process never names individually would otherwise stay on
+    // whatever it was created with, forever.
+    const databaseCodec = getRocksCompression();
+    legacyOptions.compression = databaseCodec;
+    if (databaseCodec)
+        options.compressionForAllColumnFamilies = true;
     // Apply read-only mode if enabled
     if (isReadOnlyMode()) {
         options.readOnly = true;
@@ -284,12 +407,21 @@ function getDatabases() {
     if (databasePath && (0, node_fs_1.existsSync)(databasePath)) {
         // First load all the databases from our main database folder
         // TODO: Load any databases defined with explicit storage paths from the config
-        for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
+        const entries = (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true });
+        const blockedByRestore = databasesBlockedByRestore(databasePath);
+        for (const databaseEntry of entries) {
             // in-progress migration staging dirs are not databases until atomically renamed into place
             if (databaseEntry.name.endsWith(hdbTerms_ts_1.MIGRATING_DIR_SUFFIX))
                 continue;
+            // the restore-metadata directory is reserved: never load it as a database, even if a
+            // (out-of-band) RocksDB directory happens to occupy that reserved name — the API can't
+            // create it (schemaRegex forbids the backtick), but the scan opens any CURRENT+MANIFEST dir
+            if (databaseEntry.name === restoreMarker_ts_1.RESTORE_META_DIR)
+                continue;
             const dbName = (0, path_1.basename)(databaseEntry.name, '.mdb');
             const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
+            if (blockedByRestore.has(dbName))
+                continue;
             if (databaseEntry.isFile() &&
                 (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb' &&
                 !schemaConfigs[dbName]?.path) {
@@ -334,8 +466,14 @@ function getDatabases() {
             const schemaConfig = schemaConfigs[dbName];
             const databasePath = schemaConfig.path;
             if ((0, node_fs_1.existsSync)(databasePath)) {
-                for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
+                const entries = (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true });
+                const blockedByRestore = databasesBlockedByRestore(databasePath);
+                for (const databaseEntry of entries) {
                     if (databaseEntry.name.endsWith(hdbTerms_ts_1.MIGRATING_DIR_SUFFIX))
+                        continue; // migration staging dir
+                    if (databaseEntry.name === restoreMarker_ts_1.RESTORE_META_DIR)
+                        continue; // reserved restore-metadata dir
+                    if (blockedByRestore.has((0, path_1.basename)(databaseEntry.name, '.mdb')))
                         continue;
                     if (databaseEntry.isFile() && (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb') {
                         readMetaDb((0, path_1.join)(databasePath, databaseEntry.name), (0, path_1.basename)(databaseEntry.name, '.mdb'), dbName);
@@ -413,6 +551,27 @@ function getDatabases() {
     }
     return exports.databases;
 }
+/**
+ * Scan a databases directory's entries for restore lock/marker files and return the names of
+ * databases that must not be loaded: a held restore lock means a restore is in progress in some
+ * process; an unheld lock with a surviving `.restoring` marker means a restore was interrupted
+ * mid-purge (the directory may be partial garbage) and must be rerun. The files live *next to*
+ * the database directory, so this also covers a database whose directory is missing or empty.
+ */
+function databasesBlockedByRestore(databasePath) {
+    const blocked = new Set();
+    for (const [dbName, state] of (0, restoreMarker_ts_1.scanBlockedRestores)(databasePath)) {
+        if (state === 'in-progress') {
+            logger.warn(`A restore of database '${dbName}' is in progress; not loading it`);
+            blocked.add(dbName);
+        }
+        else if (state === 'incomplete') {
+            logger.error(`Incomplete restore of database '${dbName}' detected (a restore started but did not finish); not loading it — rerun the restore to recover`);
+            blocked.add(dbName);
+        }
+    }
+    return blocked;
+}
 /**
  * This is responsible for reading the internal dbi of a single database file to get a list of all the tables and
  * their indexed or registered attributes
@@ -823,19 +982,12 @@ function setTable(tables, tableName, Table) {
     return Table;
 }
 /**
- * Get root store for a database
- * @param options
- * @returns
+ * Resolve the directory that holds (or would hold) a database's storage, from the databases
+ * config, storage path config/env, or the hdb root — without opening anything. This is the
+ * parent directory selection used by `database()`; a RocksDB database lives at
+ * `join(resolveDatabaseStorageRoot(...), databaseName)`.
  */
-function database({ database: databaseName, table: tableName }) {
-    if (!databaseName)
-        databaseName = DEFAULT_DATABASE_NAME;
-    getDatabases();
-    ensureDB(databaseName);
-    const definedDatabase = definedDatabases.get(databaseName);
-    if (definedDatabase?.rootStore) {
-        return definedDatabase.rootStore;
-    }
+function resolveDatabaseStorageRoot(databaseName, tableName) {
     const databaseConfig = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
     if (process.env.SCHEMAS_DATA_PATH) {
         databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
@@ -854,12 +1006,44 @@ function database({ database: databaseName, table: tableName }) {
     if (!databasePath) {
         throw new Error(`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`);
     }
+    return databasePath;
+}
+/**
+ * Resolve the directory path of a RocksDB database (whether or not it exists or is loaded).
+ */
+function resolveDatabasePath(databaseName) {
+    return (0, path_1.join)(resolveDatabaseStorageRoot(databaseName), databaseName);
+}
+/**
+ * Get root store for a database
+ * @param options
+ * @returns
+ */
+function database({ database: databaseName, table: tableName }) {
+    if (!databaseName)
+        databaseName = DEFAULT_DATABASE_NAME;
+    getDatabases();
+    ensureDB(databaseName);
+    const definedDatabase = definedDatabases.get(databaseName);
+    if (definedDatabase?.rootStore) {
+        return definedDatabase.rootStore;
+    }
+    const databaseConfig = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    if (process.env.SCHEMAS_DATA_PATH) {
+        databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
+    }
+    const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
+    const databasePath = resolveDatabaseStorageRoot(databaseName, tableName);
     let rootStore;
     const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
     if (useRocksdb) {
         const path = (0, path_1.join)(databasePath, tablePath ? tableName : databaseName);
         rootStore = rocksdbDatabaseEnvs.get(path);
         if (!rootStore || rootStore.status === 'closed') {
+            // this on-demand open (create_table/create_database and friends) must not resurrect a
+            // database that a restore is rewriting (or left half-purged) — the scan-time restore
+            // checks don't cover this path
+            throwIfBlockedByRestore(path, databaseName);
             rootStore = openRocksDatabase(path, {
                 disableWAL: false,
                 enableStats: true,
@@ -884,6 +1068,49 @@ function database({ database: databaseName, table: tableName }) {
         definedDatabase.rootStore = rootStore;
     return rootStore;
 }
+function throwIfBlockedByRestore(dbPath, databaseName) {
+    const restoreState = (0, restoreMarker_ts_1.checkRestoreState)(dbPath);
+    if (restoreState !== 'clear') {
+        const error = new Error(restoreState === 'in-progress'
+            ? `Database '${databaseName}' is being restored; retry when the restore completes`
+            : `Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`);
+        error.statusCode = 409;
+        throw error;
+    }
+}
+/**
+ * Take the per-database restore lock for a drop, refusing (409) if a restore holds it (in-progress)
+ * or a crashed restore left a marker (incomplete). Pushes the acquired lock onto `held` so the
+ * caller releases it after the drop. On refusal, releases anything already held and throws.
+ *
+ * The lock is not reentrant within a process, so a path already in `held` must be skipped — every
+ * table in a RocksDB database shares one root store (and one lock path), and re-acquiring it in the
+ * same drop would spuriously 409 on the second table.
+ */
+function lockDatabaseForDrop(dbPath, databaseName, held) {
+    if (held.some((h) => h.dbPath === dbPath))
+        return;
+    let lock;
+    try {
+        lock = (0, restoreMarker_ts_1.acquireRestoreLock)(dbPath);
+    }
+    catch (error) {
+        for (const h of held)
+            (0, restoreMarker_ts_1.releaseRestoreLock)(h);
+        throw error; // 409: a restore is in progress and holds the lock
+    }
+    // We now hold the lock, so no restore is active. A surviving marker is therefore debris from a
+    // crashed restore (incomplete) — refuse rather than delete a directory that still needs recovery.
+    if ((0, restoreMarker_ts_1.restoreMarkerPresent)(dbPath)) {
+        (0, restoreMarker_ts_1.releaseRestoreLock)(lock);
+        for (const h of held)
+            (0, restoreMarker_ts_1.releaseRestoreLock)(h);
+        const error = new Error(`Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`);
+        error.statusCode = 409;
+        throw error;
+    }
+    held.push(lock);
+}
 /**
  * Delete the database
  * @param databaseName
@@ -893,15 +1120,112 @@ async function dropDatabase(databaseName) {
         throw new Error('Database does not exist');
     const dbTables = exports.databases[databaseName];
     let rootStore;
+    // Hold the per-database restore lock across the entire drop so its file deletion can never
+    // interleave with a restore's purge-and-copy on the same directory — a destroy landing after a
+    // restore's copy would gut a "successful" restore, and vice versa. Restore takes the same lock
+    // (before writing its marker), so both operations serialize on this one primitive rather than on
+    // a check-then-act marker probe. Released in the finally below.
+    const restoreLocks = [];
+    try {
+        for (const tableName in dbTables) {
+            const table = dbTables[tableName];
+            rootStore = table.primaryStore.rootStore;
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase)
+                lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+            lmdbDatabaseEnvs.delete(rootStore.path);
+            rocksdbDatabaseEnvs.delete(rootStore.path);
+        }
+        for (const tableName in dbTables) {
+            exports.databaseEventsEmitter.emit('dropTable', tableName, databaseName);
+        }
+        if (databaseName === 'data') {
+            for (const tableName in exports.tables) {
+                delete exports.tables[tableName];
+            }
+            delete exports.tables[DEFINED_TABLES];
+        }
+        delete exports.databases[databaseName];
+        exports.databaseEventsEmitter.emit('dropDatabase', databaseName);
+        if (rootStore) {
+            if (rootStore.status === 'open') {
+                if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                    rootStore.close();
+                    rootStore.destroy();
+                }
+                else {
+                    await rootStore.close();
+                    await (0, promises_1.unlink)(rootStore.path);
+                }
+            }
+        }
+        else {
+            rootStore = database({ database: databaseName, table: null });
+            // a tableless database resolves its root store here rather than in the loop above, so take
+            // the drop lock now (still before any destructive step)
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase)
+                lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                rootStore.close();
+                rootStore.destroy();
+            }
+            else if (rootStore.status === 'open') {
+                await rootStore.close();
+                await (0, promises_1.unlink)(rootStore.path);
+            }
+        }
+        await (0, blob_ts_1.deleteRootBlobPathsForDB)(rootStore);
+    }
+    finally {
+        for (const lock of restoreLocks)
+            (0, restoreMarker_ts_1.releaseRestoreLock)(lock);
+    }
+}
+/**
+ * Close a RocksDB database's store handles on the current thread and unregister it, without
+ * touching its files. Used by the restore_backup flow: every thread must release its handles so
+ * `backups.restore()` can purge and rewrite the (fully closed) database directory. A subsequent
+ * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in
+ * progress, per the restore marker checks in the scan).
+ */
+function closeDatabase(databaseName) {
+    const dbTables = exports.databases[databaseName];
+    if (!dbTables)
+        return false;
+    const rootStores = new Set();
+    const closeStore = (store, description) => {
+        try {
+            store?.close?.();
+        }
+        catch (error) {
+            logger.warn(`Error closing ${description} while closing database ${databaseName}:`, error);
+        }
+    };
     for (const tableName in dbTables) {
         const table = dbTables[tableName];
-        rootStore = table.primaryStore.rootStore;
+        if (!table?.primaryStore)
+            continue;
+        if (table.primaryStore.rootStore)
+            rootStores.add(table.primaryStore.rootStore);
+        for (const indexName in table.indices || {}) {
+            closeStore(table.indices[indexName], `index ${tableName}.${indexName}`);
+        }
+        closeStore(table.primaryStore, `table ${tableName}`);
+    }
+    // a database with no tables (an empty schema, or one whose tables were all dropped) still holds
+    // an open root store, tracked only on the defined-database entry rather than any table — include
+    // it so its handles are released too (the Set dedupes it against the per-table root stores above)
+    const definedRoot = definedDatabases?.get(databaseName)?.rootStore;
+    if (definedRoot)
+        rootStores.add(definedRoot);
+    for (const rootStore of rootStores) {
+        closeStore(rootStore.dbisDb, 'attributes store');
+        closeStore(rootStore, 'root store');
         lmdbDatabaseEnvs.delete(rootStore.path);
         rocksdbDatabaseEnvs.delete(rootStore.path);
     }
-    for (const tableName in dbTables) {
-        exports.databaseEventsEmitter.emit('dropTable', tableName, databaseName);
-    }
+    const definedDatabase = definedDatabases?.get(databaseName);
+    if (definedDatabase)
+        definedDatabase.rootStore = undefined;
     if (databaseName === 'data') {
         for (const tableName in exports.tables) {
             delete exports.tables[tableName];
@@ -909,31 +1233,42 @@ async function dropDatabase(databaseName) {
         delete exports.tables[DEFINED_TABLES];
     }
     delete exports.databases[databaseName];
-    exports.databaseEventsEmitter.emit('dropDatabase', databaseName);
-    if (rootStore) {
-        if (rootStore.status === 'open') {
-            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
-                rootStore.close();
-                rootStore.destroy();
-            }
-            else {
-                await rootStore.close();
-                await (0, promises_1.unlink)(rootStore.path);
+    return true;
+}
+/**
+ * Close every RocksDB (user) database this thread has open, releasing its native handles.
+ *
+ * rocksdb-js's registry is process-global across worker threads, and a thread that exits WITHOUT
+ * closing leaks its handles (the process-global refCount never drops), while the only alternative,
+ * `shutdown()`, tears down rocksdb for the entire process. So a worker thread that opens databases
+ * and then exits — notably a job worker (jobProcess), which opens the whole database graph via
+ * `getDatabases()` and exits when the job finishes — must close its handles explicitly, or those
+ * handles linger process-wide (and, e.g., block an online `restore_backup` from confirming the
+ * database is closed). The `system` database is intentionally left open: it is non-enumerable here
+ * (skipped by the loop), is never restored online, and the exiting worker may still touch the job
+ * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`.
+ */
+function closeLoadedDatabases() {
+    // snapshot the names first: closeDatabase() deletes from `databases` as it goes
+    for (const databaseName of Object.keys(exports.databases)) {
+        const dbTables = exports.databases[databaseName];
+        if (!dbTables)
+            continue;
+        let isRocks = false;
+        for (const tableName in dbTables) {
+            if (dbTables[tableName]?.primaryStore?.rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                isRocks = true;
+                break;
             }
         }
-    }
-    else {
-        rootStore = database({ database: databaseName, table: null });
-        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
-            rootStore.close();
-            rootStore.destroy();
-        }
-        else if (rootStore.status === 'open') {
-            await rootStore.close();
-            await (0, promises_1.unlink)(rootStore.path);
+        // a tableless database exposes no table root store, so also check the defined-database
+        // entry — otherwise its open root store would leak on worker exit
+        if (!isRocks && definedDatabases?.get(databaseName)?.rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            isRocks = true;
         }
+        if (isRocks)
+            closeDatabase(databaseName);
     }
-    await (0, blob_ts_1.deleteRootBlobPathsForDB)(rootStore);
 }
 // HNSW_NO_AUTOVERSION kill-switch: when set, a NEW index initializes as legacy rather than
 // versioned. process.env values are strings, so a bare truthiness check would treat "0"/"false"
@@ -1010,7 +1345,11 @@ function openIndex(dbiKey, rootStore, attribute) {
         // Enable cache (WeakLRUCache + VT) for all custom-object index stores so the VT is
         // available before resolveIndexFormat decides the format. Versioned stores need the VT
         // for cached traversal; legacy stores pay a small per-write cache.delete() overhead only.
-        dbi = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiKey, cache: isCustomObjectIndex });
+        dbi = openRocksDatabase(rootStore.path, {
+            ...dbiInit,
+            name: dbiKey,
+            cache: isCustomObjectIndex,
+        });
         dbi.rootStore = rootStore;
         // Custom-index object stores (e.g. HNSW) write graph nodes via plain put() with no staged
         // transaction timestamp, so their values carry no version and the PrimaryRocksDatabase
@@ -1230,6 +1569,10 @@ function table(tableDefinition) {
                 clearInterruptedDropEntries(rootStore.path, tableName);
             }
             if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                // Usually a genuinely new column family (existingTableMeta above found no catalog
+                // entry), but an interrupted drop just completed above can leave the physical CF
+                // behind under its old codec even though the catalog entry is gone — same fallback
+                // as the reconcile paths covers that remnant case too.
                 primaryStore = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiName, cache: true });
             }
             else {
@@ -1950,7 +2293,9 @@ function getDefaultCompression() {
         LMDB_COMPRESSION_OPTS['dictionary'] = (0, node_fs_1.readFileSync)(STORAGE_COMPRESSION_DICTIONARY);
     if (STORAGE_COMPRESSION_THRESHOLD)
         LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
-    return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
+    // normalize disabled to false so a falsy config value ('' or null) is never persisted
+    // into table metadata as-is (openRocksDatabase maps defined-falsy to 'none')
+    return LMDB_COMPRESSION ? LMDB_COMPRESSION_OPTS : false;
 }
 /**
  * Force all RocksDB databases to flush to disk.