@harperfast/harper 5.2.0-beta.4 → 5.2.0

package/dist/resources/databases.js

@@ -38,12 +38,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
 exports.isReadOnlyMode = isReadOnlyMode;
+exports.toRocksCompression = toRocksCompression;
 exports.getTables = getTables;
 exports.getDatabases = getDatabases;
 exports.readMetaDb = readMetaDb;
 exports.resetDatabases = resetDatabases;
+exports.resolveDatabaseStorageRoot = resolveDatabaseStorageRoot;
+exports.resolveDatabasePath = resolveDatabasePath;
 exports.database = database;
 exports.dropDatabase = dropDatabase;
+exports.closeDatabase = closeDatabase;
+exports.closeLoadedDatabases = closeLoadedDatabases;
 exports.table = table;
 exports.canonicalizeIndexOptions = canonicalizeIndexOptions;
 exports.dropTableMeta = dropTableMeta;
@@ -86,6 +91,7 @@ const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
 const when_ts_1 = require("../utility/when.js");
 const rocksMemoryConfig_ts_1 = require("../utility/rocksMemoryConfig.js");
 const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
+const restoreMarker_ts_1 = require("../dataLayer/restoreMarker.js");
 /**
  * Check if Harper is running in read-only mode.
  * Read-only mode can be enabled via:
@@ -148,8 +154,24 @@ exports.NON_REPLICATING_SYSTEM_TABLES = [
 exports.databaseEventsEmitter = new node_events_1.EventEmitter();
 exports.tables = Object.create(null);
 exports.databases = Object.create(null);
+/**
+ * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata
+ * carries values where a defined falsy value (false, '') means compression was explicitly
+ * disabled and true/{ threshold, ... } mean enabled with defaults. rocksdb-js >= 2.6
+ * validates `compression` strictly and treats UNSET as "use the build default (lz4)", so
+ * disabled must be mapped to 'none' explicitly, and true to unset.
+ */
+function toRocksCompression(compression) {
+    if (compression === true)
+        return undefined;
+    if (compression !== undefined && !compression)
+        return 'none';
+    return compression;
+}
 function openRocksDatabase(path, options) {
     options.disableWAL ??= true;
+    const legacyOptions = options;
+    legacyOptions.compression = toRocksCompression(legacyOptions.compression);
     // Apply read-only mode if enabled
     if (isReadOnlyMode()) {
         options.readOnly = true;
@@ -284,12 +306,21 @@ function getDatabases() {
     if (databasePath && (0, node_fs_1.existsSync)(databasePath)) {
         // First load all the databases from our main database folder
         // TODO: Load any databases defined with explicit storage paths from the config
-        for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
+        const entries = (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true });
+        const blockedByRestore = databasesBlockedByRestore(databasePath);
+        for (const databaseEntry of entries) {
             // in-progress migration staging dirs are not databases until atomically renamed into place
             if (databaseEntry.name.endsWith(hdbTerms_ts_1.MIGRATING_DIR_SUFFIX))
                 continue;
+            // the restore-metadata directory is reserved: never load it as a database, even if a
+            // (out-of-band) RocksDB directory happens to occupy that reserved name — the API can't
+            // create it (schemaRegex forbids the backtick), but the scan opens any CURRENT+MANIFEST dir
+            if (databaseEntry.name === restoreMarker_ts_1.RESTORE_META_DIR)
+                continue;
             const dbName = (0, path_1.basename)(databaseEntry.name, '.mdb');
             const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
+            if (blockedByRestore.has(dbName))
+                continue;
             if (databaseEntry.isFile() &&
                 (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb' &&
                 !schemaConfigs[dbName]?.path) {
@@ -334,8 +365,14 @@ function getDatabases() {
             const schemaConfig = schemaConfigs[dbName];
             const databasePath = schemaConfig.path;
             if ((0, node_fs_1.existsSync)(databasePath)) {
-                for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
+                const entries = (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true });
+                const blockedByRestore = databasesBlockedByRestore(databasePath);
+                for (const databaseEntry of entries) {
                     if (databaseEntry.name.endsWith(hdbTerms_ts_1.MIGRATING_DIR_SUFFIX))
+                        continue; // migration staging dir
+                    if (databaseEntry.name === restoreMarker_ts_1.RESTORE_META_DIR)
+                        continue; // reserved restore-metadata dir
+                    if (blockedByRestore.has((0, path_1.basename)(databaseEntry.name, '.mdb')))
                         continue;
                     if (databaseEntry.isFile() && (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb') {
                         readMetaDb((0, path_1.join)(databasePath, databaseEntry.name), (0, path_1.basename)(databaseEntry.name, '.mdb'), dbName);
@@ -413,6 +450,27 @@ function getDatabases() {
     }
     return exports.databases;
 }
+/**
+ * Scan a databases directory's entries for restore lock/marker files and return the names of
+ * databases that must not be loaded: a held restore lock means a restore is in progress in some
+ * process; an unheld lock with a surviving `.restoring` marker means a restore was interrupted
+ * mid-purge (the directory may be partial garbage) and must be rerun. The files live *next to*
+ * the database directory, so this also covers a database whose directory is missing or empty.
+ */
+function databasesBlockedByRestore(databasePath) {
+    const blocked = new Set();
+    for (const [dbName, state] of (0, restoreMarker_ts_1.scanBlockedRestores)(databasePath)) {
+        if (state === 'in-progress') {
+            logger.warn(`A restore of database '${dbName}' is in progress; not loading it`);
+            blocked.add(dbName);
+        }
+        else if (state === 'incomplete') {
+            logger.error(`Incomplete restore of database '${dbName}' detected (a restore started but did not finish); not loading it — rerun the restore to recover`);
+            blocked.add(dbName);
+        }
+    }
+    return blocked;
+}
 /**
  * This is responsible for reading the internal dbi of a single database file to get a list of all the tables and
  * their indexed or registered attributes
@@ -823,19 +881,12 @@ function setTable(tables, tableName, Table) {
     return Table;
 }
 /**
- * Get root store for a database
- * @param options
- * @returns
+ * Resolve the directory that holds (or would hold) a database's storage, from the databases
+ * config, storage path config/env, or the hdb root — without opening anything. This is the
+ * parent directory selection used by `database()`; a RocksDB database lives at
+ * `join(resolveDatabaseStorageRoot(...), databaseName)`.
  */
-function database({ database: databaseName, table: tableName }) {
-    if (!databaseName)
-        databaseName = DEFAULT_DATABASE_NAME;
-    getDatabases();
-    ensureDB(databaseName);
-    const definedDatabase = definedDatabases.get(databaseName);
-    if (definedDatabase?.rootStore) {
-        return definedDatabase.rootStore;
-    }
+function resolveDatabaseStorageRoot(databaseName, tableName) {
     const databaseConfig = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
     if (process.env.SCHEMAS_DATA_PATH) {
         databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
@@ -854,12 +905,44 @@ function database({ database: databaseName, table: tableName }) {
     if (!databasePath) {
         throw new Error(`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`);
     }
+    return databasePath;
+}
+/**
+ * Resolve the directory path of a RocksDB database (whether or not it exists or is loaded).
+ */
+function resolveDatabasePath(databaseName) {
+    return (0, path_1.join)(resolveDatabaseStorageRoot(databaseName), databaseName);
+}
+/**
+ * Get root store for a database
+ * @param options
+ * @returns
+ */
+function database({ database: databaseName, table: tableName }) {
+    if (!databaseName)
+        databaseName = DEFAULT_DATABASE_NAME;
+    getDatabases();
+    ensureDB(databaseName);
+    const definedDatabase = definedDatabases.get(databaseName);
+    if (definedDatabase?.rootStore) {
+        return definedDatabase.rootStore;
+    }
+    const databaseConfig = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    if (process.env.SCHEMAS_DATA_PATH) {
+        databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
+    }
+    const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
+    const databasePath = resolveDatabaseStorageRoot(databaseName, tableName);
     let rootStore;
     const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
     if (useRocksdb) {
         const path = (0, path_1.join)(databasePath, tablePath ? tableName : databaseName);
         rootStore = rocksdbDatabaseEnvs.get(path);
         if (!rootStore || rootStore.status === 'closed') {
+            // this on-demand open (create_table/create_database and friends) must not resurrect a
+            // database that a restore is rewriting (or left half-purged) — the scan-time restore
+            // checks don't cover this path
+            throwIfBlockedByRestore(path, databaseName);
             rootStore = openRocksDatabase(path, {
                 disableWAL: false,
                 enableStats: true,
@@ -884,6 +967,49 @@ function database({ database: databaseName, table: tableName }) {
         definedDatabase.rootStore = rootStore;
     return rootStore;
 }
+function throwIfBlockedByRestore(dbPath, databaseName) {
+    const restoreState = (0, restoreMarker_ts_1.checkRestoreState)(dbPath);
+    if (restoreState !== 'clear') {
+        const error = new Error(restoreState === 'in-progress'
+            ? `Database '${databaseName}' is being restored; retry when the restore completes`
+            : `Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`);
+        error.statusCode = 409;
+        throw error;
+    }
+}
+/**
+ * Take the per-database restore lock for a drop, refusing (409) if a restore holds it (in-progress)
+ * or a crashed restore left a marker (incomplete). Pushes the acquired lock onto `held` so the
+ * caller releases it after the drop. On refusal, releases anything already held and throws.
+ *
+ * The lock is not reentrant within a process, so a path already in `held` must be skipped — every
+ * table in a RocksDB database shares one root store (and one lock path), and re-acquiring it in the
+ * same drop would spuriously 409 on the second table.
+ */
+function lockDatabaseForDrop(dbPath, databaseName, held) {
+    if (held.some((h) => h.dbPath === dbPath))
+        return;
+    let lock;
+    try {
+        lock = (0, restoreMarker_ts_1.acquireRestoreLock)(dbPath);
+    }
+    catch (error) {
+        for (const h of held)
+            (0, restoreMarker_ts_1.releaseRestoreLock)(h);
+        throw error; // 409: a restore is in progress and holds the lock
+    }
+    // We now hold the lock, so no restore is active. A surviving marker is therefore debris from a
+    // crashed restore (incomplete) — refuse rather than delete a directory that still needs recovery.
+    if ((0, restoreMarker_ts_1.restoreMarkerPresent)(dbPath)) {
+        (0, restoreMarker_ts_1.releaseRestoreLock)(lock);
+        for (const h of held)
+            (0, restoreMarker_ts_1.releaseRestoreLock)(h);
+        const error = new Error(`Database '${databaseName}' has an incomplete restore; rerun restore_backup to recover it`);
+        error.statusCode = 409;
+        throw error;
+    }
+    held.push(lock);
+}
 /**
  * Delete the database
  * @param databaseName
@@ -893,15 +1019,112 @@ async function dropDatabase(databaseName) {
         throw new Error('Database does not exist');
     const dbTables = exports.databases[databaseName];
     let rootStore;
+    // Hold the per-database restore lock across the entire drop so its file deletion can never
+    // interleave with a restore's purge-and-copy on the same directory — a destroy landing after a
+    // restore's copy would gut a "successful" restore, and vice versa. Restore takes the same lock
+    // (before writing its marker), so both operations serialize on this one primitive rather than on
+    // a check-then-act marker probe. Released in the finally below.
+    const restoreLocks = [];
+    try {
+        for (const tableName in dbTables) {
+            const table = dbTables[tableName];
+            rootStore = table.primaryStore.rootStore;
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase)
+                lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+            lmdbDatabaseEnvs.delete(rootStore.path);
+            rocksdbDatabaseEnvs.delete(rootStore.path);
+        }
+        for (const tableName in dbTables) {
+            exports.databaseEventsEmitter.emit('dropTable', tableName, databaseName);
+        }
+        if (databaseName === 'data') {
+            for (const tableName in exports.tables) {
+                delete exports.tables[tableName];
+            }
+            delete exports.tables[DEFINED_TABLES];
+        }
+        delete exports.databases[databaseName];
+        exports.databaseEventsEmitter.emit('dropDatabase', databaseName);
+        if (rootStore) {
+            if (rootStore.status === 'open') {
+                if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                    rootStore.close();
+                    rootStore.destroy();
+                }
+                else {
+                    await rootStore.close();
+                    await (0, promises_1.unlink)(rootStore.path);
+                }
+            }
+        }
+        else {
+            rootStore = database({ database: databaseName, table: null });
+            // a tableless database resolves its root store here rather than in the loop above, so take
+            // the drop lock now (still before any destructive step)
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase)
+                lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                rootStore.close();
+                rootStore.destroy();
+            }
+            else if (rootStore.status === 'open') {
+                await rootStore.close();
+                await (0, promises_1.unlink)(rootStore.path);
+            }
+        }
+        await (0, blob_ts_1.deleteRootBlobPathsForDB)(rootStore);
+    }
+    finally {
+        for (const lock of restoreLocks)
+            (0, restoreMarker_ts_1.releaseRestoreLock)(lock);
+    }
+}
+/**
+ * Close a RocksDB database's store handles on the current thread and unregister it, without
+ * touching its files. Used by the restore_backup flow: every thread must release its handles so
+ * `backups.restore()` can purge and rewrite the (fully closed) database directory. A subsequent
+ * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in
+ * progress, per the restore marker checks in the scan).
+ */
+function closeDatabase(databaseName) {
+    const dbTables = exports.databases[databaseName];
+    if (!dbTables)
+        return false;
+    const rootStores = new Set();
+    const closeStore = (store, description) => {
+        try {
+            store?.close?.();
+        }
+        catch (error) {
+            logger.warn(`Error closing ${description} while closing database ${databaseName}:`, error);
+        }
+    };
     for (const tableName in dbTables) {
         const table = dbTables[tableName];
-        rootStore = table.primaryStore.rootStore;
+        if (!table?.primaryStore)
+            continue;
+        if (table.primaryStore.rootStore)
+            rootStores.add(table.primaryStore.rootStore);
+        for (const indexName in table.indices || {}) {
+            closeStore(table.indices[indexName], `index ${tableName}.${indexName}`);
+        }
+        closeStore(table.primaryStore, `table ${tableName}`);
+    }
+    // a database with no tables (an empty schema, or one whose tables were all dropped) still holds
+    // an open root store, tracked only on the defined-database entry rather than any table — include
+    // it so its handles are released too (the Set dedupes it against the per-table root stores above)
+    const definedRoot = definedDatabases?.get(databaseName)?.rootStore;
+    if (definedRoot)
+        rootStores.add(definedRoot);
+    for (const rootStore of rootStores) {
+        closeStore(rootStore.dbisDb, 'attributes store');
+        closeStore(rootStore, 'root store');
         lmdbDatabaseEnvs.delete(rootStore.path);
         rocksdbDatabaseEnvs.delete(rootStore.path);
     }
-    for (const tableName in dbTables) {
-        exports.databaseEventsEmitter.emit('dropTable', tableName, databaseName);
-    }
+    const definedDatabase = definedDatabases?.get(databaseName);
+    if (definedDatabase)
+        definedDatabase.rootStore = undefined;
     if (databaseName === 'data') {
         for (const tableName in exports.tables) {
             delete exports.tables[tableName];
@@ -909,31 +1132,42 @@ async function dropDatabase(databaseName) {
         delete exports.tables[DEFINED_TABLES];
     }
     delete exports.databases[databaseName];
-    exports.databaseEventsEmitter.emit('dropDatabase', databaseName);
-    if (rootStore) {
-        if (rootStore.status === 'open') {
-            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
-                rootStore.close();
-                rootStore.destroy();
-            }
-            else {
-                await rootStore.close();
-                await (0, promises_1.unlink)(rootStore.path);
+    return true;
+}
+/**
+ * Close every RocksDB (user) database this thread has open, releasing its native handles.
+ *
+ * rocksdb-js's registry is process-global across worker threads, and a thread that exits WITHOUT
+ * closing leaks its handles (the process-global refCount never drops), while the only alternative,
+ * `shutdown()`, tears down rocksdb for the entire process. So a worker thread that opens databases
+ * and then exits — notably a job worker (jobProcess), which opens the whole database graph via
+ * `getDatabases()` and exits when the job finishes — must close its handles explicitly, or those
+ * handles linger process-wide (and, e.g., block an online `restore_backup` from confirming the
+ * database is closed). The `system` database is intentionally left open: it is non-enumerable here
+ * (skipped by the loop), is never restored online, and the exiting worker may still touch the job
+ * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`.
+ */
+function closeLoadedDatabases() {
+    // snapshot the names first: closeDatabase() deletes from `databases` as it goes
+    for (const databaseName of Object.keys(exports.databases)) {
+        const dbTables = exports.databases[databaseName];
+        if (!dbTables)
+            continue;
+        let isRocks = false;
+        for (const tableName in dbTables) {
+            if (dbTables[tableName]?.primaryStore?.rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                isRocks = true;
+                break;
             }
         }
-    }
-    else {
-        rootStore = database({ database: databaseName, table: null });
-        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
-            rootStore.close();
-            rootStore.destroy();
-        }
-        else if (rootStore.status === 'open') {
-            await rootStore.close();
-            await (0, promises_1.unlink)(rootStore.path);
+        // a tableless database exposes no table root store, so also check the defined-database
+        // entry — otherwise its open root store would leak on worker exit
+        if (!isRocks && definedDatabases?.get(databaseName)?.rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            isRocks = true;
         }
+        if (isRocks)
+            closeDatabase(databaseName);
     }
-    await (0, blob_ts_1.deleteRootBlobPathsForDB)(rootStore);
 }
 // HNSW_NO_AUTOVERSION kill-switch: when set, a NEW index initializes as legacy rather than
 // versioned. process.env values are strings, so a bare truthiness check would treat "0"/"false"
@@ -1950,7 +2184,9 @@ function getDefaultCompression() {
         LMDB_COMPRESSION_OPTS['dictionary'] = (0, node_fs_1.readFileSync)(STORAGE_COMPRESSION_DICTIONARY);
     if (STORAGE_COMPRESSION_THRESHOLD)
         LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
-    return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
+    // normalize disabled to false so a falsy config value ('' or null) is never persisted
+    // into table metadata as-is (openRocksDatabase maps defined-falsy to 'none')
+    return LMDB_COMPRESSION ? LMDB_COMPRESSION_OPTS : false;
 }
 /**
  * Force all RocksDB databases to flush to disk.