@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/dist/resources/dataLoader.js

@@ -12,10 +12,10 @@ const node_crypto_1 = require("node:crypto");
 const yaml_1 = require("yaml");
 const databases_ts_1 = require("./databases.js");
 const manageThreads_1 = require("../server/threads/manageThreads");
-const commonErrors_js_1 = require("../utility/errors/commonErrors.js");
-const hdbError_js_1 = require("../utility/errors/hdbError.js");
-const harper_logger_js_1 = __importDefault(require("../utility/logging/harper_logger.js"));
-const dataLoaderLogger = harper_logger_js_1.default.forComponent('dataLoader');
+const commonErrors_ts_1 = require("../utility/errors/commonErrors.js");
+const hdbError_ts_1 = require("../utility/errors/hdbError.js");
+const harper_logger_ts_1 = __importDefault(require("../utility/logging/harper_logger.js"));
+const dataLoaderLogger = harper_logger_ts_1.default.forComponent('dataLoader');
 /** System table name for storing data loader hashes */
 const DATA_LOADER_HASH_TABLE = 'hdb_dataloader_hash';
 /** Lazy-initialized cache for the hash tracking table */
@@ -333,8 +333,8 @@ async function loadDataFile({ contents, absolutePath }, tablesRef, databasesRef)
 /**
  * Base class for DataLoader specific errors
  */
-class DataLoaderError extends hdbError_js_1.ClientError {
-    constructor(message, statusCode = commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST) {
+class DataLoaderError extends hdbError_ts_1.ClientError {
+    constructor(message, statusCode = commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST) {
         super(message, statusCode);
         this.name = 'DataLoaderError';
     }
@@ -345,7 +345,7 @@ exports.DataLoaderError = DataLoaderError;
  */
 class UnsupportedFileExtensionError extends DataLoaderError {
     constructor(filePath, extension) {
-        super(`Unsupported file extension in ${(0, node_path_1.basename)(filePath)}: ${extension}. Only YAML and JSON files are supported.`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
+        super(`Unsupported file extension in ${(0, node_path_1.basename)(filePath)}: ${extension}. Only YAML and JSON files are supported.`, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
         this.name = 'UnsupportedFileExtensionError';
     }
 }
@@ -355,7 +355,7 @@ exports.UnsupportedFileExtensionError = UnsupportedFileExtensionError;
  */
 class FileParseError extends DataLoaderError {
     constructor(filePath, originalError) {
-        super(`Failed to parse data file ${(0, node_path_1.basename)(filePath)}: ${originalError.message}`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
+        super(`Failed to parse data file ${(0, node_path_1.basename)(filePath)}: ${originalError.message}`, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
         this.name = 'FileParseError';
     }
 }
@@ -365,7 +365,7 @@ exports.FileParseError = FileParseError;
  */
 class EmptyFileError extends DataLoaderError {
     constructor(filePath) {
-        super(`Data file ${(0, node_path_1.basename)(filePath)} is empty or invalid`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
+        super(`Data file ${(0, node_path_1.basename)(filePath)} is empty or invalid`, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
         this.name = 'EmptyFileError';
     }
 }
@@ -375,7 +375,7 @@ exports.EmptyFileError = EmptyFileError;
  */
 class MissingRequiredPropertyError extends DataLoaderError {
     constructor(filePath, property) {
-        super(`Data file ${(0, node_path_1.basename)(filePath)} is missing required "${property}" property`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
+        super(`Data file ${(0, node_path_1.basename)(filePath)} is missing required "${property}" property`, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
         this.name = 'MissingRequiredPropertyError';
     }
 }
@@ -385,7 +385,7 @@ exports.MissingRequiredPropertyError = MissingRequiredPropertyError;
  */
 class InvalidPropertyTypeError extends DataLoaderError {
     constructor(filePath, property, expectedType) {
-        super(`Data file ${(0, node_path_1.basename)(filePath)} has invalid "${property}" property, expected ${expectedType}`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
+        super(`Data file ${(0, node_path_1.basename)(filePath)} has invalid "${property}" property, expected ${expectedType}`, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
         this.name = 'InvalidPropertyTypeError';
     }
 }
@@ -395,7 +395,7 @@ exports.InvalidPropertyTypeError = InvalidPropertyTypeError;
  */
 class SystemDatabaseError extends DataLoaderError {
     constructor(database, table) {
-        super(`Cannot load data into system database: ${database}.${table}`, commonErrors_js_1.HTTP_STATUS_CODES.FORBIDDEN);
+        super(`Cannot load data into system database: ${database}.${table}`, commonErrors_ts_1.HTTP_STATUS_CODES.FORBIDDEN);
         this.name = 'SystemDatabaseError';
     }
 }
@@ -405,7 +405,7 @@ exports.SystemDatabaseError = SystemDatabaseError;
  */
 class RecordProcessingError extends DataLoaderError {
     constructor(tableIdentifier, originalError) {
-        super(`Failed to process record in ${tableIdentifier}: ${originalError.message}`, commonErrors_js_1.HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR);
+        super(`Failed to process record in ${tableIdentifier}: ${originalError.message}`, commonErrors_ts_1.HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR);
         this.name = 'RecordProcessingError';
     }
 }

package/dist/resources/databases.d.ts

@@ -2,6 +2,7 @@ import { EventEmitter } from 'node:events';
 import { type Database, type RootDatabase } from 'lmdb';
 import { makeTable } from './Table.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
+export declare function isReadOnlyMode(): boolean;
 declare const DEFINED_TABLES: unique symbol;
 export declare const NON_REPLICATING_SYSTEM_TABLES: string[];
 export type Table = ReturnType<typeof makeTable> & {
@@ -29,6 +30,10 @@ interface LMDBRootDatabase extends RootDatabase {
     needsDeletion?: boolean;
     path?: string;
     status?: 'open' | 'closed';
+    store: any;
+    retryRisk?: number;
+    flushed: Promise<boolean>;
+    rootStore?: LMDBRootDatabase;
 }
 interface RocksDatabaseEx extends RocksDatabase {
     customIndex?: any;
@@ -44,6 +49,10 @@ interface RocksRootDatabase extends RocksDatabaseEx {
     auditStore?: RocksDatabaseEx;
     databaseName?: string;
     dbisDb?: RocksDatabaseEx;
+    store: any;
+    retryRisk?: number;
+    flushed: Promise<boolean>;
+    rootStore?: RocksRootDatabase;
 }
 export type RootDatabaseKind = LMDBRootDatabase | RocksRootDatabase;
 export type DatabaseWatcherEventMap = {
@@ -138,4 +147,8 @@ export declare function onRemovedDB(listener: (databaseName: string) => void): {
 export declare function getDefaultCompression(): {
     startingOffset: number;
 };
+/**
+ * Force all RocksDB databases to flush to disk.
+ */
+export declare function flushDatabases(): Promise<void[]>;
 export {};

package/dist/resources/databases.js

@@ -37,6 +37,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
+exports.isReadOnlyMode = isReadOnlyMode;
 exports.getTables = getTables;
 exports.getDatabases = getDatabases;
 exports.readMetaDb = readMetaDb;
@@ -49,63 +50,127 @@ exports.onUpdatedTable = onUpdatedTable;
 exports.onRemovedTable = onRemovedTable;
 exports.onRemovedDB = onRemovedDB;
 exports.getDefaultCompression = getDefaultCompression;
+exports.flushDatabases = flushDatabases;
 const node_events_1 = require("node:events");
-const environmentManager_js_1 = require("../utility/environment/environmentManager.js");
-const terms_js_1 = require("../utility/lmdb/terms.js");
+const environmentManager_ts_1 = require("../utility/environment/environmentManager.js");
+const terms_ts_1 = require("../utility/lmdb/terms.js");
 const lmdb_1 = require("lmdb");
 const path_1 = require("path");
 const node_fs_1 = require("node:fs");
 const promises_1 = require("node:fs/promises");
 const initializePaths_js_1 = require("../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js");
 const Table_ts_1 = require("./Table.js");
-const OpenEnvironmentObject_js_1 = __importDefault(require("../utility/lmdb/OpenEnvironmentObject.js"));
+const OpenEnvironmentObject_ts_1 = __importDefault(require("../utility/lmdb/OpenEnvironmentObject.js"));
 const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
 const configUtils_js_1 = require("../config/configUtils.js");
 const globals_js_1 = require("../globals.js");
-const commonUtility_js_1 = require("../utility/lmdb/commonUtility.js");
+const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
 const signalling = __importStar(require("../utility/signalling.js"));
 const itc_js_1 = require("../server/threads/itc.js");
 const worker_threads_1 = require("worker_threads");
-const harper_logger_js_1 = __importDefault(require("../utility/logging/harper_logger.js"));
-const { forComponent } = harper_logger_js_1.default;
+const harper_logger_ts_1 = __importDefault(require("../utility/logging/harper_logger.js"));
+const { forComponent } = harper_logger_ts_1.default;
 const manageThreads = __importStar(require("../server/threads/manageThreads.js"));
 const auditStore_ts_1 = require("./auditStore.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
 const blob_ts_1 = require("./blob.js");
 const customIndexes_ts_1 = require("./indexes/customIndexes.js");
-const OpenDBIObject_js_1 = require("../utility/lmdb/OpenDBIObject.js");
+const OpenDBIObject_ts_1 = require("../utility/lmdb/OpenDBIObject.js");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const replayLogs_ts_1 = require("./replayLogs.js");
 const node_os_1 = require("node:os");
 const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
 const when_ts_1 = require("../utility/when.js");
 const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
+/**
+ * Check if Harper is running in read-only mode.
+ * Read-only mode can be enabled via:
+ * - HARPER_READONLY environment variable (truthy value)
+ * - --readonly CLI flag
+ * - storage.readOnly config setting
+ */
+let _isReadOnlyMode;
+function isReadOnlyMode() {
+    if (_isReadOnlyMode !== undefined)
+        return _isReadOnlyMode;
+    // Check environment variable
+    const envReadOnly = process.env.HARPER_READONLY;
+    if (envReadOnly && envReadOnly !== '0' && envReadOnly !== 'false') {
+        _isReadOnlyMode = true;
+        return true;
+    }
+    // Check CLI flag (simple argv check)
+    if (process.argv.includes('--readonly') || process.argv.includes('--read-only')) {
+        _isReadOnlyMode = true;
+        return true;
+    }
+    // Check config setting
+    if ((0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_READONLY)) {
+        _isReadOnlyMode = true;
+        return true;
+    }
+    _isReadOnlyMode = false;
+    return false;
+}
 function createOpenDBIObject(dupSort = false, isPrimary = false) {
-    return new OpenDBIObject_js_1.OpenDBIObject(dupSort, isPrimary);
+    return new OpenDBIObject_ts_1.OpenDBIObject(dupSort, isPrimary);
 }
 const logger = forComponent('storage');
 const DEFAULT_DATABASE_NAME = 'data';
 const DEFINED_TABLES = Symbol('defined-tables');
-const DEFAULT_COMPRESSION_THRESHOLD = ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
-(0, environmentManager_js_1.initSync)();
+const DEFAULT_COMPRESSION_THRESHOLD = ((0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
+(0, environmentManager_ts_1.initSync)();
 // I don't know if this is the best place for this, but somewhere we need to specify which tables
 // replicate by default:
 exports.NON_REPLICATING_SYSTEM_TABLES = [
     'hdb_temp',
     'hdb_certificate',
     'hdb_raw_analytics',
+    'hdb_model_calls',
     'hdb_session_will',
     'hdb_job',
     'hdb_info',
+    'mcp_session',
 ];
 exports.databaseEventsEmitter = new node_events_1.EventEmitter();
 exports.tables = Object.create(null);
 exports.databases = Object.create(null);
-const MEMORY_FOR_ROCKS_DB = Math.min(process.constrainedMemory?.() ?? Infinity, (0, node_os_1.totalmem)()) * 0.25; // 25% of available memory
 function openRocksDatabase(path, options) {
     options.disableWAL ??= true;
-    rocksdb_js_1.RocksDatabase.config({ blockCacheSize: MEMORY_FOR_ROCKS_DB });
+    // Apply read-only mode if enabled
+    if (isReadOnlyMode()) {
+        options.readOnly = true;
+    }
+    // Read RocksDB memory config lazily so env/CLI overrides applied after module load are
+    // respected. The block cache falls back to 25% of constrained (cgroup) memory when not
+    // configured; the WriteBufferManager is opt-in (0 disables).
+    //
+    // We enforce types rather than coerce — values from YAML config and env vars flow
+    // through configUtils.castConfigValue which produces proper numbers/booleans/null,
+    // so anything else is misconfiguration and should fall through to the default.
+    //
+    // Note: writeBufferManagerCostToCache and writeBufferManagerAllowStall are fixed at WBM
+    // creation time inside rocksdb-js (the underlying RocksDB API doesn't support changing
+    // costToCache on a live manager, and allowStall is only re-applied when explicitly changed).
+    // In practice that's fine — these come from process-level config that doesn't change.
+    const configuredBlockCacheSize = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE);
+    const blockCacheSize = typeof configuredBlockCacheSize === 'number' && configuredBlockCacheSize > 0
+        ? configuredBlockCacheSize
+        : Math.min(process.constrainedMemory?.() ?? Infinity, (0, node_os_1.totalmem)()) * 0.25;
+    const writeBufferManagerSize = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE);
+    const writeBufferManagerCostToCache = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE);
+    const writeBufferManagerAllowStall = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL);
+    rocksdb_js_1.RocksDatabase.config({
+        blockCacheSize,
+        ...(typeof writeBufferManagerSize === 'number' && writeBufferManagerSize > 0 ? { writeBufferManagerSize } : {}),
+        ...(typeof writeBufferManagerCostToCache === 'boolean' ? { writeBufferManagerCostToCache } : {}),
+        ...(typeof writeBufferManagerAllowStall === 'boolean' ? { writeBufferManagerAllowStall } : {}),
+    });
     if (!(0, node_fs_1.existsSync)(path)) {
+        // Don't create directories in read-only mode
+        if (isReadOnlyMode()) {
+            throw new Error(`Database cannot be created in read-only mode: ${path}`);
+        }
         (0, node_fs_1.mkdirSync)(path, { recursive: true });
     }
     let db;
@@ -157,19 +222,17 @@ function getDatabases() {
     }
     loadedDatabases = true;
     definedDatabases = new Map();
-    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
+    const hdbBasePath = (0, environmentManager_ts_1.getHdbBasePath)();
     let databasePath = hdbBasePath && (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME);
-    const schemaConfigs = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    const schemaConfigs = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
     // not sure why this doesn't work with the environmemt manager
     if (process.env.SCHEMAS_DATA_PATH)
         schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
     databasePath =
         process.env.STORAGE_PATH ||
             (0, configUtils_js_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
-            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, path_1.join)((0, environmentManager_js_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
-    if (!databasePath)
-        return;
-    if ((0, node_fs_1.existsSync)(databasePath)) {
+            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, path_1.join)((0, environmentManager_ts_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
+    if (databasePath && (0, node_fs_1.existsSync)(databasePath)) {
         // First load all the databases from our main database folder
         // TODO: Load any databases defined with explicit storage paths from the config
         for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
@@ -278,7 +341,7 @@ function getDatabases() {
             }
         }
     }
-    if ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.ANALYTICS_REPLICATE) === false) {
+    if ((0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.ANALYTICS_REPLICATE) === false) {
         if (!exports.NON_REPLICATING_SYSTEM_TABLES.includes('hdb_analytics'))
             exports.NON_REPLICATING_SYSTEM_TABLES.push('hdb_analytics');
     }
@@ -304,7 +367,7 @@ function getDatabases() {
  * @param databaseName
  */
 function readMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME, auditPath, isLegacy) {
-    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+    const envInit = new OpenEnvironmentObject_ts_1.default(path, isReadOnlyMode());
     try {
         let rootStore = lmdbDatabaseEnvs.get(path);
         if (rootStore) {
@@ -338,7 +401,10 @@ function readRocksMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAM
             rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true });
             rocksdbDatabaseEnvs.set(path, rootStore);
             initStores(path, rootStore, databaseName, defaultTable);
-            (0, replayLogs_ts_1.replayLogs)(rootStore, exports.databases[databaseName]);
+            // Skip transaction log replay in read-only mode
+            if (!isReadOnlyMode()) {
+                (0, replayLogs_ts_1.replayLogs)(rootStore, exports.databases[databaseName]);
+            }
         }
         return rootStore;
     }
@@ -348,7 +414,7 @@ function readRocksMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAM
     }
 }
 function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLegacy) {
-    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+    const envInit = new OpenEnvironmentObject_ts_1.default(path, isReadOnlyMode());
     const internalDbiInit = createOpenDBIObject(false);
     let attributesDbi = rootStore.dbisDb;
     if (!attributesDbi) {
@@ -356,11 +422,11 @@ function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLe
             attributesDbi = openRocksDatabase(rootStore.path, {
                 ...internalDbiInit,
                 disableWAL: false,
-                name: terms_js_1.INTERNAL_DBIS_NAME,
+                name: terms_ts_1.INTERNAL_DBIS_NAME,
             });
         }
         else {
-            attributesDbi = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+            attributesDbi = rootStore.openDB(terms_ts_1.INTERNAL_DBIS_NAME, internalDbiInit);
         }
         rootStore.dbisDb = attributesDbi;
     }
@@ -440,7 +506,7 @@ function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLe
         let indices = {}, existingAttributes = [];
         let tableId;
         let primaryStore;
-        const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
+        const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
         const trackDeletes = primaryAttribute.trackDeletes;
         const expiration = primaryAttribute.expiration;
         const eviction = primaryAttribute.eviction;
@@ -471,7 +537,7 @@ function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLe
             const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
             dbiInit.compression = primaryAttribute.compression;
             if (dbiInit.compression) {
-                const compressionThreshold = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
+                const compressionThreshold = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
                 dbiInit.compression.threshold = compressionThreshold;
             }
             if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
@@ -620,21 +686,26 @@ function database({ database: databaseName, table: tableName }) {
     if (definedDatabase?.rootStore) {
         return definedDatabase.rootStore;
     }
-    const databaseConfig = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    const databaseConfig = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
     if (process.env.SCHEMAS_DATA_PATH) {
         databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
     }
     const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
-    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
+    const hdbBasePath = (0, environmentManager_ts_1.getHdbBasePath)();
     const databasePath = tablePath ||
         databaseConfig[databaseName]?.path ||
         process.env.STORAGE_PATH ||
         (0, configUtils_js_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
-        ((0, node_fs_1.existsSync)((0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
+        (hdbBasePath && (0, node_fs_1.existsSync)((0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
             ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME)
-            : (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME));
+            : hdbBasePath
+                ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)
+                : undefined);
+    if (!databasePath) {
+        throw new Error(`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`);
+    }
     let rootStore;
-    const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
+    const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
     if (useRocksdb) {
         const path = (0, path_1.join)(databasePath, tablePath ? tableName : databaseName);
         rootStore = rocksdbDatabaseEnvs.get(path);
@@ -651,7 +722,7 @@ function database({ database: databaseName, table: tableName }) {
         rootStore = lmdbDatabaseEnvs.get(path);
         if (!rootStore || rootStore.status === 'closed') {
             // TODO: validate database name
-            const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+            const envInit = new OpenEnvironmentObject_ts_1.default(path, isReadOnlyMode());
             rootStore = (0, lmdb_1.open)(envInit);
             lmdbDatabaseEnvs.set(path, rootStore);
         }
@@ -801,7 +872,7 @@ function table(tableDefinition) {
         primaryKeyAttribute.compression = getDefaultCompression();
         if (trackDeletes)
             primaryKeyAttribute.trackDeletes = true;
-        audit = primaryKeyAttribute.audit = typeof audit === 'boolean' ? audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
+        audit = primaryKeyAttribute.audit = typeof audit === 'boolean' ? audit : (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
         if (expiration)
             primaryKeyAttribute.expiration = expiration;
         if (eviction)
@@ -826,11 +897,11 @@ function table(tableDefinition) {
             attributesDbi = rootStore.dbisDb = openRocksDatabase(rootStore.path, {
                 ...internalDbiInit,
                 disableWAL: false,
-                name: terms_js_1.INTERNAL_DBIS_NAME,
+                name: terms_ts_1.INTERNAL_DBIS_NAME,
             });
         }
         else {
-            attributesDbi = rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+            attributesDbi = rootStore.dbisDb = rootStore.openDB(terms_ts_1.INTERNAL_DBIS_NAME, internalDbiInit);
         }
         exclusiveLock(); // get an exclusive lock on the database so we can verify that we are the only thread creating the table (and assigning the table id)
         if (attributesDbi.getSync(dbiName)) {
@@ -885,11 +956,11 @@ function table(tableDefinition) {
             rootStore.dbisDb = openRocksDatabase(rootStore.path, {
                 ...internalDbiInit,
                 disableWAL: false,
-                name: terms_js_1.INTERNAL_DBIS_NAME,
+                name: terms_ts_1.INTERNAL_DBIS_NAME,
             });
         }
         else {
-            rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+            rootStore.dbisDb = rootStore.openDB(terms_ts_1.INTERNAL_DBIS_NAME, internalDbiInit);
         }
         attributesDbi = rootStore.dbisDb;
     }
@@ -946,7 +1017,7 @@ function table(tableDefinition) {
                     const updatedPrimaryAttribute = { ...attributeDescriptor };
                     if (typeof audit === 'boolean') {
                         if (audit)
-                            Table.enableAuditing(audit);
+                            Table.enableAuditing();
                         updatedPrimaryAttribute.audit = audit;
                     }
                     if (expiration)
@@ -968,27 +1039,47 @@ function table(tableDefinition) {
             // note that non-indexed attributes do not need a dbi
             if (attributeDescriptor?.attribute && !attributeDescriptor.name)
                 attributeDescriptor.indexed = true; // legacy descriptor
-            const changed = !attributeDescriptor ||
+            // Some index options affect only search, not the stored structure (e.g. HNSW's
+            // efConstructionSearch). Changing those should persist the new metadata but NOT trigger a
+            // reindex. A custom index declares such keys via a static `searchOnlyOptions`.
+            const indexType = attribute.indexed && typeof attribute.indexed === 'object' ? attribute.indexed.type : undefined;
+            const searchOnlyOptions = (indexType && customIndexes_ts_1.CUSTOM_INDEXES[indexType]?.searchOnlyOptions) || [];
+            const stripSearchOnly = (indexed) => {
+                if (!indexed || typeof indexed !== 'object' || searchOnlyOptions.length === 0)
+                    return indexed;
+                const copy = { ...indexed };
+                for (const key of searchOnlyOptions)
+                    delete copy[key];
+                return copy;
+            };
+            const commonChanged = !attributeDescriptor ||
                 attributeDescriptor.type !== attribute.type ||
-                JSON.stringify(attributeDescriptor.indexed) !== JSON.stringify(attribute.indexed) ||
                 attributeDescriptor.nullable !== attribute.nullable ||
                 attributeDescriptor.version !== attribute.version ||
                 attributeDescriptor.enumerable !== attribute.enumerable ||
                 JSON.stringify(attributeDescriptor.properties) !== JSON.stringify(attribute.properties) ||
-                JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements);
+                JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements) ||
+                // Include `embed` so a source/model change refreshes the embed registry.
+                JSON.stringify(attributeDescriptor.embed) !== JSON.stringify(attribute.embed);
+            // any metadata difference (drives persistence)
+            const changed = commonChanged || JSON.stringify(attributeDescriptor?.indexed) !== JSON.stringify(attribute.indexed);
+            // structure-affecting difference (drives reindex) — ignores search-only option changes
+            const structurallyChanged = commonChanged ||
+                JSON.stringify(stripSearchOnly(attributeDescriptor?.indexed)) !==
+                    JSON.stringify(stripSearchOnly(attribute.indexed));
             if (attribute.indexed) {
                 const dbi = openIndex(dbiKey, rootStore, attribute);
                 if (changed ||
-                    attributeDescriptor.indexingFailed ||
-                    (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
-                    attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
+                    attributeDescriptor?.indexingFailed ||
+                    (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
+                    attributeDescriptor?.restartNumber < worker_threads_1.workerData?.restartNumber) {
                     hasChanges = true;
                     exclusiveLock();
                     attributeDescriptor = attributesDbi.getSync(dbiKey);
-                    if (changed ||
-                        attributeDescriptor.indexingFailed ||
-                        (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
-                        attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
+                    if (structurallyChanged ||
+                        attributeDescriptor?.indexingFailed ||
+                        (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
+                        attributeDescriptor?.restartNumber < worker_threads_1.workerData?.restartNumber) {
                         hasChanges = true;
                         if (attribute.indexNulls === undefined)
                             attribute.indexNulls = true;
@@ -1007,6 +1098,16 @@ function table(tableDefinition) {
                             attributesToIndex.push(attribute);
                         }
                     }
+                    else if (attributeDescriptor.indexingPID) {
+                        // Metadata-only change (e.g. a search-only option like efConstructionSearch) while a
+                        // backfill is in progress: we did NOT re-trigger indexing, so carry over the in-progress
+                        // indexing state instead of persisting a descriptor that looks complete — otherwise other
+                        // workers / a reload would treat the still-partial index as ready and return incomplete results.
+                        attribute.indexingPID = attributeDescriptor.indexingPID;
+                        attribute.lastIndexedKey = attributeDescriptor.lastIndexedKey;
+                        if (attributeDescriptor.indexingFailed)
+                            attribute.indexingFailed = attributeDescriptor.indexingFailed;
+                    }
                     attributesDbi.put(dbiKey, attribute);
                 }
                 // If a migration is in progress (indexingPID set), any newly opened dbi must also
@@ -1136,7 +1237,7 @@ async function runIndexing(Table, attributes, indicesToRemove) {
                             index.customIndex.index(key, value);
                             continue;
                         }
-                        const values = (0, commonUtility_js_1.getIndexedValues)(value, index.indexNulls);
+                        const values = (0, commonUtility_ts_1.getIndexedValues)(value, index.indexNulls);
                         if (values) {
                             for (let i = 0, l = values.length; i < l; i++) {
                                 lastResolution = index.put(values[i], key);
@@ -1297,9 +1398,9 @@ function onRemovedDB(listener) {
     };
 }
 function getDefaultCompression() {
-    const LMDB_COMPRESSION = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION);
-    const STORAGE_COMPRESSION_DICTIONARY = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_DICTIONARY);
-    const STORAGE_COMPRESSION_THRESHOLD = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD;
+    const LMDB_COMPRESSION = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION);
+    const STORAGE_COMPRESSION_DICTIONARY = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_DICTIONARY);
+    const STORAGE_COMPRESSION_THRESHOLD = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD;
     const LMDB_COMPRESSION_OPTS = { startingOffset: 32 };
     if (STORAGE_COMPRESSION_DICTIONARY)
         LMDB_COMPRESSION_OPTS['dictionary'] = (0, node_fs_1.readFileSync)(STORAGE_COMPRESSION_DICTIONARY);
@@ -1307,4 +1408,11 @@ function getDefaultCompression() {
         LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
     return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
 }
+/**
+ * Force all RocksDB databases to flush to disk.
+ */
+async function flushDatabases() {
+    // flush all RocksDB databases
+    return Promise.all(Array.from(rocksdbDatabaseEnvs.values()).map((db) => db.flush()));
+}
 //# sourceMappingURL=databases.js.map