@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/dist/resources/databases.js

@@ -37,6 +37,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
+exports.isReadOnlyMode = isReadOnlyMode;
 exports.getTables = getTables;
 exports.getDatabases = getDatabases;
 exports.readMetaDb = readMetaDb;
@@ -49,60 +50,97 @@ exports.onUpdatedTable = onUpdatedTable;
 exports.onRemovedTable = onRemovedTable;
 exports.onRemovedDB = onRemovedDB;
 exports.getDefaultCompression = getDefaultCompression;
+exports.flushDatabases = flushDatabases;
 const node_events_1 = require("node:events");
-const environmentManager_js_1 = require("../utility/environment/environmentManager.js");
-const terms_js_1 = require("../utility/lmdb/terms.js");
+const environmentManager_ts_1 = require("../utility/environment/environmentManager.js");
+const terms_ts_1 = require("../utility/lmdb/terms.js");
 const lmdb_1 = require("lmdb");
 const path_1 = require("path");
 const node_fs_1 = require("node:fs");
 const promises_1 = require("node:fs/promises");
 const initializePaths_js_1 = require("../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js");
 const Table_ts_1 = require("./Table.js");
-const OpenEnvironmentObject_js_1 = __importDefault(require("../utility/lmdb/OpenEnvironmentObject.js"));
+const OpenEnvironmentObject_ts_1 = __importDefault(require("../utility/lmdb/OpenEnvironmentObject.js"));
 const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
 const configUtils_js_1 = require("../config/configUtils.js");
 const globals_js_1 = require("../globals.js");
-const commonUtility_js_1 = require("../utility/lmdb/commonUtility.js");
+const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
 const signalling = __importStar(require("../utility/signalling.js"));
 const itc_js_1 = require("../server/threads/itc.js");
 const worker_threads_1 = require("worker_threads");
-const harper_logger_js_1 = __importDefault(require("../utility/logging/harper_logger.js"));
-const { forComponent } = harper_logger_js_1.default;
+const harper_logger_ts_1 = __importDefault(require("../utility/logging/harper_logger.js"));
+const { forComponent } = harper_logger_ts_1.default;
 const manageThreads = __importStar(require("../server/threads/manageThreads.js"));
 const auditStore_ts_1 = require("./auditStore.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
 const blob_ts_1 = require("./blob.js");
 const customIndexes_ts_1 = require("./indexes/customIndexes.js");
-const OpenDBIObject_js_1 = require("../utility/lmdb/OpenDBIObject.js");
+const OpenDBIObject_ts_1 = require("../utility/lmdb/OpenDBIObject.js");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const replayLogs_ts_1 = require("./replayLogs.js");
 const node_os_1 = require("node:os");
 const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
 const when_ts_1 = require("../utility/when.js");
 const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
+/**
+ * Check if Harper is running in read-only mode.
+ * Read-only mode can be enabled via:
+ * - HARPER_READONLY environment variable (truthy value)
+ * - --readonly CLI flag
+ * - storage.readOnly config setting
+ */
+let _isReadOnlyMode;
+function isReadOnlyMode() {
+    if (_isReadOnlyMode !== undefined)
+        return _isReadOnlyMode;
+    // Check environment variable
+    const envReadOnly = process.env.HARPER_READONLY;
+    if (envReadOnly && envReadOnly !== '0' && envReadOnly !== 'false') {
+        _isReadOnlyMode = true;
+        return true;
+    }
+    // Check CLI flag (simple argv check)
+    if (process.argv.includes('--readonly') || process.argv.includes('--read-only')) {
+        _isReadOnlyMode = true;
+        return true;
+    }
+    // Check config setting
+    if ((0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_READONLY)) {
+        _isReadOnlyMode = true;
+        return true;
+    }
+    _isReadOnlyMode = false;
+    return false;
+}
 function createOpenDBIObject(dupSort = false, isPrimary = false) {
-    return new OpenDBIObject_js_1.OpenDBIObject(dupSort, isPrimary);
+    return new OpenDBIObject_ts_1.OpenDBIObject(dupSort, isPrimary);
 }
 const logger = forComponent('storage');
 const DEFAULT_DATABASE_NAME = 'data';
 const DEFINED_TABLES = Symbol('defined-tables');
-const DEFAULT_COMPRESSION_THRESHOLD = ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
-(0, environmentManager_js_1.initSync)();
+const DEFAULT_COMPRESSION_THRESHOLD = ((0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
+(0, environmentManager_ts_1.initSync)();
 // I don't know if this is the best place for this, but somewhere we need to specify which tables
 // replicate by default:
 exports.NON_REPLICATING_SYSTEM_TABLES = [
     'hdb_temp',
     'hdb_certificate',
     'hdb_raw_analytics',
+    'hdb_model_calls',
     'hdb_session_will',
     'hdb_job',
     'hdb_info',
+    'mcp_session',
 ];
 exports.databaseEventsEmitter = new node_events_1.EventEmitter();
 exports.tables = Object.create(null);
 exports.databases = Object.create(null);
 function openRocksDatabase(path, options) {
     options.disableWAL ??= true;
+    // Apply read-only mode if enabled
+    if (isReadOnlyMode()) {
+        options.readOnly = true;
+    }
     // Read RocksDB memory config lazily so env/CLI overrides applied after module load are
     // respected. The block cache falls back to 25% of constrained (cgroup) memory when not
     // configured; the WriteBufferManager is opt-in (0 disables).
@@ -115,13 +153,13 @@ function openRocksDatabase(path, options) {
     // creation time inside rocksdb-js (the underlying RocksDB API doesn't support changing
     // costToCache on a live manager, and allowStall is only re-applied when explicitly changed).
     // In practice that's fine — these come from process-level config that doesn't change.
-    const configuredBlockCacheSize = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE);
+    const configuredBlockCacheSize = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE);
     const blockCacheSize = typeof configuredBlockCacheSize === 'number' && configuredBlockCacheSize > 0
         ? configuredBlockCacheSize
         : Math.min(process.constrainedMemory?.() ?? Infinity, (0, node_os_1.totalmem)()) * 0.25;
-    const writeBufferManagerSize = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE);
-    const writeBufferManagerCostToCache = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE);
-    const writeBufferManagerAllowStall = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL);
+    const writeBufferManagerSize = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE);
+    const writeBufferManagerCostToCache = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE);
+    const writeBufferManagerAllowStall = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL);
     rocksdb_js_1.RocksDatabase.config({
         blockCacheSize,
         ...(typeof writeBufferManagerSize === 'number' && writeBufferManagerSize > 0 ? { writeBufferManagerSize } : {}),
@@ -129,6 +167,10 @@ function openRocksDatabase(path, options) {
         ...(typeof writeBufferManagerAllowStall === 'boolean' ? { writeBufferManagerAllowStall } : {}),
     });
     if (!(0, node_fs_1.existsSync)(path)) {
+        // Don't create directories in read-only mode
+        if (isReadOnlyMode()) {
+            throw new Error(`Database cannot be created in read-only mode: ${path}`);
+        }
         (0, node_fs_1.mkdirSync)(path, { recursive: true });
     }
     let db;
@@ -180,19 +222,17 @@ function getDatabases() {
     }
     loadedDatabases = true;
     definedDatabases = new Map();
-    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
+    const hdbBasePath = (0, environmentManager_ts_1.getHdbBasePath)();
     let databasePath = hdbBasePath && (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME);
-    const schemaConfigs = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    const schemaConfigs = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
     // not sure why this doesn't work with the environmemt manager
     if (process.env.SCHEMAS_DATA_PATH)
         schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
     databasePath =
         process.env.STORAGE_PATH ||
             (0, configUtils_js_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
-            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, path_1.join)((0, environmentManager_js_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
-    if (!databasePath)
-        return;
-    if ((0, node_fs_1.existsSync)(databasePath)) {
+            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, path_1.join)((0, environmentManager_ts_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
+    if (databasePath && (0, node_fs_1.existsSync)(databasePath)) {
         // First load all the databases from our main database folder
         // TODO: Load any databases defined with explicit storage paths from the config
         for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
@@ -301,7 +341,7 @@ function getDatabases() {
             }
         }
     }
-    if ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.ANALYTICS_REPLICATE) === false) {
+    if ((0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.ANALYTICS_REPLICATE) === false) {
         if (!exports.NON_REPLICATING_SYSTEM_TABLES.includes('hdb_analytics'))
             exports.NON_REPLICATING_SYSTEM_TABLES.push('hdb_analytics');
     }
@@ -327,7 +367,7 @@ function getDatabases() {
  * @param databaseName
  */
 function readMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME, auditPath, isLegacy) {
-    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+    const envInit = new OpenEnvironmentObject_ts_1.default(path, isReadOnlyMode());
     try {
         let rootStore = lmdbDatabaseEnvs.get(path);
         if (rootStore) {
@@ -361,7 +401,10 @@ function readRocksMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAM
             rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true });
             rocksdbDatabaseEnvs.set(path, rootStore);
             initStores(path, rootStore, databaseName, defaultTable);
-            (0, replayLogs_ts_1.replayLogs)(rootStore, exports.databases[databaseName]);
+            // Skip transaction log replay in read-only mode
+            if (!isReadOnlyMode()) {
+                (0, replayLogs_ts_1.replayLogs)(rootStore, exports.databases[databaseName]);
+            }
         }
         return rootStore;
     }
@@ -371,7 +414,7 @@ function readRocksMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAM
     }
 }
 function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLegacy) {
-    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+    const envInit = new OpenEnvironmentObject_ts_1.default(path, isReadOnlyMode());
     const internalDbiInit = createOpenDBIObject(false);
     let attributesDbi = rootStore.dbisDb;
     if (!attributesDbi) {
@@ -379,11 +422,11 @@ function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLe
             attributesDbi = openRocksDatabase(rootStore.path, {
                 ...internalDbiInit,
                 disableWAL: false,
-                name: terms_js_1.INTERNAL_DBIS_NAME,
+                name: terms_ts_1.INTERNAL_DBIS_NAME,
             });
         }
         else {
-            attributesDbi = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+            attributesDbi = rootStore.openDB(terms_ts_1.INTERNAL_DBIS_NAME, internalDbiInit);
         }
         rootStore.dbisDb = attributesDbi;
     }
@@ -463,7 +506,7 @@ function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLe
         let indices = {}, existingAttributes = [];
         let tableId;
         let primaryStore;
-        const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
+        const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
         const trackDeletes = primaryAttribute.trackDeletes;
         const expiration = primaryAttribute.expiration;
         const eviction = primaryAttribute.eviction;
@@ -494,9 +537,12 @@ function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLe
             const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
             dbiInit.compression = primaryAttribute.compression;
             if (dbiInit.compression) {
-                const compressionThreshold = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
+                const compressionThreshold = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
                 dbiInit.compression.threshold = compressionThreshold;
             }
+            // per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+            if (typeof primaryAttribute.randomAccessFields === 'boolean')
+                dbiInit.randomAccessStructure = primaryAttribute.randomAccessFields;
             if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
                 primaryStore = (0, RecordEncoder_ts_1.handleLocalTimeForGets)(openRocksDatabase(rootStore.path, { ...dbiInit, name: primaryAttribute.key }), rootStore);
             }
@@ -643,21 +689,26 @@ function database({ database: databaseName, table: tableName }) {
     if (definedDatabase?.rootStore) {
         return definedDatabase.rootStore;
     }
-    const databaseConfig = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    const databaseConfig = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
     if (process.env.SCHEMAS_DATA_PATH) {
         databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
     }
     const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
-    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
+    const hdbBasePath = (0, environmentManager_ts_1.getHdbBasePath)();
     const databasePath = tablePath ||
         databaseConfig[databaseName]?.path ||
         process.env.STORAGE_PATH ||
         (0, configUtils_js_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
-        ((0, node_fs_1.existsSync)((0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
+        (hdbBasePath && (0, node_fs_1.existsSync)((0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
             ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME)
-            : (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME));
+            : hdbBasePath
+                ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)
+                : undefined);
+    if (!databasePath) {
+        throw new Error(`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`);
+    }
     let rootStore;
-    const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
+    const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
     if (useRocksdb) {
         const path = (0, path_1.join)(databasePath, tablePath ? tableName : databaseName);
         rootStore = rocksdbDatabaseEnvs.get(path);
@@ -674,7 +725,7 @@ function database({ database: databaseName, table: tableName }) {
         rootStore = lmdbDatabaseEnvs.get(path);
         if (!rootStore || rootStore.status === 'closed') {
             // TODO: validate database name
-            const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+            const envInit = new OpenEnvironmentObject_ts_1.default(path, isReadOnlyMode());
             rootStore = (0, lmdb_1.open)(envInit);
             lmdbDatabaseEnvs.set(path, rootStore);
         }
@@ -741,11 +792,13 @@ async function dropDatabase(databaseName) {
 function openIndex(dbiKey, rootStore, attribute) {
     const objectStorage = attribute.isPrimaryKey || (attribute.indexed.type && customIndexes_ts_1.CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore);
     const dbiInit = createOpenDBIObject(!objectStorage, objectStorage);
-    // Custom-index object stores (e.g. HNSW vector graphs) must keep writing typed structs regardless
-    // of the storage.randomAccessFields opt-out — their internal nodes are mutated in place and depend
-    // on random-access struct encoding (see IndexRecordEncoder).
+    // Custom-index object stores (e.g. HNSW vector graphs) hold fixed-shape internal nodes —
+    // numeric-keyed per-level connection arrays and quantized bins — that rely on random-access
+    // struct encoding. Keep them in struct mode regardless of the table's storage.randomAccessFields
+    // setting: their node shapes are controlled, so the wide/variably-typed OOM + divergence risks
+    // that motivate the table-level default-off don't apply, and disabling structs corrupts the graph.
     if (attribute.indexed?.type && customIndexes_ts_1.CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore) {
-        dbiInit.encoder = { Encoder: RecordEncoder_ts_1.IndexRecordEncoder };
+        dbiInit.randomAccessStructure = true;
     }
     let dbi;
     if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
@@ -781,7 +834,7 @@ function openIndex(dbiKey, rootStore, attribute) {
  * @param replicate
  */
 function table(tableDefinition) {
-    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, trackDeletes, schemaDefined, origin, } = tableDefinition;
+    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, randomAccessFields, trackDeletes, schemaDefined, origin, description, properties, hidden, } = tableDefinition;
     if (!databaseName)
         databaseName = DEFAULT_DATABASE_NAME;
     const rootStore = database({ database: databaseName, table: tableName });
@@ -819,18 +872,23 @@ function table(tableDefinition) {
         if (splitSegments == undefined)
             splitSegments = Table.splitSegments;
         Table.attributes.splice(0, Table.attributes.length, ...attributes);
+        // Refresh class-level schema metadata to track docstring/directive changes across reloads.
+        Table.description = description;
+        Table.properties = properties;
+        Table.hidden = hidden;
     }
     else {
         const auditStore = rootStore.auditStore;
         primaryKeyAttribute = attributes.find((attribute) => attribute.isPrimaryKey) || {};
         primaryKey = primaryKeyAttribute.name;
         primaryKeyAttribute.isPrimaryKey = true;
+        primaryKeyAttribute.is_hash_attribute = true; // backward-compat: harperdb@4.x reads this field to open the DBI with correct flags
         primaryKeyAttribute.schemaDefined = schemaDefined;
         // can't change compression after the fact (except threshold), so save only when we create the table
         primaryKeyAttribute.compression = getDefaultCompression();
         if (trackDeletes)
             primaryKeyAttribute.trackDeletes = true;
-        audit = primaryKeyAttribute.audit = typeof audit === 'boolean' ? audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
+        audit = primaryKeyAttribute.audit = typeof audit === 'boolean' ? audit : (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
         if (expiration)
             primaryKeyAttribute.expiration = expiration;
         if (eviction)
@@ -841,6 +899,14 @@ function table(tableDefinition) {
             primaryKeyAttribute.sealed = sealed;
         if (typeof replicate === 'boolean')
             primaryKeyAttribute.replicate = replicate;
+        // An explicit directive PINS this table's encoding: we persist the boolean, so later changes
+        // to the global storage.randomAccessFields default never affect this table. Tables WITHOUT the
+        // directive are intentionally not persisted here — they follow the current global default on
+        // each open (a runtime lever to flip encoding fleet-wide). Switching either way is safe: the
+        // struct READ hook always stays on and struct (0x20-0x3f) vs classic-record (0x40-0x7f) bytes
+        // are disjoint, so already-written records still decode; only the encoding of NEW writes changes.
+        if (typeof randomAccessFields === 'boolean')
+            primaryKeyAttribute.randomAccessFields = randomAccessFields;
         if (origin) {
             if (!primaryKeyAttribute.origins)
                 primaryKeyAttribute.origins = [origin];
@@ -850,16 +916,19 @@ function table(tableDefinition) {
         logger.trace(`${tableName} table loading, opening primary store`);
         const dbiInit = createOpenDBIObject(false, true);
         dbiInit.compression = primaryKeyAttribute.compression;
+        // per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+        if (typeof primaryKeyAttribute.randomAccessFields === 'boolean')
+            dbiInit.randomAccessStructure = primaryKeyAttribute.randomAccessFields;
         const dbiName = tableName + '/';
         if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
             attributesDbi = rootStore.dbisDb = openRocksDatabase(rootStore.path, {
                 ...internalDbiInit,
                 disableWAL: false,
-                name: terms_js_1.INTERNAL_DBIS_NAME,
+                name: terms_ts_1.INTERNAL_DBIS_NAME,
             });
         }
         else {
-            attributesDbi = rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+            attributesDbi = rootStore.dbisDb = rootStore.openDB(terms_ts_1.INTERNAL_DBIS_NAME, internalDbiInit);
         }
         exclusiveLock(); // get an exclusive lock on the database so we can verify that we are the only thread creating the table (and assigning the table id)
         if (attributesDbi.getSync(dbiName)) {
@@ -903,6 +972,9 @@ function table(tableDefinition) {
             attributes,
             schemaDefined,
             dbisDB: attributesDbi,
+            description,
+            properties,
+            hidden,
         }));
         Table.schemaVersion = 1;
         hasChanges = true;
@@ -914,11 +986,11 @@ function table(tableDefinition) {
             rootStore.dbisDb = openRocksDatabase(rootStore.path, {
                 ...internalDbiInit,
                 disableWAL: false,
-                name: terms_js_1.INTERNAL_DBIS_NAME,
+                name: terms_ts_1.INTERNAL_DBIS_NAME,
             });
         }
         else {
-            rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+            rootStore.dbisDb = rootStore.openDB(terms_ts_1.INTERNAL_DBIS_NAME, internalDbiInit);
         }
         attributesDbi = rootStore.dbisDb;
     }
@@ -975,7 +1047,7 @@ function table(tableDefinition) {
                     const updatedPrimaryAttribute = { ...attributeDescriptor };
                     if (typeof audit === 'boolean') {
                         if (audit)
-                            Table.enableAuditing(audit);
+                            Table.enableAuditing();
                         updatedPrimaryAttribute.audit = audit;
                     }
                     if (expiration)
@@ -997,27 +1069,47 @@ function table(tableDefinition) {
             // note that non-indexed attributes do not need a dbi
             if (attributeDescriptor?.attribute && !attributeDescriptor.name)
                 attributeDescriptor.indexed = true; // legacy descriptor
-            const changed = !attributeDescriptor ||
+            // Some index options affect only search, not the stored structure (e.g. HNSW's
+            // efConstructionSearch). Changing those should persist the new metadata but NOT trigger a
+            // reindex. A custom index declares such keys via a static `searchOnlyOptions`.
+            const indexType = attribute.indexed && typeof attribute.indexed === 'object' ? attribute.indexed.type : undefined;
+            const searchOnlyOptions = (indexType && customIndexes_ts_1.CUSTOM_INDEXES[indexType]?.searchOnlyOptions) || [];
+            const stripSearchOnly = (indexed) => {
+                if (!indexed || typeof indexed !== 'object' || searchOnlyOptions.length === 0)
+                    return indexed;
+                const copy = { ...indexed };
+                for (const key of searchOnlyOptions)
+                    delete copy[key];
+                return copy;
+            };
+            const commonChanged = !attributeDescriptor ||
                 attributeDescriptor.type !== attribute.type ||
-                JSON.stringify(attributeDescriptor.indexed) !== JSON.stringify(attribute.indexed) ||
                 attributeDescriptor.nullable !== attribute.nullable ||
                 attributeDescriptor.version !== attribute.version ||
                 attributeDescriptor.enumerable !== attribute.enumerable ||
                 JSON.stringify(attributeDescriptor.properties) !== JSON.stringify(attribute.properties) ||
-                JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements);
+                JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements) ||
+                // Include `embed` so a source/model change refreshes the embed registry.
+                JSON.stringify(attributeDescriptor.embed) !== JSON.stringify(attribute.embed);
+            // any metadata difference (drives persistence)
+            const changed = commonChanged || JSON.stringify(attributeDescriptor?.indexed) !== JSON.stringify(attribute.indexed);
+            // structure-affecting difference (drives reindex) — ignores search-only option changes
+            const structurallyChanged = commonChanged ||
+                JSON.stringify(stripSearchOnly(attributeDescriptor?.indexed)) !==
+                    JSON.stringify(stripSearchOnly(attribute.indexed));
             if (attribute.indexed) {
                 const dbi = openIndex(dbiKey, rootStore, attribute);
                 if (changed ||
-                    attributeDescriptor.indexingFailed ||
-                    (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
-                    attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
+                    attributeDescriptor?.indexingFailed ||
+                    (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
+                    attributeDescriptor?.restartNumber < worker_threads_1.workerData?.restartNumber) {
                     hasChanges = true;
                     exclusiveLock();
                     attributeDescriptor = attributesDbi.getSync(dbiKey);
-                    if (changed ||
-                        attributeDescriptor.indexingFailed ||
-                        (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
-                        attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
+                    if (structurallyChanged ||
+                        attributeDescriptor?.indexingFailed ||
+                        (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
+                        attributeDescriptor?.restartNumber < worker_threads_1.workerData?.restartNumber) {
                         hasChanges = true;
                         if (attribute.indexNulls === undefined)
                             attribute.indexNulls = true;
@@ -1036,6 +1128,16 @@ function table(tableDefinition) {
                             attributesToIndex.push(attribute);
                         }
                     }
+                    else if (attributeDescriptor.indexingPID) {
+                        // Metadata-only change (e.g. a search-only option like efConstructionSearch) while a
+                        // backfill is in progress: we did NOT re-trigger indexing, so carry over the in-progress
+                        // indexing state instead of persisting a descriptor that looks complete — otherwise other
+                        // workers / a reload would treat the still-partial index as ready and return incomplete results.
+                        attribute.indexingPID = attributeDescriptor.indexingPID;
+                        attribute.lastIndexedKey = attributeDescriptor.lastIndexedKey;
+                        if (attributeDescriptor.indexingFailed)
+                            attribute.indexingFailed = attributeDescriptor.indexingFailed;
+                    }
                     attributesDbi.put(dbiKey, attribute);
                 }
                 // If a migration is in progress (indexingPID set), any newly opened dbi must also
@@ -1150,6 +1252,10 @@ async function runIndexing(Table, attributes, indicesToRemove) {
                 // TODO: Do we ever need to interrupt due to a schema change that was not a restart?
                 //if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up
                 outstanding++;
+                // Custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`, so the
+                // outstanding-based yield below never fires for them. Track that this row did synchronous
+                // indexing work so we can still yield the event loop after it.
+                let didSynchronousIndexing = false;
                 // every index operation needs to be guarded by the version still be the same. If it has already changed before
                 // we index, that's fine because indexing is idempotent, we can just put the same values again. If it changes
                 // during the indexing, the indexing here will fail. This is also fine because it means the other thread will have
@@ -1163,9 +1269,10 @@ async function runIndexing(Table, attributes, indicesToRemove) {
                         const value = record && (resolver ? resolver(record) : record[property]);
                         if (index.customIndex) {
                             index.customIndex.index(key, value);
+                            didSynchronousIndexing = true;
                             continue;
                         }
-                        const values = (0, commonUtility_js_1.getIndexedValues)(value, index.indexNulls);
+                        const values = (0, commonUtility_ts_1.getIndexedValues)(value, index.indexNulls);
                         if (values) {
                             for (let i = 0, l = values.length; i < l; i++) {
                                 lastResolution = index.put(values[i], key);
@@ -1202,6 +1309,8 @@ async function runIndexing(Table, attributes, indicesToRemove) {
                     await lastResolution;
                 else if (outstanding > MIN_OUTSTANDING_INDEXING)
                     await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
+                else if (didSynchronousIndexing)
+                    await new Promise((resolve) => setImmediate(resolve)); // custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`; without this yield a large backfill runs in a single event-loop turn, starving keepalive/replication and queries and never letting the isIndexing flag be observed
             }
         }
         // Await the last pending put. If it rejects, that is also an indexing error.
@@ -1326,9 +1435,9 @@ function onRemovedDB(listener) {
     };
 }
 function getDefaultCompression() {
-    const LMDB_COMPRESSION = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION);
-    const STORAGE_COMPRESSION_DICTIONARY = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_DICTIONARY);
-    const STORAGE_COMPRESSION_THRESHOLD = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD;
+    const LMDB_COMPRESSION = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION);
+    const STORAGE_COMPRESSION_DICTIONARY = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_DICTIONARY);
+    const STORAGE_COMPRESSION_THRESHOLD = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD;
     const LMDB_COMPRESSION_OPTS = { startingOffset: 32 };
     if (STORAGE_COMPRESSION_DICTIONARY)
         LMDB_COMPRESSION_OPTS['dictionary'] = (0, node_fs_1.readFileSync)(STORAGE_COMPRESSION_DICTIONARY);
@@ -1336,4 +1445,11 @@ function getDefaultCompression() {
         LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
     return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
 }
+/**
+ * Force all RocksDB databases to flush to disk.
+ */
+async function flushDatabases() {
+    // flush all RocksDB databases
+    return Promise.all(Array.from(rocksdbDatabaseEnvs.values()).map((db) => db.flush()));
+}
 //# sourceMappingURL=databases.js.map