@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/resources/databases.ts

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'node:events';
-import { initSync, getHdbBasePath, get as envGet } from '../utility/environment/environmentManager.js';
-import { INTERNAL_DBIS_NAME } from '../utility/lmdb/terms.js';
+import { initSync, getHdbBasePath, get as envGet } from '../utility/environment/environmentManager.ts';
+import { INTERNAL_DBIS_NAME } from '../utility/lmdb/terms.ts';
 import { open, compareKeys, type Database, type RootDatabase } from 'lmdb';
 import { join, extname, basename } from 'path';
 import { existsSync, readdirSync, readFileSync, mkdirSync } from 'node:fs';
@@ -10,22 +10,22 @@ import {
 	getTransactionAuditStoreBasePath,
 } from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js';
 import { makeTable } from './Table.ts';
-import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
+import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts';
 import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
 import { getConfigPath } from '../config/configUtils.js';
 import { _assignPackageExport } from '../globals.js';
-import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
-import * as signalling from '../utility/signalling.js';
+import { getIndexedValues } from '../utility/lmdb/commonUtility.ts';
+import * as signalling from '../utility/signalling.ts';
 import { SchemaEventMsg } from '../server/threads/itc.js';
 import { workerData } from 'worker_threads';
-import harperLogger from '../utility/logging/harper_logger.js';
+import harperLogger from '../utility/logging/harper_logger.ts';
 const { forComponent } = harperLogger;
 import * as manageThreads from '../server/threads/manageThreads.js';
 import { openAuditStore, readAuditEntry, createAuditEntry, type AuditRecord } from './auditStore.ts';
-import { handleLocalTimeForGets, IndexRecordEncoder } from './RecordEncoder.ts';
+import { handleLocalTimeForGets } from './RecordEncoder.ts';
 import { deleteRootBlobPathsForDB } from './blob.ts';
 import { CUSTOM_INDEXES } from './indexes/customIndexes.ts';
-import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.js';
+import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.ts';
 import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
 import { replayLogs } from './replayLogs.ts';
 import { totalmem } from 'node:os';
@@ -33,6 +33,36 @@ import { RocksIndexStore } from './RocksIndexStore.ts';
 import { when } from '../utility/when.ts';
 import { isProcessRunning } from '../utility/processManagement/processManagement.js';
 
+/**
+ * Check if Harper is running in read-only mode.
+ * Read-only mode can be enabled via:
+ * - HARPER_READONLY environment variable (truthy value)
+ * - --readonly CLI flag
+ * - storage.readOnly config setting
+ */
+let _isReadOnlyMode: boolean | undefined;
+export function isReadOnlyMode(): boolean {
+	if (_isReadOnlyMode !== undefined) return _isReadOnlyMode;
+	// Check environment variable
+	const envReadOnly = process.env.HARPER_READONLY;
+	if (envReadOnly && envReadOnly !== '0' && envReadOnly !== 'false') {
+		_isReadOnlyMode = true;
+		return true;
+	}
+	// Check CLI flag (simple argv check)
+	if (process.argv.includes('--readonly') || process.argv.includes('--read-only')) {
+		_isReadOnlyMode = true;
+		return true;
+	}
+	// Check config setting
+	if (envGet(CONFIG_PARAMS.STORAGE_READONLY)) {
+		_isReadOnlyMode = true;
+		return true;
+	}
+	_isReadOnlyMode = false;
+	return false;
+}
+
 function createOpenDBIObject(dupSort = false, isPrimary = false) {
 	return new OpenDBIObject(dupSort, isPrimary);
 }
@@ -48,9 +78,11 @@ export const NON_REPLICATING_SYSTEM_TABLES = [
 	'hdb_temp',
 	'hdb_certificate',
 	'hdb_raw_analytics',
+	'hdb_model_calls',
 	'hdb_session_will',
 	'hdb_job',
 	'hdb_info',
+	'mcp_session',
 ];
 
 export type Table = ReturnType<typeof makeTable> & {
@@ -80,6 +112,10 @@ interface LMDBRootDatabase extends RootDatabase {
 	needsDeletion?: boolean;
 	path?: string;
 	status?: 'open' | 'closed';
+	store: any;
+	retryRisk?: number;
+	flushed: Promise<boolean>;
+	rootStore?: LMDBRootDatabase;
 }
 
 interface RocksDatabaseEx extends RocksDatabase {
@@ -95,6 +131,10 @@ interface RocksRootDatabase extends RocksDatabaseEx {
 	auditStore?: RocksDatabaseEx;
 	databaseName?: string;
 	dbisDb?: RocksDatabaseEx;
+	store: any;
+	retryRisk?: number;
+	flushed: Promise<boolean>;
+	rootStore?: RocksRootDatabase;
 }
 
 export type RootDatabaseKind = LMDBRootDatabase | RocksRootDatabase;
@@ -112,6 +152,10 @@ export const databases: Databases = Object.create(null);
 
 function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) {
 	options.disableWAL ??= true;
+	// Apply read-only mode if enabled
+	if (isReadOnlyMode()) {
+		options.readOnly = true;
+	}
 	// Read RocksDB memory config lazily so env/CLI overrides applied after module load are
 	// respected. The block cache falls back to 25% of constrained (cgroup) memory when not
 	// configured; the WriteBufferManager is opt-in (0 disables).
@@ -139,25 +183,29 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo
 		...(typeof writeBufferManagerAllowStall === 'boolean' ? { writeBufferManagerAllowStall } : {}),
 	});
 	if (!existsSync(path)) {
+		// Don't create directories in read-only mode
+		if (isReadOnlyMode()) {
+			throw new Error(`Database cannot be created in read-only mode: ${path}`);
+		}
 		mkdirSync(path, { recursive: true });
 	}
 	let db: RocksRootDatabase;
 	if (options.dupSort) {
-		db = new RocksIndexStore(path, options).open() as RocksDatabaseEx;
+		db = new RocksIndexStore(path, options).open() as any;
 	} else {
-		db = RocksDatabase.open(path, options) as RocksDatabaseEx;
+		db = RocksDatabase.open(path, options) as any;
 		// the RocksDB put and remove return promises, which masks thrown errors in non-awaiting calls to put/remove,
 		// making them unsafe to replace LMDB methods, which will synchronously throw errors if there is a problem
-		db.put = db.putSync;
-		db.remove = db.removeSync;
-		db.encoder.name = options.name;
+		db.put = db.putSync as any;
+		db.remove = db.removeSync as any;
+		(db.encoder as any).name = options.name;
 	}
 	db.env = {};
 	return db;
 }
 
 const lmdbDatabaseEnvs = new Map<string, LMDBRootDatabase>();
-const rocksdbDatabaseEnvs = new Map<string, RocksDatabaseEx>();
+const rocksdbDatabaseEnvs = new Map<string, RocksRootDatabase>();
 
 // set the following in both global and exports
 _assignPackageExport('databases', databases);
@@ -206,9 +254,7 @@ export function getDatabases(): Databases {
 		process.env.STORAGE_PATH ||
 		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
 		(databasePath && (existsSync(databasePath) ? databasePath : join(getHdbBasePath(), LEGACY_DATABASES_DIR_NAME)));
-	if (!databasePath) return;
-
-	if (existsSync(databasePath)) {
+	if (databasePath && existsSync(databasePath)) {
 		// First load all the databases from our main database folder
 		// TODO: Load any databases defined with explicit storage paths from the config
 		for (const databaseEntry of readdirSync(databasePath, { withFileTypes: true })) {
@@ -359,13 +405,13 @@ export function readMetaDb(
 	auditPath?: string,
 	isLegacy?: boolean
 ) {
-	const envInit = new OpenEnvironmentObject(path, false);
+	const envInit = new OpenEnvironmentObject(path, isReadOnlyMode());
 	try {
 		let rootStore = lmdbDatabaseEnvs.get(path);
 		if (rootStore) {
 			rootStore.needsDeletion = false;
 		} else {
-			rootStore = open(envInit);
+			rootStore = open(envInit) as any;
 			lmdbDatabaseEnvs.set(path, rootStore);
 		}
 
@@ -387,14 +433,17 @@ function readRocksMetaDb(path: string, defaultTable?: string, databaseName: stri
 			}
 		}
 
-		let rootStore: RocksDatabaseEx | undefined = rocksdbDatabaseEnvs.get(path);
+		let rootStore: RocksRootDatabase | undefined = rocksdbDatabaseEnvs.get(path);
 		if (rootStore) {
 			initStores(path, rootStore, databaseName, defaultTable);
 		} else {
-			rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true }) as RocksDatabaseEx;
+			rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true }) as any;
 			rocksdbDatabaseEnvs.set(path, rootStore);
 			initStores(path, rootStore, databaseName, defaultTable);
-			replayLogs(rootStore, databases[databaseName]);
+			// Skip transaction log replay in read-only mode
+			if (!isReadOnlyMode()) {
+				replayLogs(rootStore, databases[databaseName]);
+			}
 		}
 		return rootStore;
 	} catch (error) {
@@ -411,7 +460,7 @@ function initStores(
 	auditPath?: string,
 	isLegacy?: boolean
 ) {
-	const envInit = new OpenEnvironmentObject(path, false);
+	const envInit = new OpenEnvironmentObject(path, isReadOnlyMode());
 	const internalDbiInit = createOpenDBIObject(false);
 	let attributesDbi = rootStore.dbisDb;
 	if (!attributesDbi) {
@@ -420,9 +469,9 @@ function initStores(
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
-			}) as RocksDatabaseEx;
+			} as any) as RocksDatabaseEx;
 		} else {
-			attributesDbi = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
+			attributesDbi = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit as any);
 		}
 		rootStore.dbisDb = attributesDbi;
 	}
@@ -441,7 +490,7 @@ function initStores(
 							encode: (auditRecord: AuditRecord) => createAuditEntry(auditRecord),
 							decode: (encoding: Buffer) => readAuditEntry(encoding),
 						},
-					});
+					}) as any;
 				}
 				auditStore.isLegacy = true;
 			}
@@ -452,7 +501,7 @@ function initStores(
 
 	const tables = ensureDB(databaseName);
 	const definedTables = tables[DEFINED_TABLES];
-	definedTables.rootStore = rootStore;
+	(definedTables as any).rootStore = rootStore;
 	const tablesToLoad = new Map<string, any>();
 
 	for (const result of attributesDbi.getRange({ start: false })) {
@@ -519,16 +568,16 @@ function initStores(
 		} else {
 			tableId = primaryAttribute.tableId;
 			if (tableId) {
-				if (tableId >= (attributesDbi.getSync(NEXT_TABLE_ID) || 0)) {
-					attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
+				if (tableId >= ((attributesDbi as any).getSync(NEXT_TABLE_ID) || 0)) {
+					(attributesDbi as any).putSync(NEXT_TABLE_ID, tableId + 1);
 					logger.info(`Updating next table id (it was out of sync) to ${tableId + 1} for ${tableName}`);
 				}
 			} else {
-				primaryAttribute.tableId = tableId = attributesDbi.getSync(NEXT_TABLE_ID);
+				primaryAttribute.tableId = tableId = (attributesDbi as any).getSync(NEXT_TABLE_ID);
 				if (!tableId) tableId = 1;
 				logger.debug(`Table {tableName} missing an id, assigning {tableId}`);
-				attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
-				attributesDbi.putSync(primaryAttribute.key, primaryAttribute);
+				(attributesDbi as any).putSync(NEXT_TABLE_ID, tableId + 1);
+				(attributesDbi as any).putSync(primaryAttribute.key, primaryAttribute);
 			}
 			const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
 			dbiInit.compression = primaryAttribute.compression;
@@ -537,13 +586,19 @@ function initStores(
 					envGet(CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
 				dbiInit.compression.threshold = compressionThreshold;
 			}
+			// per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+			if (typeof primaryAttribute.randomAccessFields === 'boolean')
+				dbiInit.randomAccessStructure = primaryAttribute.randomAccessFields;
 			if (rootStore instanceof RocksDatabase) {
 				primaryStore = handleLocalTimeForGets(
-					openRocksDatabase(rootStore.path, { ...dbiInit, name: primaryAttribute.key }),
+					openRocksDatabase(rootStore.path, { ...dbiInit, name: primaryAttribute.key } as any),
 					rootStore
 				);
 			} else {
-				primaryStore = handleLocalTimeForGets(rootStore.openDB(primaryAttribute.key, dbiInit), rootStore);
+				primaryStore = handleLocalTimeForGets(
+					(rootStore as any).openDB(primaryAttribute.key, dbiInit as any),
+					rootStore
+				);
 			}
 			rootStore.databaseName = databaseName;
 			primaryStore.tableId = tableId;
@@ -658,10 +713,14 @@ interface TableDefinition {
 	sealed?: boolean;
 	splitSegments?: boolean;
 	replicate?: boolean;
+	randomAccessFields?: boolean;
 	trackDeletes?: boolean;
 	attributes: any[];
 	schemaDefined?: boolean;
 	origin?: string;
+	description?: string;
+	properties?: Record<string, any>;
+	hidden?: boolean;
 }
 /**
  * Ensure that we have this database object (that holds a set of tables) set up
@@ -712,8 +771,8 @@ export function database({ database: databaseName, table: tableName }) {
 	getDatabases();
 	ensureDB(databaseName);
 	const definedDatabase = definedDatabases.get(databaseName);
-	if (definedDatabase?.rootStore) {
-		return definedDatabase.rootStore;
+	if ((definedDatabase as any)?.rootStore) {
+		return (definedDatabase as any).rootStore;
 	}
 	const databaseConfig = envGet(CONFIG_PARAMS.DATABASES) || {};
 	if (process.env.SCHEMAS_DATA_PATH) {
@@ -728,9 +787,17 @@ export function database({ database: databaseName, table: tableName }) {
 		databaseConfig[databaseName]?.path ||
 		process.env.STORAGE_PATH ||
 		getConfigPath(CONFIG_PARAMS.STORAGE_PATH) ||
-		(existsSync(join(hdbBasePath, DATABASES_DIR_NAME))
+		(hdbBasePath && existsSync(join(hdbBasePath, DATABASES_DIR_NAME))
 			? join(hdbBasePath, DATABASES_DIR_NAME)
-			: join(hdbBasePath, LEGACY_DATABASES_DIR_NAME));
+			: hdbBasePath
+				? join(hdbBasePath, LEGACY_DATABASES_DIR_NAME)
+				: undefined);
+
+	if (!databasePath) {
+		throw new Error(
+			`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`
+		);
+	}
 
 	let rootStore: RootDatabaseKind;
 	const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || envGet(CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
@@ -741,23 +808,23 @@ export function database({ database: databaseName, table: tableName }) {
 			rootStore = openRocksDatabase(path, {
 				disableWAL: false,
 				enableStats: true,
-			});
-			rocksdbDatabaseEnvs.set(path, rootStore);
+			}) as any;
+			rocksdbDatabaseEnvs.set(path, rootStore as any);
 		}
 	} else {
 		const path = join(databasePath, `${tablePath ? tableName : databaseName}.mdb`);
 		rootStore = lmdbDatabaseEnvs.get(path);
 		if (!rootStore || rootStore.status === 'closed') {
 			// TODO: validate database name
-			const envInit = new OpenEnvironmentObject(path, false);
-			rootStore = open(envInit);
-			lmdbDatabaseEnvs.set(path, rootStore);
+			const envInit = new OpenEnvironmentObject(path, isReadOnlyMode());
+			rootStore = open(envInit) as any;
+			lmdbDatabaseEnvs.set(path, rootStore as any);
 		}
 	}
 	if (!rootStore.auditStore) {
-		rootStore.auditStore = openAuditStore(rootStore);
+		rootStore.auditStore = openAuditStore(rootStore as any);
 	}
-	if (definedDatabase) definedDatabase.rootStore = rootStore;
+	if (definedDatabase) (definedDatabase as any).rootStore = rootStore;
 	return rootStore;
 }
 /**
@@ -818,11 +885,13 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any)
 	const objectStorage =
 		attribute.isPrimaryKey || (attribute.indexed.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore);
 	const dbiInit = createOpenDBIObject(!objectStorage, objectStorage);
-	// Custom-index object stores (e.g. HNSW vector graphs) must keep writing typed structs regardless
-	// of the storage.randomAccessFields opt-out — their internal nodes are mutated in place and depend
-	// on random-access struct encoding (see IndexRecordEncoder).
+	// Custom-index object stores (e.g. HNSW vector graphs) hold fixed-shape internal nodes —
+	// numeric-keyed per-level connection arrays and quantized bins — that rely on random-access
+	// struct encoding. Keep them in struct mode regardless of the table's storage.randomAccessFields
+	// setting: their node shapes are controlled, so the wide/variably-typed OOM + divergence risks
+	// that motivate the table-level default-off don't apply, and disabling structs corrupts the graph.
 	if (attribute.indexed?.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore) {
-		dbiInit.encoder = { Encoder: IndexRecordEncoder };
+		dbiInit.randomAccessStructure = true;
 	}
 	let dbi:
 		| LMDBDatabase
@@ -833,10 +902,10 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any)
 				rootStore?: RocksRootDatabase;
 		  });
 	if (rootStore instanceof RocksDatabase) {
-		dbi = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiKey });
-		dbi.rootStore = rootStore;
+		dbi = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiKey } as any) as any;
+		(dbi as any).rootStore = rootStore;
 	} else {
-		dbi = rootStore.openDB(dbiKey, dbiInit);
+		dbi = (rootStore as any).openDB(dbiKey, dbiInit as any);
 	}
 	if (attribute.indexed.type) {
 		const CustomIndex = CUSTOM_INDEXES[attribute.indexed.type];
@@ -875,9 +944,13 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		sealed,
 		splitSegments,
 		replicate,
+		randomAccessFields,
 		trackDeletes,
 		schemaDefined,
 		origin,
+		description,
+		properties,
+		hidden,
 	} = tableDefinition;
 	if (!databaseName) databaseName = DEFAULT_DATABASE_NAME;
 	const rootStore = database({ database: databaseName, table: tableName });
@@ -911,11 +984,16 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		// it table already exists, get the split segments setting
 		if (splitSegments == undefined) splitSegments = Table.splitSegments;
 		Table.attributes.splice(0, Table.attributes.length, ...attributes);
+		// Refresh class-level schema metadata to track docstring/directive changes across reloads.
+		Table.description = description;
+		Table.properties = properties;
+		Table.hidden = hidden;
 	} else {
 		const auditStore = rootStore.auditStore;
 		primaryKeyAttribute = attributes.find((attribute) => attribute.isPrimaryKey) || {};
 		primaryKey = primaryKeyAttribute.name;
 		primaryKeyAttribute.isPrimaryKey = true;
+		primaryKeyAttribute.is_hash_attribute = true; // backward-compat: harperdb@4.x reads this field to open the DBI with correct flags
 		primaryKeyAttribute.schemaDefined = schemaDefined;
 		// can't change compression after the fact (except threshold), so save only when we create the table
 		primaryKeyAttribute.compression = getDefaultCompression();
@@ -927,6 +1005,13 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		primaryKeyAttribute.splitSegments = splitSegments; // always default to not splitting segments going forward
 		if (typeof sealed === 'boolean') primaryKeyAttribute.sealed = sealed;
 		if (typeof replicate === 'boolean') primaryKeyAttribute.replicate = replicate;
+		// An explicit directive PINS this table's encoding: we persist the boolean, so later changes
+		// to the global storage.randomAccessFields default never affect this table. Tables WITHOUT the
+		// directive are intentionally not persisted here — they follow the current global default on
+		// each open (a runtime lever to flip encoding fleet-wide). Switching either way is safe: the
+		// struct READ hook always stays on and struct (0x20-0x3f) vs classic-record (0x40-0x7f) bytes
+		// are disjoint, so already-written records still decode; only the encoding of NEW writes changes.
+		if (typeof randomAccessFields === 'boolean') primaryKeyAttribute.randomAccessFields = randomAccessFields;
 		if (origin) {
 			if (!primaryKeyAttribute.origins) primaryKeyAttribute.origins = [origin];
 			else if (!primaryKeyAttribute.origins.includes(origin)) primaryKeyAttribute.origins.push(origin);
@@ -934,20 +1019,23 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		logger.trace(`${tableName} table loading, opening primary store`);
 		const dbiInit = createOpenDBIObject(false, true);
 		dbiInit.compression = primaryKeyAttribute.compression;
+		// per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+		if (typeof primaryKeyAttribute.randomAccessFields === 'boolean')
+			dbiInit.randomAccessStructure = primaryKeyAttribute.randomAccessFields;
 		const dbiName = tableName + '/';
 
 		if (rootStore instanceof RocksDatabase) {
-			attributesDbi = rootStore.dbisDb = openRocksDatabase(rootStore.path, {
+			attributesDbi = (rootStore as any).dbisDb = openRocksDatabase(rootStore.path, {
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
-			});
+			} as any);
 		} else {
-			attributesDbi = rootStore.dbisDb = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
+			attributesDbi = (rootStore as any).dbisDb = (rootStore as any).openDB(INTERNAL_DBIS_NAME, internalDbiInit as any);
 		}
 
 		exclusiveLock(); // get an exclusive lock on the database so we can verify that we are the only thread creating the table (and assigning the table id)
-		if (attributesDbi.getSync(dbiName)) {
+		if ((attributesDbi as any).getSync(dbiName)) {
 			// table was created while we were setting up
 			if (releaseExclusiveLock) releaseExclusiveLock();
 			resetDatabases();
@@ -956,9 +1044,9 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 
 		let primaryStore;
 		if (rootStore instanceof RocksDatabase) {
-			primaryStore = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiName });
+			primaryStore = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiName } as any);
 		} else {
-			primaryStore = rootStore.openDB(dbiName, dbiInit);
+			primaryStore = (rootStore as any).openDB(dbiName, dbiInit as any);
 		}
 		primaryStore = handleLocalTimeForGets(primaryStore, rootStore);
 		rootStore.databaseName = databaseName;
@@ -990,6 +1078,9 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 				attributes,
 				schemaDefined,
 				dbisDB: attributesDbi,
+				description,
+				properties,
+				hidden,
 			})
 		);
 		Table.schemaVersion = 1;
@@ -1000,15 +1091,15 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 	const indices = Table.indices;
 	if (!attributesDbi) {
 		if (rootStore instanceof RocksDatabase) {
-			rootStore.dbisDb = openRocksDatabase(rootStore.path, {
+			(rootStore as any).dbisDb = openRocksDatabase(rootStore.path, {
 				...internalDbiInit,
 				disableWAL: false,
 				name: INTERNAL_DBIS_NAME,
-			});
+			} as any);
 		} else {
-			rootStore.dbisDb = rootStore.openDB(INTERNAL_DBIS_NAME, internalDbiInit);
+			(rootStore as any).dbisDb = (rootStore as any).openDB(INTERNAL_DBIS_NAME, internalDbiInit as any);
 		}
-		attributesDbi = rootStore.dbisDb;
+		attributesDbi = (rootStore as any).dbisDb;
 	}
 	Table.dbisDB = attributesDbi;
 	const indicesToRemove = [];
@@ -1058,7 +1149,7 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 				) {
 					const updatedPrimaryAttribute = { ...attributeDescriptor };
 					if (typeof audit === 'boolean') {
-						if (audit) Table.enableAuditing(audit);
+						if (audit) Table.enableAuditing();
 						updatedPrimaryAttribute.audit = audit;
 					}
 					if (expiration) updatedPrimaryAttribute.expiration = +expiration;
@@ -1076,31 +1167,51 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 
 			// note that non-indexed attributes do not need a dbi
 			if (attributeDescriptor?.attribute && !attributeDescriptor.name) attributeDescriptor.indexed = true; // legacy descriptor
-			const changed =
+			// Some index options affect only search, not the stored structure (e.g. HNSW's
+			// efConstructionSearch). Changing those should persist the new metadata but NOT trigger a
+			// reindex. A custom index declares such keys via a static `searchOnlyOptions`.
+			const indexType = attribute.indexed && typeof attribute.indexed === 'object' ? attribute.indexed.type : undefined;
+			const searchOnlyOptions: string[] = (indexType && CUSTOM_INDEXES[indexType]?.searchOnlyOptions) || [];
+			const stripSearchOnly = (indexed: any): any => {
+				if (!indexed || typeof indexed !== 'object' || searchOnlyOptions.length === 0) return indexed;
+				const copy = { ...indexed };
+				for (const key of searchOnlyOptions) delete copy[key];
+				return copy;
+			};
+			const commonChanged =
 				!attributeDescriptor ||
 				attributeDescriptor.type !== attribute.type ||
-				JSON.stringify(attributeDescriptor.indexed) !== JSON.stringify(attribute.indexed) ||
 				attributeDescriptor.nullable !== attribute.nullable ||
 				attributeDescriptor.version !== attribute.version ||
 				attributeDescriptor.enumerable !== attribute.enumerable ||
 				JSON.stringify(attributeDescriptor.properties) !== JSON.stringify(attribute.properties) ||
-				JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements);
+				JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements) ||
+				// Include `embed` so a source/model change refreshes the embed registry.
+				JSON.stringify(attributeDescriptor.embed) !== JSON.stringify(attribute.embed);
+			// any metadata difference (drives persistence)
+			const changed =
+				commonChanged || JSON.stringify(attributeDescriptor?.indexed) !== JSON.stringify(attribute.indexed);
+			// structure-affecting difference (drives reindex) — ignores search-only option changes
+			const structurallyChanged =
+				commonChanged ||
+				JSON.stringify(stripSearchOnly(attributeDescriptor?.indexed)) !==
+					JSON.stringify(stripSearchOnly(attribute.indexed));
 			if (attribute.indexed) {
 				const dbi = openIndex(dbiKey, rootStore, attribute);
 				if (
 					changed ||
-					attributeDescriptor.indexingFailed ||
-					(attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
-					attributeDescriptor.restartNumber < workerData?.restartNumber
+					attributeDescriptor?.indexingFailed ||
+					(attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
+					attributeDescriptor?.restartNumber < workerData?.restartNumber
 				) {
 					hasChanges = true;
 					exclusiveLock();
 					attributeDescriptor = attributesDbi.getSync(dbiKey);
 					if (
-						changed ||
-						attributeDescriptor.indexingFailed ||
-						(attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
-						attributeDescriptor.restartNumber < workerData?.restartNumber
+						structurallyChanged ||
+						attributeDescriptor?.indexingFailed ||
+						(attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
+						attributeDescriptor?.restartNumber < workerData?.restartNumber
 					) {
 						hasChanges = true;
 						if (attribute.indexNulls === undefined) attribute.indexNulls = true;
@@ -1118,6 +1229,14 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 							// we only set indexing nulls to true if new or reindexing, we can't have partial indexing of null
 							attributesToIndex.push(attribute);
 						}
+					} else if (attributeDescriptor.indexingPID) {
+						// Metadata-only change (e.g. a search-only option like efConstructionSearch) while a
+						// backfill is in progress: we did NOT re-trigger indexing, so carry over the in-progress
+						// indexing state instead of persisting a descriptor that looks complete — otherwise other
+						// workers / a reload would treat the still-partial index as ready and return incomplete results.
+						attribute.indexingPID = attributeDescriptor.indexingPID;
+						attribute.lastIndexedKey = attributeDescriptor.lastIndexedKey;
+						if (attributeDescriptor.indexingFailed) attribute.indexingFailed = attributeDescriptor.indexingFailed;
 					}
 					attributesDbi.put(dbiKey, attribute);
 				}
@@ -1228,6 +1347,10 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 				// TODO: Do we ever need to interrupt due to a schema change that was not a restart?
 				//if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up
 				outstanding++;
+				// Custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`, so the
+				// outstanding-based yield below never fires for them. Track that this row did synchronous
+				// indexing work so we can still yield the event loop after it.
+				let didSynchronousIndexing = false;
 				// every index operation needs to be guarded by the version still be the same. If it has already changed before
 				// we index, that's fine because indexing is idempotent, we can just put the same values again. If it changes
 				// during the indexing, the indexing here will fail. This is also fine because it means the other thread will have
@@ -1241,6 +1364,7 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 						const value = record && (resolver ? resolver(record) : record[property]);
 						if (index.customIndex) {
 							index.customIndex.index(key, value);
+							didSynchronousIndexing = true;
 							continue;
 						}
 						const values = getIndexedValues(value, index.indexNulls);
@@ -1279,7 +1403,9 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 					if (interrupted) return;
 				}
 				if (outstanding > MAX_OUTSTANDING_INDEXING) await lastResolution;
-				else if (outstanding > MIN_OUTSTANDING_INDEXING) await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
+				else if (outstanding > MIN_OUTSTANDING_INDEXING)
+					await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
+				else if (didSynchronousIndexing) await new Promise((resolve) => setImmediate(resolve)); // custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`; without this yield a large backfill runs in a single event-loop turn, starving keepalive/replication and queries and never letting the isIndexing flag be observed
 			}
 		}
 		// Await the last pending put. If it rejects, that is also an indexing error.
@@ -1414,3 +1540,11 @@ export function getDefaultCompression() {
 	if (STORAGE_COMPRESSION_THRESHOLD) LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
 	return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
 }
+
+/**
+ * Force all RocksDB databases to flush to disk.
+ */
+export async function flushDatabases() {
+	// flush all RocksDB databases
+	return Promise.all(Array.from(rocksdbDatabaseEnvs.values()).map((db) => db.flush()));
+}