@harperfast/harper 5.0.0-alpha.10

package/dist/resources/databases.js

@@ -0,0 +1,1220 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.databases = exports.tables = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
+exports.getTables = getTables;
+exports.getDatabases = getDatabases;
+exports.readMetaDb = readMetaDb;
+exports.resetDatabases = resetDatabases;
+exports.database = database;
+exports.dropDatabase = dropDatabase;
+exports.table = table;
+exports.dropTableMeta = dropTableMeta;
+exports.onUpdatedTable = onUpdatedTable;
+exports.onRemovedDB = onRemovedDB;
+exports.getDefaultCompression = getDefaultCompression;
+const environmentManager_js_1 = require("../utility/environment/environmentManager.js");
+const terms_js_1 = require("../utility/lmdb/terms.js");
+const lmdb_1 = require("lmdb");
+const path_1 = require("path");
+const node_fs_1 = require("node:fs");
+const promises_1 = require("node:fs/promises");
+const initializePaths_js_1 = require("../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js");
+const Table_ts_1 = require("./Table.js");
+const OpenEnvironmentObject_js_1 = __importDefault(require("../utility/lmdb/OpenEnvironmentObject.js"));
+const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
+const globals_js_1 = require("../globals.js");
+const commonUtility_js_1 = require("../utility/lmdb/commonUtility.js");
+const signalling = __importStar(require("../utility/signalling.js"));
+const itc_js_1 = require("../server/threads/itc.js");
+const worker_threads_1 = require("worker_threads");
+const harper_logger_js_1 = __importDefault(require("../utility/logging/harper_logger.js"));
+const { forComponent } = harper_logger_js_1.default;
+const manageThreads = __importStar(require("../server/threads/manageThreads.js"));
+const auditStore_ts_1 = require("./auditStore.js");
+const RecordEncoder_ts_1 = require("./RecordEncoder.js");
+const blob_ts_1 = require("./blob.js");
+const customIndexes_ts_1 = require("./indexes/customIndexes.js");
+const OpenDBIObject_js_1 = require("../utility/lmdb/OpenDBIObject.js");
+const rocksdb_js_1 = require("@harperfast/rocksdb-js");
+const replayLogs_ts_1 = require("./replayLogs.js");
+const node_os_1 = require("node:os");
+const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
+const when_ts_1 = require("../utility/when.js");
+const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
+function createOpenDBIObject(dupSort = false, isPrimary = false) {
+    return new OpenDBIObject_js_1.OpenDBIObject(dupSort, isPrimary);
+}
+const logger = forComponent('storage');
+const DEFAULT_DATABASE_NAME = 'data';
+const DEFINED_TABLES = Symbol('defined-tables');
+const DEFAULT_COMPRESSION_THRESHOLD = ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
+(0, environmentManager_js_1.initSync)();
+// I don't know if this is the best place for this, but somewhere we need to specify which tables
+// replicate by default:
+exports.NON_REPLICATING_SYSTEM_TABLES = [
+    'hdb_temp',
+    'hdb_certificate',
+    'hdb_raw_analytics',
+    'hdb_session_will',
+    'hdb_job',
+    'hdb_info',
+];
+exports.tables = Object.create(null);
+exports.databases = Object.create(null);
+const MEMORY_FOR_ROCKS_DB = Math.min(process.constrainedMemory?.() ?? Infinity, (0, node_os_1.totalmem)()) * 0.25; // 25% of available memory
+function openRocksDatabase(path, options) {
+    options.disableWAL ??= true;
+    rocksdb_js_1.RocksDatabase.config({ blockCacheSize: MEMORY_FOR_ROCKS_DB });
+    if (!(0, node_fs_1.existsSync)(path)) {
+        (0, node_fs_1.mkdirSync)(path, { recursive: true });
+    }
+    let db;
+    if (options.dupSort) {
+        db = rocksdb_js_1.RocksDatabase.open(new RocksIndexStore_ts_1.RocksIndexStore(path, options));
+    }
+    else {
+        db = rocksdb_js_1.RocksDatabase.open(path, options);
+        db.encoder.name = options.name;
+    }
+    db.env = {};
+    return db;
+}
+const lmdbDatabaseEnvs = new Map();
+const rocksdbDatabaseEnvs = new Map();
+// set the following in both global and exports
+(0, globals_js_1._assignPackageExport)('databases', exports.databases);
+(0, globals_js_1._assignPackageExport)('tables', exports.tables);
+const NEXT_TABLE_ID = Symbol.for('next-table-id');
+const tableListeners = [];
+const dbRemovalListeners = [];
+let loadedDatabases; // indicates if we have loaded databases from the file system yet
+// This is used to track all the databases that are found when iterating through the file system so that anything that is missing
+// can be removed:
+let definedDatabases;
+/**
+ * This gets the set of tables from the default database ("data").
+ */
+function getTables() {
+    if (!loadedDatabases) {
+        getDatabases();
+    }
+    return exports.tables || {};
+}
+/**
+ * This provides the main entry point for getting the set of all Harper tables (organized by schemas/databases).
+ * This proactively scans the known
+ * databases/schemas directories and finds any databases and opens them. This done proactively so that there is a fast
+ * object available to all consumers that doesn't require runtime checks for database open states.
+ * This also attaches the audit store associated with table. Note that legacy tables had a single audit table per db table
+ * but in newer multi-table databases, there is one consistent, integrated audit table for the database since transactions
+ * can span any tables in the database.
+ */
+function getDatabases() {
+    if (loadedDatabases) {
+        return exports.databases;
+    }
+    loadedDatabases = true;
+    definedDatabases = new Map();
+    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
+    let databasePath = hdbBasePath && (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME);
+    const schemaConfigs = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    // not sure why this doesn't work with the environmemt manager
+    if (process.env.SCHEMAS_DATA_PATH)
+        schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
+    databasePath =
+        process.env.STORAGE_PATH ||
+            (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
+            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, path_1.join)((0, environmentManager_js_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
+    if (!databasePath)
+        return;
+    if ((0, node_fs_1.existsSync)(databasePath)) {
+        // First load all the databases from our main database folder
+        // TODO: Load any databases defined with explicit storage paths from the config
+        for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
+            const dbName = (0, path_1.basename)(databaseEntry.name, '.mdb');
+            const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
+            if (databaseEntry.isFile() &&
+                (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb' &&
+                !schemaConfigs[dbName]?.path) {
+                logger.trace(`loading lmdb database: ${dbPath}`);
+                readMetaDb(dbPath, null, dbName);
+                continue;
+            }
+            try {
+                const files = (0, node_fs_1.readdirSync)(dbPath, { withFileTypes: true });
+                if (files.find((file) => file.name === 'CURRENT')?.isFile() &&
+                    files.some((file) => file.name.startsWith('MANIFEST-')) &&
+                    !schemaConfigs[dbName]?.path) {
+                    readRocksMetaDb(dbPath, null, dbName);
+                    continue;
+                }
+            }
+            catch (err) {
+                if (!('code' in err && (err.code === 'ENOENT' || err.code === 'ENOTDIR'))) {
+                    throw err;
+                }
+            }
+        }
+    }
+    // now we load databases from the legacy "schema" directory folder structure
+    const baseSchemaPath = (0, initializePaths_js_1.getBaseSchemaPath)();
+    if ((0, node_fs_1.existsSync)(baseSchemaPath)) {
+        for (const schemaEntry of (0, node_fs_1.readdirSync)(baseSchemaPath, { withFileTypes: true })) {
+            if (!schemaEntry.isFile()) {
+                const schemaPath = (0, path_1.join)(baseSchemaPath, schemaEntry.name);
+                const schemaAuditPath = (0, path_1.join)((0, initializePaths_js_1.getTransactionAuditStoreBasePath)(), schemaEntry.name);
+                for (const tableEntry of (0, node_fs_1.readdirSync)(schemaPath, { withFileTypes: true })) {
+                    if (tableEntry.isFile() && (0, path_1.extname)(tableEntry.name).toLowerCase() === '.mdb') {
+                        const auditPath = (0, path_1.join)(schemaAuditPath, tableEntry.name);
+                        readMetaDb((0, path_1.join)(schemaPath, tableEntry.name), (0, path_1.basename)(tableEntry.name, '.mdb'), schemaEntry.name, auditPath, true);
+                    }
+                }
+            }
+        }
+    }
+    if (schemaConfigs) {
+        for (const dbName in schemaConfigs) {
+            const schemaConfig = schemaConfigs[dbName];
+            const databasePath = schemaConfig.path;
+            if ((0, node_fs_1.existsSync)(databasePath)) {
+                for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
+                    if (databaseEntry.isFile() && (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb') {
+                        readMetaDb((0, path_1.join)(databasePath, databaseEntry.name), (0, path_1.basename)(databaseEntry.name, '.mdb'), dbName);
+                    }
+                    else {
+                        try {
+                            const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
+                            const files = (0, node_fs_1.readdirSync)(dbPath, { withFileTypes: true });
+                            if (files.find((file) => file.name === 'CURRENT')?.isFile() &&
+                                files.some((file) => file.name.startsWith('MANIFEST-'))) {
+                                readRocksMetaDb(dbPath, null, dbName);
+                                continue;
+                            }
+                        }
+                        catch (err) {
+                            if (!('code' in err && (err.code === 'ENOENT' || err.code === 'ENOTDIR'))) {
+                                throw err;
+                            }
+                        }
+                    }
+                }
+            }
+            const tableConfigs = schemaConfig.tables;
+            if (tableConfigs) {
+                for (const tableName in tableConfigs) {
+                    const tableConfig = tableConfigs[tableName];
+                    const tablePath = (0, path_1.join)(tableConfig.path, (0, path_1.basename)(tableName + '.mdb'));
+                    if ((0, node_fs_1.existsSync)(tablePath)) {
+                        readMetaDb(tablePath, tableName, dbName, null, true);
+                    }
+                }
+            }
+            //TODO: Iterate configured table paths
+        }
+    }
+    // now remove any databases or tables that have been removed
+    for (const dbName in exports.databases) {
+        const definedTables = definedDatabases.get(dbName);
+        if (definedTables) {
+            const tables = exports.databases[dbName];
+            if (dbName.includes('delete'))
+                logger.trace(`defined tables ${Array.from(definedTables.keys())}`);
+            for (const tableName in tables) {
+                if (!definedTables.has(tableName)) {
+                    logger.trace(`delete table class ${tableName}`);
+                    delete tables[tableName];
+                }
+            }
+        }
+        else {
+            delete exports.databases[dbName];
+            if (dbName === 'data') {
+                for (const tableName in exports.tables) {
+                    delete exports.tables[tableName];
+                }
+                delete exports.tables[DEFINED_TABLES];
+            }
+        }
+    }
+    if ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.ANALYTICS_REPLICATE) === false) {
+        if (!exports.NON_REPLICATING_SYSTEM_TABLES.includes('hdb_analytics'))
+            exports.NON_REPLICATING_SYSTEM_TABLES.push('hdb_analytics');
+    }
+    else {
+        // auditing must be enabled for replication
+        exports.databases.system?.hdb_analytics?.enableAuditing();
+        exports.databases.system?.hdb_analytics_hostname?.enableAuditing();
+    }
+    if (exports.databases.system) {
+        for (const tableName of exports.NON_REPLICATING_SYSTEM_TABLES) {
+            if (exports.databases.system[tableName]) {
+                exports.databases.system[tableName].replicate = false;
+            }
+        }
+    }
+    return exports.databases;
+}
+/**
+ * This is responsible for reading the internal dbi of a single database file to get a list of all the tables and
+ * their indexed or registered attributes
+ * @param path
+ * @param defaultTable
+ * @param databaseName
+ */
+function readMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME, auditPath, isLegacy) {
+    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+    try {
+        let rootStore = lmdbDatabaseEnvs.get(path);
+        if (rootStore) {
+            rootStore.needsDeletion = false;
+        }
+        else {
+            rootStore = (0, lmdb_1.open)(envInit);
+            lmdbDatabaseEnvs.set(path, rootStore);
+        }
+        return initStores(path, rootStore, databaseName, defaultTable, auditPath, isLegacy);
+    }
+    catch (error) {
+        error.message += ` opening database ${path}`;
+        throw error;
+    }
+}
+function readRocksMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME) {
+    try {
+        logger.trace(`loading rocksdb database: ${path}`);
+        if (process.env.HARPER_PARENT_PROCESS_PID) {
+            const parentProcessPid = parseInt(process.env.HARPER_PARENT_PROCESS_PID);
+            if ((0, processManagement_js_1.isProcessRunning)(parentProcessPid)) {
+                logger.info(`Parent process ${parentProcessPid} is still running!`);
+            }
+        }
+        let rootStore = rocksdbDatabaseEnvs.get(path);
+        if (rootStore) {
+            initStores(path, rootStore, databaseName, defaultTable);
+        }
+        else {
+            rootStore = openRocksDatabase(path, { disableWAL: false });
+            rocksdbDatabaseEnvs.set(path, rootStore);
+            initStores(path, rootStore, databaseName, defaultTable);
+            (0, replayLogs_ts_1.replayLogs)(rootStore, exports.databases[databaseName]);
+        }
+        return rootStore;
+    }
+    catch (error) {
+        error.message += ` opening database ${path}`;
+        throw error;
+    }
+}
+function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLegacy) {
+    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+    const internalDbiInit = createOpenDBIObject(false);
+    let dbisStore = rootStore.dbisDb;
+    if (!dbisStore) {
+        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            dbisStore = openRocksDatabase(rootStore.path, {
+                ...internalDbiInit,
+                disableWAL: false,
+                name: terms_js_1.INTERNAL_DBIS_NAME,
+            });
+        }
+        else {
+            dbisStore = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+        }
+        rootStore.dbisDb = dbisStore;
+    }
+    let auditStore = rootStore.auditStore;
+    if (!auditStore) {
+        if (auditPath) {
+            if ((0, node_fs_1.existsSync)(auditPath)) {
+                envInit.path = auditPath;
+                if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                    auditStore = (0, auditStore_ts_1.openAuditStore)(rootStore);
+                }
+                else {
+                    auditStore = (0, lmdb_1.open)({
+                        ...envInit,
+                        encoder: {
+                            encode: (auditRecord) => (0, auditStore_ts_1.createAuditEntry)(auditRecord),
+                            decode: (encoding) => (0, auditStore_ts_1.readAuditEntry)(encoding),
+                        },
+                    });
+                }
+                auditStore.isLegacy = true;
+            }
+        }
+        else {
+            auditStore = (0, auditStore_ts_1.openAuditStore)(rootStore);
+        }
+    }
+    const tables = ensureDB(databaseName);
+    const definedTables = tables[DEFINED_TABLES];
+    definedTables.rootStore = rootStore;
+    const tablesToLoad = new Map();
+    for (const result of dbisStore.getRange({ start: false })) {
+        const { key, value } = result;
+        let [tableName, attribute_name] = key.toString().split('/');
+        if (attribute_name === '') {
+            // primary key
+            attribute_name = value.name;
+        }
+        else if (!attribute_name) {
+            attribute_name = tableName;
+            tableName = defaultTable;
+            if (!value.name) {
+                // legacy attribute
+                value.name = attribute_name;
+                value.indexed = !value.isPrimaryKey;
+            }
+        }
+        definedTables?.add(tableName);
+        let tableDef = tablesToLoad.get(tableName);
+        if (!tableDef)
+            tablesToLoad.set(tableName, (tableDef = { attributes: [] }));
+        if (attribute_name == null || value.isPrimaryKey)
+            tableDef.primary = value;
+        if (attribute_name != null)
+            tableDef.attributes.push(value);
+        Object.defineProperty(value, 'key', { value: key, configurable: true });
+    }
+    for (const [tableName, tableDef] of tablesToLoad) {
+        let { attributes, primary: primaryAttribute } = tableDef;
+        if (!primaryAttribute) {
+            // this isn't defined, find it in the attributes
+            for (const attribute of attributes) {
+                if (attribute.isPrimaryKey) {
+                    primaryAttribute = attribute;
+                    break;
+                }
+            }
+            if (!primaryAttribute) {
+                logger.warn(`Unable to find a primary key attribute on table ${tableName}, with attributes: ${JSON.stringify(attributes)}`);
+                continue;
+            }
+        }
+        // if the table has already been defined, use that class, don't create a new one
+        let table = tables[tableName];
+        let indices = {}, existingAttributes = [];
+        let tableId;
+        let primaryStore;
+        const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
+        const trackDeletes = primaryAttribute.trackDeletes;
+        const expiration = primaryAttribute.expiration;
+        const eviction = primaryAttribute.eviction;
+        const sealed = primaryAttribute.sealed;
+        const splitSegments = primaryAttribute.splitSegments;
+        const replicate = primaryAttribute.replicate;
+        if (table) {
+            indices = table.indices;
+            existingAttributes = table.attributes;
+            table.schemaVersion++;
+        }
+        else {
+            tableId = primaryAttribute.tableId;
+            if (tableId) {
+                if (tableId >= (dbisStore.getSync(NEXT_TABLE_ID) || 0)) {
+                    dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
+                    logger.info(`Updating next table id (it was out of sync) to ${tableId + 1} for ${tableName}`);
+                }
+            }
+            else {
+                primaryAttribute.tableId = tableId = dbisStore.getSync(NEXT_TABLE_ID);
+                if (!tableId)
+                    tableId = 1;
+                logger.debug(`Table {tableName} missing an id, assigning {tableId}`);
+                dbisStore.putSync(NEXT_TABLE_ID, tableId + 1);
+                dbisStore.putSync(primaryAttribute.key, primaryAttribute);
+            }
+            const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
+            dbiInit.compression = primaryAttribute.compression;
+            if (dbiInit.compression) {
+                const compressionThreshold = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
+                dbiInit.compression.threshold = compressionThreshold;
+            }
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                primaryStore = (0, RecordEncoder_ts_1.handleLocalTimeForGets)(openRocksDatabase(rootStore.path, { ...dbiInit, name: primaryAttribute.key }), rootStore);
+            }
+            else {
+                primaryStore = (0, RecordEncoder_ts_1.handleLocalTimeForGets)(rootStore.openDB(primaryAttribute.key, dbiInit), rootStore);
+            }
+            rootStore.databaseName = databaseName;
+            primaryStore.tableId = tableId;
+        }
+        let attributesUpdated;
+        for (const attribute of attributes) {
+            attribute.attribute = attribute.name;
+            try {
+                // now load the non-primary keys, opening the dbs as necessary for indices
+                if (!attribute.isPrimaryKey && (attribute.indexed || (attribute.attribute && !attribute.name))) {
+                    if (!indices[attribute.name]) {
+                        const dbi = openIndex(attribute.key, rootStore, attribute);
+                        indices[attribute.name] = dbi;
+                        indices[attribute.name].indexNulls = attribute.indexNulls;
+                    }
+                    const existingAttribute = existingAttributes.find((existingAttribute) => existingAttribute.name === attribute.name);
+                    if (existingAttribute)
+                        existingAttributes.splice(existingAttributes.indexOf(existingAttribute), 1, attribute);
+                    else
+                        existingAttributes.push(attribute);
+                    attributesUpdated = true;
+                }
+            }
+            catch (error) {
+                logger.error(`Error trying to update attribute`, attribute, existingAttributes, indices, error);
+            }
+        }
+        for (const existingAttribute of existingAttributes) {
+            const attribute = attributes.find((attribute) => attribute.name === existingAttribute.name);
+            if (!attribute) {
+                if (existingAttribute.isPrimaryKey) {
+                    logger.error('Unable to remove existing primary key attribute', existingAttribute);
+                    continue;
+                }
+                if (existingAttribute.indexed) {
+                    // we only remove attributes if they were indexed, in order to support dropAttribute that removes dynamic indexed attributes
+                    existingAttributes.splice(existingAttributes.indexOf(existingAttribute), 1);
+                    attributesUpdated = true;
+                }
+            }
+        }
+        if (table) {
+            if (attributesUpdated) {
+                table.schemaVersion++;
+                table.updatedAttributes();
+            }
+        }
+        else {
+            table = setTable(tables, tableName, (0, Table_ts_1.makeTable)({
+                primaryStore,
+                auditStore,
+                audit,
+                sealed,
+                splitSegments,
+                replicate,
+                expirationMS: expiration && expiration * 1000,
+                evictionMS: eviction && eviction * 1000,
+                trackDeletes,
+                tableName,
+                tableId,
+                primaryKey: primaryAttribute.name,
+                databasePath: isLegacy ? `${databaseName}/${tableName}` : databaseName,
+                databaseName,
+                indices,
+                attributes,
+                schemaDefined: primaryAttribute.schemaDefined,
+                dbisDB: dbisStore,
+            }));
+            table.schemaVersion = 1;
+            for (const listener of tableListeners) {
+                listener(table);
+            }
+        }
+    }
+    return rootStore;
+}
+function resetDatabases() {
+    loadedDatabases = false;
+    for (const store of Object.values(lmdbDatabaseEnvs)) {
+        store.needsDeletion = true;
+    }
+    getDatabases();
+    for (const [path, store] of lmdbDatabaseEnvs) {
+        if (store.needsDeletion && !path.endsWith('system.mdb')) {
+            store.close();
+            lmdbDatabaseEnvs.delete(path);
+            const db = exports.databases[store.databaseName];
+            for (const tableName in db) {
+                const table = db[tableName];
+                if (table.primaryStore.path === path) {
+                    delete exports.databases[store.databaseName];
+                    dbRemovalListeners.forEach((listener) => listener(store.databaseName));
+                    break;
+                }
+            }
+        }
+    }
+    return exports.databases;
+}
+/**
+ * Ensure that we have this database object (that holds a set of tables) set up
+ * @param databaseName
+ * @returns
+ */
+function ensureDB(databaseName) {
+    let dbTables = exports.databases[databaseName];
+    if (!dbTables) {
+        if (databaseName === 'data')
+            // preserve the data tables objet
+            dbTables = exports.databases[databaseName] = exports.tables;
+        else if (databaseName === 'system')
+            // make system non-enumerable
+            Object.defineProperty(exports.databases, 'system', {
+                value: (dbTables = Object.create(null)),
+                configurable: true, // no enum
+            });
+        else {
+            dbTables = exports.databases[databaseName] = Object.create(null);
+        }
+    }
+    if (definedDatabases && !definedDatabases.has(databaseName)) {
+        const definedTables = new Set(); // we create this so we can determine what was found in a reset and remove any removed dbs/tables
+        dbTables[DEFINED_TABLES] = definedTables;
+        definedDatabases.set(databaseName, definedTables);
+    }
+    return dbTables;
+}
+/**
+ * Set the table class into the database's tables object
+ * @param tables
+ * @param tableName
+ * @param Table
+ * @returns
+ */
+function setTable(tables, tableName, Table) {
+    tables[tableName] = Table;
+    return Table;
+}
+/**
+ * Get root store for a database
+ * @param options
+ * @returns
+ */
+function database({ database: databaseName, table: tableName }) {
+    if (!databaseName)
+        databaseName = DEFAULT_DATABASE_NAME;
+    getDatabases();
+    ensureDB(databaseName);
+    const definedDatabase = definedDatabases.get(databaseName);
+    if (definedDatabase?.rootStore) {
+        return definedDatabase.rootStore;
+    }
+    const databaseConfig = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
+    if (process.env.SCHEMAS_DATA_PATH) {
+        databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
+    }
+    const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
+    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
+    const databasePath = tablePath ||
+        databaseConfig[databaseName]?.path ||
+        process.env.STORAGE_PATH ||
+        (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
+        ((0, node_fs_1.existsSync)((0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
+            ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME)
+            : (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME));
+    let rootStore;
+    const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
+    if (useRocksdb) {
+        const path = (0, path_1.join)(databasePath, tablePath ? tableName : databaseName);
+        rootStore = rocksdbDatabaseEnvs.get(path);
+        if (!rootStore || rootStore.status === 'closed') {
+            rootStore = openRocksDatabase(path, {
+                disableWAL: false,
+            });
+            rocksdbDatabaseEnvs.set(path, rootStore);
+        }
+    }
+    else {
+        const path = (0, path_1.join)(databasePath, `${tablePath ? tableName : databaseName}.mdb`);
+        rootStore = lmdbDatabaseEnvs.get(path);
+        if (!rootStore || rootStore.status === 'closed') {
+            // TODO: validate database name
+            const envInit = new OpenEnvironmentObject_js_1.default(path, false);
+            rootStore = (0, lmdb_1.open)(envInit);
+            lmdbDatabaseEnvs.set(path, rootStore);
+        }
+    }
+    if (!rootStore.auditStore) {
+        rootStore.auditStore = (0, auditStore_ts_1.openAuditStore)(rootStore);
+    }
+    if (definedDatabase)
+        definedDatabase.rootStore = rootStore;
+    return rootStore;
+}
+/**
+ * Delete the database
+ * @param databaseName
+ */
+async function dropDatabase(databaseName) {
+    if (!exports.databases[databaseName])
+        throw new Error('Database does not exist');
+    const dbTables = exports.databases[databaseName];
+    let rootStore;
+    for (const tableName in dbTables) {
+        const table = dbTables[tableName];
+        rootStore = table.primaryStore.rootStore;
+        lmdbDatabaseEnvs.delete(rootStore.path);
+        rocksdbDatabaseEnvs.delete(rootStore.path);
+        if (rootStore.status === 'open') {
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                rootStore.close();
+                rootStore.destroy();
+            }
+            else if (rootStore.status === 'open') {
+                await rootStore.close();
+                await (0, promises_1.unlink)(rootStore.path);
+            }
+        }
+    }
+    if (!rootStore) {
+        rootStore = database({ database: databaseName, table: null });
+        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            rootStore.destroy();
+        }
+        else if (rootStore.status === 'open') {
+            await rootStore.close();
+            await (0, promises_1.unlink)(rootStore.path);
+        }
+    }
+    if (databaseName === 'data') {
+        for (const tableName in exports.tables) {
+            delete exports.tables[tableName];
+        }
+        delete exports.tables[DEFINED_TABLES];
+    }
+    delete exports.databases[databaseName];
+    dbRemovalListeners.forEach((listener) => listener(databaseName));
+    await (0, blob_ts_1.deleteRootBlobPathsForDB)(rootStore);
+}
+// opens an index, consulting with custom indexes that may use alternate store configuration
+function openIndex(dbiKey, rootStore, attribute) {
+    const objectStorage = attribute.isPrimaryKey || (attribute.indexed.type && customIndexes_ts_1.CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore);
+    const dbiInit = createOpenDBIObject(!objectStorage, objectStorage);
+    let dbi;
+    if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+        dbi = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiKey });
+        dbi.rootStore = rootStore;
+    }
+    else {
+        dbi = rootStore.openDB(dbiKey, dbiInit);
+    }
+    if (attribute.indexed.type) {
+        const CustomIndex = customIndexes_ts_1.CUSTOM_INDEXES[attribute.indexed.type];
+        if (CustomIndex) {
+            dbi.customIndex = new CustomIndex(dbi, attribute.indexed);
+        }
+        else {
+            logger.error(`The indexing type '${attribute.indexed.type}' is unknown`);
+        }
+    }
+    return dbi;
+}
+/**
+ * This can be called to ensure that the specified table exists and if it does not exist, it should be created.
+ * @param tableName
+ * @param databaseName
+ * @param customPath
+ * @param expiration
+ * @param eviction
+ * @param scanInterval
+ * @param attributes
+ * @param audit
+ * @param sealed
+ * @param splitSegments
+ * @param replicate
+ */
+function table(tableDefinition) {
+    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, trackDeletes, schemaDefined, origin, } = tableDefinition;
+    if (!databaseName)
+        databaseName = DEFAULT_DATABASE_NAME;
+    const rootStore = database({ database: databaseName, table: tableName });
+    const tables = exports.databases[databaseName];
+    logger.trace(`Defining ${tableName} in ${databaseName}`);
+    let Table = tables?.[tableName];
+    if (rootStore.status === 'closed') {
+        throw new Error(`Can not use a closed data store for ${tableName}`);
+    }
+    let primaryKey;
+    let primaryKeyAttribute;
+    let attributesDbi;
+    if (schemaDefined == undefined)
+        schemaDefined = true;
+    const internalDbiInit = createOpenDBIObject(false);
+    for (const attribute of attributes) {
+        if (attribute.attribute && !attribute.name) {
+            // there is some legacy code that calls the attribute's name the attribute's attribute
+            attribute.name = attribute.attribute;
+            attribute.indexed = true;
+        }
+        else
+            attribute.attribute = attribute.name;
+        if (attribute.expiresAt)
+            attribute.indexed = true;
+    }
+    let hasChanges;
+    let releaseExclusiveLock;
+    if (Table) {
+        primaryKey = Table.primaryKey;
+        if (Table.primaryStore.rootStore.status === 'closed') {
+            throw new Error(`Can not use a closed data store from ${tableName} class`);
+        }
+        // it table already exists, get the split segments setting
+        if (splitSegments == undefined)
+            splitSegments = Table.splitSegments;
+        Table.attributes.splice(0, Table.attributes.length, ...attributes);
+    }
+    else {
+        const auditStore = rootStore.auditStore;
+        primaryKeyAttribute = attributes.find((attribute) => attribute.isPrimaryKey) || {};
+        primaryKey = primaryKeyAttribute.name;
+        primaryKeyAttribute.isPrimaryKey = true;
+        primaryKeyAttribute.schemaDefined = schemaDefined;
+        // can't change compression after the fact (except threshold), so save only when we create the table
+        primaryKeyAttribute.compression = getDefaultCompression();
+        if (trackDeletes)
+            primaryKeyAttribute.trackDeletes = true;
+        audit = primaryKeyAttribute.audit = typeof audit === 'boolean' ? audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
+        if (expiration)
+            primaryKeyAttribute.expiration = expiration;
+        if (eviction)
+            primaryKeyAttribute.eviction = eviction;
+        splitSegments ??= false;
+        primaryKeyAttribute.splitSegments = splitSegments; // always default to not splitting segments going forward
+        if (typeof sealed === 'boolean')
+            primaryKeyAttribute.sealed = sealed;
+        if (typeof replicate === 'boolean')
+            primaryKeyAttribute.replicate = replicate;
+        if (origin) {
+            if (!primaryKeyAttribute.origins)
+                primaryKeyAttribute.origins = [origin];
+            else if (!primaryKeyAttribute.origins.includes(origin))
+                primaryKeyAttribute.origins.push(origin);
+        }
+        logger.trace(`${tableName} table loading, opening primary store`);
+        const dbiInit = createOpenDBIObject(false, true);
+        dbiInit.compression = primaryKeyAttribute.compression;
+        const dbiName = tableName + '/';
+        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            attributesDbi = rootStore.dbisDb = openRocksDatabase(rootStore.path, {
+                ...internalDbiInit,
+                disableWAL: false,
+                name: terms_js_1.INTERNAL_DBIS_NAME,
+            });
+        }
+        else {
+            attributesDbi = rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+        }
+        exclusiveLock(); // get an exclusive lock on the database so we can verify that we are the only thread creating the table (and assigning the table id)
+        if (attributesDbi.getSync(dbiName)) {
+            // table was created while we were setting up
+            if (releaseExclusiveLock)
+                releaseExclusiveLock();
+            resetDatabases();
+            return table(tableDefinition);
+        }
+        let primaryStore;
+        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            primaryStore = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiName });
+        }
+        else {
+            primaryStore = rootStore.openDB(dbiName, dbiInit);
+        }
+        primaryStore = (0, RecordEncoder_ts_1.handleLocalTimeForGets)(primaryStore, rootStore);
+        rootStore.databaseName = databaseName;
+        primaryStore.tableId = attributesDbi.getSync(NEXT_TABLE_ID);
+        logger.trace(`Assigning new table id ${primaryStore.tableId} for ${tableName}`);
+        if (!primaryStore.tableId)
+            primaryStore.tableId = 1;
+        attributesDbi.put(NEXT_TABLE_ID, primaryStore.tableId + 1);
+        primaryKeyAttribute.tableId = primaryStore.tableId;
+        Table = setTable(tables, tableName, (0, Table_ts_1.makeTable)({
+            primaryStore,
+            auditStore,
+            audit,
+            sealed,
+            splitSegments,
+            replicate,
+            trackDeletes,
+            expirationMS: expiration && expiration * 1000,
+            evictionMS: eviction && eviction * 1000,
+            primaryKey,
+            tableName,
+            tableId: primaryStore.tableId,
+            databasePath: databaseName,
+            databaseName,
+            indices: {},
+            attributes,
+            schemaDefined,
+            dbisDB: attributesDbi,
+        }));
+        Table.schemaVersion = 1;
+        hasChanges = true;
+        attributesDbi.put(dbiName, primaryKeyAttribute);
+    }
+    const indices = Table.indices;
+    if (!attributesDbi) {
+        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            rootStore.dbisDb = openRocksDatabase(rootStore.path, {
+                ...internalDbiInit,
+                disableWAL: false,
+                name: terms_js_1.INTERNAL_DBIS_NAME,
+            });
+        }
+        else {
+            rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
+        }
+        attributesDbi = rootStore.dbisDb;
+    }
+    Table.dbisDB = attributesDbi;
+    const indicesToRemove = [];
+    for (const { key, value } of attributesDbi.getRange({ start: true })) {
+        let [attributeTableName, attribute_name] = key.toString().split('/');
+        if (attribute_name === '')
+            attribute_name = value.name; // primary key
+        if (attribute_name) {
+            if (attributeTableName !== tableName)
+                continue;
+        }
+        else {
+            // table attribute for a table with no primary key, we don't want to remove this, so continue on
+            continue;
+        }
+        const attribute = attributes.find((attribute) => attribute.name === attribute_name);
+        const removeIndex = !attribute?.indexed && value.indexed && !value.isPrimaryKey;
+        if (!attribute || removeIndex) {
+            exclusiveLock();
+            hasChanges = true;
+            if (!attribute)
+                attributesDbi.remove(key);
+            if (removeIndex) {
+                const indexDbi = Table.indices[attributeTableName];
+                if (indexDbi)
+                    indicesToRemove.push(indexDbi);
+            }
+        }
+    }
+    const attributesToIndex = [];
+    try {
+        // TODO: If we have attributes and the schemaDefined flag is not set, turn it on
+        // iterate through the attributes to ensure that we have all the dbis created and indexed
+        for (const attribute of attributes || []) {
+            if (attribute.relationship || attribute.computed) {
+                hasChanges = true; // need to update the table so the computed properties are translated to property resolvers
+                if (attribute.relationship)
+                    continue;
+            }
+            let dbiKey = tableName + '/' + (attribute.name || '');
+            Object.defineProperty(attribute, 'key', { value: dbiKey, configurable: true });
+            let attributeDescriptor = attributesDbi.getSync(dbiKey);
+            if (attribute.isPrimaryKey) {
+                attributeDescriptor = attributeDescriptor || attributesDbi.getSync((dbiKey = tableName + '/')) || {};
+                // primary key can't change indexing, but settings can change
+                if ((audit !== undefined && audit !== Table.audit) ||
+                    (sealed !== undefined && sealed !== Table.sealed) ||
+                    (replicate !== undefined && replicate !== Table.replicate) ||
+                    (+expiration || undefined) !== (+attributeDescriptor.expiration || undefined) ||
+                    (+eviction || undefined) !== (+attributeDescriptor.eviction || undefined) ||
+                    attribute.type !== attributeDescriptor.type) {
+                    const updatedPrimaryAttribute = { ...attributeDescriptor };
+                    if (typeof audit === 'boolean') {
+                        if (audit)
+                            Table.enableAuditing(audit);
+                        updatedPrimaryAttribute.audit = audit;
+                    }
+                    if (expiration)
+                        updatedPrimaryAttribute.expiration = +expiration;
+                    if (eviction)
+                        updatedPrimaryAttribute.eviction = +eviction;
+                    if (sealed !== undefined)
+                        updatedPrimaryAttribute.sealed = sealed;
+                    if (replicate !== undefined)
+                        updatedPrimaryAttribute.replicate = replicate;
+                    if (attribute.type)
+                        updatedPrimaryAttribute.type = attribute.type;
+                    hasChanges = true; // send out notification of the change
+                    exclusiveLock();
+                    attributesDbi.put(dbiKey, updatedPrimaryAttribute);
+                }
+                continue;
+            }
+            // note that non-indexed attributes do not need a dbi
+            if (attributeDescriptor?.attribute && !attributeDescriptor.name)
+                attributeDescriptor.indexed = true; // legacy descriptor
+            const changed = !attributeDescriptor ||
+                attributeDescriptor.type !== attribute.type ||
+                JSON.stringify(attributeDescriptor.indexed) !== JSON.stringify(attribute.indexed) ||
+                attributeDescriptor.nullable !== attribute.nullable ||
+                attributeDescriptor.version !== attribute.version ||
+                attributeDescriptor.enumerable !== attribute.enumerable ||
+                JSON.stringify(attributeDescriptor.properties) !== JSON.stringify(attribute.properties) ||
+                JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements);
+            if (attribute.indexed) {
+                const dbi = openIndex(dbiKey, rootStore, attribute);
+                if (changed ||
+                    (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
+                    attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
+                    hasChanges = true;
+                    exclusiveLock();
+                    attributeDescriptor = attributesDbi.getSync(dbiKey);
+                    if (changed ||
+                        (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
+                        attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
+                        hasChanges = true;
+                        if (attribute.indexNulls === undefined)
+                            attribute.indexNulls = true;
+                        let hasExistingData = false;
+                        for (let _entry of Table.primaryStore.getRange({ start: true })) {
+                            hasExistingData = true;
+                            break;
+                        }
+                        if (hasExistingData) {
+                            attribute.lastIndexedKey = attributeDescriptor?.lastIndexedKey ?? undefined;
+                            attribute.indexingPID = process.pid;
+                            dbi.isIndexing = true;
+                            Object.defineProperty(attribute, 'dbi', { value: dbi });
+                            // we only set indexing nulls to true if new or reindexing, we can't have partial indexing of null
+                            attributesToIndex.push(attribute);
+                        }
+                    }
+                    attributesDbi.put(dbiKey, attribute);
+                }
+                if (attributeDescriptor?.indexNulls && attribute.indexNulls === undefined)
+                    attribute.indexNulls = true;
+                dbi.indexNulls = attribute.indexNulls;
+                indices[attribute.name] = dbi;
+            }
+            else if (changed) {
+                hasChanges = true;
+                exclusiveLock();
+                attributesDbi.put(dbiKey, attribute);
+            }
+        }
+    }
+    finally {
+        if (releaseExclusiveLock)
+            releaseExclusiveLock();
+    }
+    if (hasChanges) {
+        Table.schemaVersion++;
+        Table.updatedAttributes();
+    }
+    logger.trace(`${tableName} table loading, running index`);
+    if (attributesToIndex.length > 0 || indicesToRemove.length > 0) {
+        Table.indexingOperation = runIndexing(Table, attributesToIndex, indicesToRemove);
+    }
+    else if (hasChanges)
+        signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName));
+    Table.origin = origin;
+    if (hasChanges) {
+        for (const listener of tableListeners) {
+            listener(Table, origin !== 'cluster');
+        }
+    }
+    if (expiration || eviction || scanInterval)
+        Table.setTTLExpiration({
+            expiration,
+            eviction,
+            scanInterval,
+        });
+    logger.trace(`${tableName} table loaded`);
+    return Table;
+    // Acquire an exclusive lock for attribute updates
+    function exclusiveLock() {
+        if (releaseExclusiveLock)
+            return;
+        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+            while (!rootStore.tryLock('update-attributes')) { } // use a spin lock, we really need an synchronous exclusive lock here
+            releaseExclusiveLock = () => {
+                rootStore.unlock('update-attributes');
+            };
+        }
+        else {
+            // we only need an exclusive transaction lock in lmdb
+            rootStore.transactionSync(() => {
+                return {
+                    then(callback) {
+                        releaseExclusiveLock = callback;
+                    },
+                };
+            });
+        }
+    }
+}
+const MAX_OUTSTANDING_INDEXING = 1000;
+const MIN_OUTSTANDING_INDEXING = 10;
+async function runIndexing(Table, attributes, indicesToRemove) {
+    try {
+        logger.info(`Indexing ${Table.tableName} attributes`, attributes);
+        await signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName));
+        let lastResolution;
+        for (const index of indicesToRemove) {
+            lastResolution = index.drop();
+        }
+        let interrupted;
+        const attributeErrorReported = {};
+        let indexed = 0;
+        const attributesLength = attributes.length;
+        await new Promise((resolve) => setImmediate(resolve)); // yield event turn, indexing should consistently take at least one event turn
+        if (attributesLength > 0) {
+            let start;
+            for (const attribute of attributes) {
+                // if we are resuming, we need to start from the last key we indexed by all attributes
+                if ((0, lmdb_1.compareKeys)(attribute.lastIndexedKey, start) < 0)
+                    start = attribute.lastIndexedKey;
+                if (attribute.lastIndexedKey == undefined) {
+                    // if we are starting from the beginning, clear out any previous index entries since we are rewriting
+                    if (attribute.dbi.clearAsync) {
+                        // LMDB, note that we don't need to wait for this to complete, just gets enqueued in front of the other writes
+                        attribute.dbi.clearAsync();
+                    }
+                    else {
+                        await attribute.dbi.clear();
+                    }
+                }
+            }
+            let outstanding = 0;
+            // this means that a new attribute has been introduced that needs to be indexed
+            for (const { key, value: record } of Table.primaryStore.getRange({
+                start,
+                lazy: attributesLength < 4,
+                versions: true,
+                snapshot: false, // don't hold a read transaction this whole time
+            })) {
+                if (!record)
+                    continue; // deletion entry
+                // TODO: Do we ever need to interrupt due to a schema change that was not a restart?
+                //if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up
+                outstanding++;
+                // every index operation needs to be guarded by the version still be the same. If it has already changed before
+                // we index, that's fine because indexing is idempotent, we can just put the same values again. If it changes
+                // during the indexing, the indexing here will fail. This is also fine because it means the other thread will have
+                // performed indexing and we don't need to do anything further
+                for (let i = 0; i < attributesLength; i++) {
+                    const attribute = attributes[i];
+                    const property = attribute.name;
+                    const index = attribute.dbi;
+                    try {
+                        const resolver = attribute.resolve;
+                        const value = record && (resolver ? resolver(record) : record[property]);
+                        if (index.customIndex) {
+                            index.customIndex.index(key, value);
+                            continue;
+                        }
+                        const values = (0, commonUtility_js_1.getIndexedValues)(value, index.indexNulls);
+                        if (values) {
+                            for (let i = 0, l = values.length; i < l; i++) {
+                                lastResolution = index.put(values[i], key);
+                            }
+                        }
+                    }
+                    catch (error) {
+                        if (!attributeErrorReported[property]) {
+                            // just report an indexing error once per attribute so we don't spam the logs
+                            attributeErrorReported[property] = true;
+                            logger.error(`Error indexing attribute ${property}`, error);
+                        }
+                    }
+                }
+                (0, when_ts_1.when)(lastResolution, () => outstanding--, (error) => {
+                    outstanding--;
+                    logger.error(error);
+                });
+                if (worker_threads_1.workerData && worker_threads_1.workerData.restartNumber !== manageThreads.restartNumber) {
+                    interrupted = true;
+                }
+                if (++indexed % 100 === 0 || interrupted) {
+                    // occasionally update our progress so if we crash, we can resume
+                    for (const attribute of attributes) {
+                        attribute.lastIndexedKey = key;
+                        Table.dbisDB.put(attribute.key, attribute);
+                    }
+                    if (interrupted)
+                        return;
+                }
+                if (outstanding > MAX_OUTSTANDING_INDEXING)
+                    await lastResolution;
+                else if (outstanding > MIN_OUTSTANDING_INDEXING)
+                    await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
+            }
+            // update the attributes to indicate that we are finished
+            for (const attribute of attributes) {
+                delete attribute.lastIndexedKey;
+                delete attribute.indexingPID;
+                attribute.dbi.isIndexing = false;
+                lastResolution = Table.dbisDB.put(attribute.key, attribute);
+            }
+        }
+        await lastResolution;
+        // now notify all the threads that we are done and the index is ready to use
+        await signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'indexing-finished', Table.databaseName, Table.tableName));
+        logger.info(`Finished indexing ${Table.tableName} attributes`, attributes);
+    }
+    catch (error) {
+        logger.error('Error in indexing', error);
+    }
+}
+function dropTableMeta({ table: tableName, database: databaseName }) {
+    const rootStore = database({ database: databaseName, table: tableName });
+    const removals = [];
+    const dbisDb = rootStore.dbisDb;
+    for (const key of dbisDb.getKeys({ start: tableName + '/', end: tableName + '0' })) {
+        removals.push(dbisDb.remove(key));
+    }
+    return Promise.all(removals);
+}
+function onUpdatedTable(listener) {
+    tableListeners.push(listener);
+    return {
+        remove() {
+            const index = tableListeners.indexOf(listener);
+            if (index > -1)
+                tableListeners.splice(index, 1);
+        },
+    };
+}
+function onRemovedDB(listener) {
+    dbRemovalListeners.push(listener);
+    return {
+        remove() {
+            const index = dbRemovalListeners.indexOf(listener);
+            if (index > -1)
+                dbRemovalListeners.splice(index, 1);
+        },
+    };
+}
+function getDefaultCompression() {
+    const LMDB_COMPRESSION = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION);
+    const STORAGE_COMPRESSION_DICTIONARY = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_DICTIONARY);
+    const STORAGE_COMPRESSION_THRESHOLD = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD;
+    const LMDB_COMPRESSION_OPTS = { startingOffset: 32 };
+    if (STORAGE_COMPRESSION_DICTIONARY)
+        LMDB_COMPRESSION_OPTS['dictionary'] = (0, node_fs_1.readFileSync)(STORAGE_COMPRESSION_DICTIONARY);
+    if (STORAGE_COMPRESSION_THRESHOLD)
+        LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
+    return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
+}
+//# sourceMappingURL=databases.js.map