@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

package/dist/resources/databases.js

@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BRANCH_REMOVING_SUFFIX = exports.BRANCH_ROOT_DIR = exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
+exports.CHECKPOINT_ALGORITHM = exports.BRANCH_REMOVING_SUFFIX = exports.BRANCH_ROOT_DIR = exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
 exports.isReadOnlyMode = isReadOnlyMode;
 exports.getRocksCompression = getRocksCompression;
 exports.resetRocksCompression = resetRocksCompression;
@@ -65,6 +65,8 @@ exports.table = table;
 exports.scopedTableFactory = scopedTableFactory;
 exports.reloadBranchAt = reloadBranchAt;
 exports.canonicalizeIndexOptions = canonicalizeIndexOptions;
+exports.setIndexingCheckpointPeriod = setIndexingCheckpointPeriod;
+exports.resumeStartKey = resumeStartKey;
 exports.dropTableMeta = dropTableMeta;
 exports.onUpdatedTable = onUpdatedTable;
 exports.onRemovedTable = onRemovedTable;
@@ -72,10 +74,11 @@ exports.onRemovedDB = onRemovedDB;
 exports.getDefaultCompression = getDefaultCompression;
 exports.flushDatabases = flushDatabases;
 const node_events_1 = require("node:events");
+const node_crypto_1 = require("node:crypto");
 const environmentManager_ts_1 = require("../utility/environment/environmentManager.js");
 const terms_ts_1 = require("../utility/lmdb/terms.js");
 const lmdb_1 = require("lmdb");
-const path_1 = require("path");
+const node_path_1 = require("node:path");
 const node_fs_1 = require("node:fs");
 const promises_1 = require("node:fs/promises");
 const initializePaths_js_1 = require("../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js");
@@ -98,13 +101,14 @@ const blob_ts_1 = require("./blob.js");
 const storageReclamation_ts_1 = require("../server/storageReclamation.js");
 const common_validators_ts_1 = require("../validation/common_validators.js");
 const customIndexes_ts_1 = require("./indexes/customIndexes.js");
+const hnswPlaneBinding_ts_1 = require("./indexes/hnswPlaneBinding.js");
 const OpenDBIObject_ts_1 = require("../utility/lmdb/OpenDBIObject.js");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const PrimaryRocksDatabase_ts_1 = require("./PrimaryRocksDatabase.js");
 const replayLogs_ts_1 = require("./replayLogs.js");
+const hnswDerivedIndex_ts_1 = require("./indexes/hnswDerivedIndex.js");
 const node_os_1 = require("node:os");
 const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
-const when_ts_1 = require("../utility/when.js");
 const rocksMemoryConfig_ts_1 = require("../utility/rocksMemoryConfig.js");
 const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
 const restoreMarker_ts_1 = require("../dataLayer/restoreMarker.js");
@@ -417,6 +421,23 @@ function applyDurableDeclaration(attribute, descriptor) {
             delete attribute[field];
     }
 }
+/**
+ * True when a descriptor claims an index build no live operation in this process can own. The PID and
+ * worker generation cannot answer that alone: a container reuses PID 1 and starts the in-memory
+ * generation back at 1 while the persisted one is higher. A descriptor with no incarnation was written
+ * before the field existed, so it belongs to an earlier process; a thread started without one of its
+ * own cannot judge, and falls back rather than declaring a live build dead.
+ */
+function isAbandonedIndexBuild(descriptor, currentRestartGeneration) {
+    if (!descriptor)
+        return false;
+    if (descriptor.indexingPID && descriptor.indexingPID !== process.pid)
+        return true;
+    if (descriptor.restartNumber < currentRestartGeneration)
+        return true;
+    const incarnation = manageThreads.processIncarnation;
+    return !!descriptor.indexingPID && incarnation != null && descriptor.indexingIncarnation !== incarnation;
+}
 // How many times the schema load will try to finish a tombstoned drop before
 // giving up for the rest of this process's lifetime. A drop that fails once
 // almost always fails identically forever - the usual cause is a RocksDB
@@ -492,7 +513,7 @@ function getDatabases() {
     definedDatabases = new Map();
     relationshipsToHydrate = [];
     const hdbBasePath = (0, environmentManager_ts_1.getHdbBasePath)();
-    let databasePath = hdbBasePath && (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME);
+    let databasePath = hdbBasePath && (0, node_path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME);
     const schemaConfigs = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
     // not sure why this doesn't work with the environmemt manager
     if (process.env.SCHEMAS_DATA_PATH)
@@ -500,7 +521,7 @@ function getDatabases() {
     databasePath =
         process.env.STORAGE_PATH ||
             (0, configUtils_ts_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
-            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, path_1.join)((0, environmentManager_ts_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
+            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, node_path_1.join)((0, environmentManager_ts_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
     if (databasePath && (0, node_fs_1.existsSync)(databasePath)) {
         // First load all the databases from our main database folder
         // TODO: Load any databases defined with explicit storage paths from the config
@@ -518,14 +539,14 @@ function getDatabases() {
             // branch directories are process-local derivatives, never databases in their own right
             if (databaseEntry.name === exports.BRANCH_ROOT_DIR)
                 continue;
-            const dbName = (0, path_1.basename)(databaseEntry.name, '.mdb');
-            const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
+            const dbName = (0, node_path_1.basename)(databaseEntry.name, '.mdb');
+            const dbPath = (0, node_path_1.join)(databasePath, databaseEntry.name);
             if (blockedByRestore.has(dbName))
                 continue;
             if (isOpenBranchPath(dbPath))
                 continue;
             if (databaseEntry.isFile() &&
-                (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb' &&
+                (0, node_path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb' &&
                 !schemaConfigs[dbName]?.path) {
                 logger.trace(`loading lmdb database: ${dbPath}`);
                 readMetaDb(dbPath, null, dbName);
@@ -552,12 +573,12 @@ function getDatabases() {
     if ((0, node_fs_1.existsSync)(baseSchemaPath)) {
         for (const schemaEntry of (0, node_fs_1.readdirSync)(baseSchemaPath, { withFileTypes: true })) {
             if (!schemaEntry.isFile()) {
-                const schemaPath = (0, path_1.join)(baseSchemaPath, schemaEntry.name);
-                const schemaAuditPath = (0, path_1.join)((0, initializePaths_js_1.getTransactionAuditStoreBasePath)(), schemaEntry.name);
+                const schemaPath = (0, node_path_1.join)(baseSchemaPath, schemaEntry.name);
+                const schemaAuditPath = (0, node_path_1.join)((0, initializePaths_js_1.getTransactionAuditStoreBasePath)(), schemaEntry.name);
                 for (const tableEntry of (0, node_fs_1.readdirSync)(schemaPath, { withFileTypes: true })) {
-                    if (tableEntry.isFile() && (0, path_1.extname)(tableEntry.name).toLowerCase() === '.mdb') {
-                        const auditPath = (0, path_1.join)(schemaAuditPath, tableEntry.name);
-                        readMetaDb((0, path_1.join)(schemaPath, tableEntry.name), (0, path_1.basename)(tableEntry.name, '.mdb'), schemaEntry.name, auditPath, true);
+                    if (tableEntry.isFile() && (0, node_path_1.extname)(tableEntry.name).toLowerCase() === '.mdb') {
+                        const auditPath = (0, node_path_1.join)(schemaAuditPath, tableEntry.name);
+                        readMetaDb((0, node_path_1.join)(schemaPath, tableEntry.name), (0, node_path_1.basename)(tableEntry.name, '.mdb'), schemaEntry.name, auditPath, true);
                     }
                 }
             }
@@ -577,16 +598,16 @@ function getDatabases() {
                         continue; // reserved restore-metadata dir
                     if (databaseEntry.name === exports.BRANCH_ROOT_DIR)
                         continue; // reserved branch root
-                    if (blockedByRestore.has((0, path_1.basename)(databaseEntry.name, '.mdb')))
+                    if (blockedByRestore.has((0, node_path_1.basename)(databaseEntry.name, '.mdb')))
                         continue;
-                    if (isOpenBranchPath((0, path_1.join)(databasePath, databaseEntry.name)))
+                    if (isOpenBranchPath((0, node_path_1.join)(databasePath, databaseEntry.name)))
                         continue;
-                    if (databaseEntry.isFile() && (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb') {
-                        readMetaDb((0, path_1.join)(databasePath, databaseEntry.name), (0, path_1.basename)(databaseEntry.name, '.mdb'), dbName);
+                    if (databaseEntry.isFile() && (0, node_path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb') {
+                        readMetaDb((0, node_path_1.join)(databasePath, databaseEntry.name), (0, node_path_1.basename)(databaseEntry.name, '.mdb'), dbName);
                     }
                     else {
                         try {
-                            const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
+                            const dbPath = (0, node_path_1.join)(databasePath, databaseEntry.name);
                             const files = (0, node_fs_1.readdirSync)(dbPath, { withFileTypes: true });
                             if (files.find((file) => file.name === 'CURRENT')?.isFile() &&
                                 files.some((file) => file.name.startsWith('MANIFEST-'))) {
@@ -606,7 +627,7 @@ function getDatabases() {
             if (tableConfigs) {
                 for (const tableName in tableConfigs) {
                     const tableConfig = tableConfigs[tableName];
-                    const tablePath = (0, path_1.join)(tableConfig.path, (0, path_1.basename)(tableName + '.mdb'));
+                    const tablePath = (0, node_path_1.join)(tableConfig.path, (0, node_path_1.basename)(tableName + '.mdb'));
                     if ((0, node_fs_1.existsSync)(tablePath)) {
                         readMetaDb(tablePath, tableName, dbName, null, true);
                     }
@@ -843,6 +864,7 @@ function readMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME, au
             rootStore = (0, lmdb_1.open)(envInit);
             lmdbDatabaseEnvs.set(path, rootStore);
         }
+        rootStore.dbisDb?.resetReadTxn();
         return initStores(path, rootStore, databaseName, { defaultTable, auditPath, isLegacy });
     }
     catch (error) {
@@ -885,7 +907,7 @@ function readRocksMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAM
 }
 function initStores(path, rootStore, databaseName, { defaultTable, auditPath, isLegacy, destination, storeName, openedStores } = {}) {
     // a store with no tables never reaches the per-table loop below, and blob roots resolve from this
-    rootStore.databaseName = storeName ?? databaseName;
+    rootStore.databaseName ??= storeName ?? databaseName;
     const envInit = new OpenEnvironmentObject_ts_1.default(path, isReadOnlyMode());
     const internalDbiInit = createOpenDBIObject(false);
     let attributesDbi = rootStore.dbisDb;
@@ -921,6 +943,9 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
                     });
                 }
                 auditStore.isLegacy = true;
+                // A legacy standalone audit root skips openAuditStore, so give it a floor here or it
+                // reports its retention horizon as permanently unknown.
+                (0, auditStore_ts_1.establishAuditFloor)(auditStore);
             }
         }
         else {
@@ -1046,6 +1071,8 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
         let table = tables[tableName];
         // unless its store was migrated to a different engine (e.g. LMDB to RocksDB on startup)
         const recreateForEngineChange = !!table && table.primaryStore?.rootStore instanceof rocksdb_js_1.RocksDatabase !== rootStore instanceof rocksdb_js_1.RocksDatabase;
+        const recreateForTableIdChange = !!table && primaryAttribute.tableId != null && table.tableId !== primaryAttribute.tableId;
+        const recreateTable = recreateForEngineChange || recreateForTableIdChange;
         let indices = {}, existingAttributes = [];
         let tableId;
         let primaryStore;
@@ -1057,7 +1084,9 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
         const cacheControl = primaryAttribute.cacheControl;
         const splitSegments = primaryAttribute.splitSegments;
         const replicate = primaryAttribute.replicate;
-        if (table && !recreateForEngineChange) {
+        if (table && !recreateTable) {
+            if (primaryAttribute.audit === true && table.audit !== true)
+                table.enableAuditing();
             indices = table.indices;
             existingAttributes = table.attributes;
             table.schemaVersion++;
@@ -1108,6 +1137,8 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
                         indices[attribute.name] = dbi;
                         indices[attribute.name].indexNulls = attribute.indexNulls;
                     }
+                    // the only way a thread that never declares the schema reaches Table.indices
+                    indices[attribute.name].isIndexing = !!attribute.indexingPID;
                     const existingAttribute = existingAttributes.find((existingAttribute) => existingAttribute.name === attribute.name);
                     if (existingAttribute)
                         existingAttributes.splice(existingAttributes.indexOf(existingAttribute), 1, attribute);
@@ -1167,13 +1198,15 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
             existingAttributes.splice(existingAttributes.indexOf(existingAttribute), 1);
             attributesUpdated = true;
         }
-        if (table && !recreateForEngineChange) {
+        if (table && !recreateTable) {
             if (attributesUpdated) {
                 table.schemaVersion++;
                 table.updatedAttributes();
             }
         }
         else {
+            if (recreateForTableIdChange)
+                table.cleanup();
             table = setTable(tables, tableName, (0, Table_ts_1.makeTable)({
                 // A branch builds into a caller-owned destination; its tables must refuse DDL.
                 isBranch: Boolean(destination),
@@ -1201,6 +1234,8 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
             if (!destination)
                 exports.databaseEventsEmitter.emit('updateTable', table);
         }
+        void table.derivedIndexRuntime?.close();
+        table.derivedIndexRuntime = (0, hnswDerivedIndex_ts_1.attachDerivedIndexes)(table);
         if (Array.isArray(primaryAttribute.relationships)) {
             relationshipsToHydrate.push({ table, databaseName, tableName, definitions: primaryAttribute.relationships });
         }
@@ -1239,7 +1274,7 @@ function resolveBranchPath(baseName, appName) {
             throw new Error(`Invalid ${label} name for a branch path: ${JSON.stringify(segment)}`);
         }
     }
-    return (0, path_1.join)(resolveDatabaseStorageRoot(baseName), exports.BRANCH_ROOT_DIR, appName, baseName);
+    return (0, node_path_1.join)(resolveDatabaseStorageRoot(baseName), exports.BRANCH_ROOT_DIR, appName, baseName);
 }
 /** `undefined` marks a path reserved by an open still in flight, which owns it just as firmly. */
 const openBranches = new Map();
@@ -1487,6 +1522,8 @@ function openBranchDatabase(path, databaseName, storeName, blobRoots) {
     openBranches.set(path, undefined);
     retakeBranchIdentity(storeName);
     try {
+        // before the open: table load schedules TTL, eviction and audit cleanup, which ask who owns this store
+        manageThreads.markBranchStorePath(path);
         rootStore = readRocksMetaDb(path, null, databaseName, { destination: tables, storeName, openedStores });
         // Pin the handle to the roots the caller proved this branch was published with, before it is
         // handed out. A row's `storageIndex` is a position in that list, so resolving through current
@@ -1498,6 +1535,7 @@ function openBranchDatabase(path, databaseName, storeName, blobRoots) {
     }
     catch (error) {
         openBranches.delete(path);
+        manageThreads.markBranchStorePath(path, false);
         releaseBranchIdentity(storeName);
         const stranded = rocksdbDatabaseEnvs.get(path);
         rocksdbDatabaseEnvs.delete(path);
@@ -1522,6 +1560,7 @@ function openBranchDatabase(path, databaseName, storeName, blobRoots) {
             openBranches.delete(path);
             releaseBranchIdentity(storeName);
             rocksdbDatabaseEnvs.delete(path);
+            manageThreads.markBranchStorePath(path, false);
             closeBranchHandles(path, rootStore, openedStores, tables);
         },
     };
@@ -1647,10 +1686,10 @@ function resolveDatabaseStorageRoot(databaseName, tableName) {
         databaseConfig[databaseName]?.path ||
         process.env.STORAGE_PATH ||
         (0, configUtils_ts_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
-        (hdbBasePath && (0, node_fs_1.existsSync)((0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
-            ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME)
+        (hdbBasePath && (0, node_fs_1.existsSync)((0, node_path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
+            ? (0, node_path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME)
             : hdbBasePath
-                ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)
+                ? (0, node_path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)
                 : undefined);
     if (!databasePath) {
         throw new Error(`Unable to determine database storage path. Ensure STORAGE_PATH, HDB_ROOT, or a valid config path is set.`);
@@ -1661,7 +1700,7 @@ function resolveDatabaseStorageRoot(databaseName, tableName) {
  * Resolve the directory path of a RocksDB database (whether or not it exists or is loaded).
  */
 function resolveDatabasePath(databaseName) {
-    return (0, path_1.join)(resolveDatabaseStorageRoot(databaseName), databaseName);
+    return (0, node_path_1.join)(resolveDatabaseStorageRoot(databaseName), databaseName);
 }
 /**
  * Get root store for a database
@@ -1686,7 +1725,7 @@ function database({ database: databaseName, table: tableName }) {
     let rootStore;
     const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
     if (useRocksdb) {
-        const path = (0, path_1.join)(databasePath, tablePath ? tableName : databaseName);
+        const path = (0, node_path_1.join)(databasePath, tablePath ? tableName : databaseName);
         // the scan is not the only way to reach a branch's directory: a branch leaves its store in
         // `rocksdbDatabaseEnvs`, so without this an on-demand open would staple it onto
         // `definedDatabases` and the next `closeDatabase` would close it under the live handle
@@ -1709,7 +1748,7 @@ function database({ database: databaseName, table: tableName }) {
         }
     }
     else {
-        const path = (0, path_1.join)(databasePath, `${tablePath ? tableName : databaseName}.mdb`);
+        const path = (0, node_path_1.join)(databasePath, `${tablePath ? tableName : databaseName}.mdb`);
         rootStore = lmdbDatabaseEnvs.get(path);
         if (!rootStore || rootStore.status === 'closed') {
             // TODO: validate database name
@@ -2062,6 +2101,9 @@ function openIndex(dbiKey, rootStore, attribute) {
         const CustomIndex = customIndexes_ts_1.CUSTOM_INDEXES[attribute.indexed.type];
         if (CustomIndex) {
             indexStore.customIndex = new CustomIndex(indexStore, attribute.indexed);
+            // derived state whose maintaining option is now off must not linger to be adopted
+            // stale on a later re-enable
+            indexStore.customIndex.cleanupDisabledPlane?.();
         }
         else {
             logger.error(`The indexing type '${attribute.indexed.type}' is unknown`);
@@ -2096,14 +2138,17 @@ const GLOBAL_TARGET = {
 /**
  * The factory a branched application declares tables through: each declaration goes to the branch
  * of the database it names, or to `table()` itself for a database the application did not branch.
- * An unbranched application gets `table` by identity -- no wrapper, no per-call routing.
+ * An unbranched, shared application gets `table` by identity. An isolated application still gets a
+ * wrapper so its own declarations can claim their single-threaded maintenance work.
  */
-function scopedTableFactory(branches) {
-    if (!branches?.size)
+function scopedTableFactory(branches, isolatedApplicationOwner = false) {
+    if (!branches?.size && !isolatedApplicationOwner)
         return table;
     return function scopedTable(tableDefinition) {
+        if (isolatedApplicationOwner)
+            tableDefinition = { ...tableDefinition, isolatedApplicationOwner: true };
         // `||`, not `??`: `table()` resolves every falsy name to the default database
-        const branch = branches.get(tableDefinition.database || DEFAULT_DATABASE_NAME);
+        const branch = branches?.get(tableDefinition.database || DEFAULT_DATABASE_NAME);
         return branch ? declareTable(branchTarget(branch), tableDefinition) : table(tableDefinition);
     };
 }
@@ -2160,7 +2205,9 @@ function reloadBranchAt(path) {
     return branch.tables;
 }
 function declareTable(target, tableDefinition) {
-    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, randomAccessFields, trackDeletes, schemaDefined, schemaRelationshipsDefined, origin, description, properties, hidden, cacheControl, } = tableDefinition;
+    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, randomAccessFields, trackDeletes, schemaDefined, schemaRelationshipsDefined, origin, description, properties, hidden, cacheControl, isolatedApplicationOwner, } = tableDefinition;
+    const auditExplicitlyEnabled = audit === true;
+    const auditExplicitlyDisabled = audit === false;
     if (!databaseName)
         databaseName = DEFAULT_DATABASE_NAME;
     // Reject reserved names here too, not only at the operations API: a database
@@ -2197,27 +2244,137 @@ function declareTable(target, tableDefinition) {
         schemaDefined = true;
     const relationshipDefinitions = schemaRelationshipsDefined ? normalizeRelationships(attributes) : undefined;
     const internalDbiInit = createOpenDBIObject(false);
-    for (const attribute of attributes) {
-        if (attribute.attribute && !attribute.name) {
-            // there is some legacy code that calls the attribute's name the attribute's attribute
-            attribute.name = attribute.attribute;
-            attribute.indexed = true;
+    let releaseExclusiveLock;
+    const hasHnswAtEntry = attributes.some((attribute) => attribute.indexed?.type === 'HNSW');
+    const persistedPrimaryDescriptor = (catalog) => {
+        const declaredPrimaryKey = attributes.find((attribute) => attribute.isPrimaryKey)?.name ?? Table?.primaryKey;
+        if (declaredPrimaryKey) {
+            const key = `${tableName}/${declaredPrimaryKey}`;
+            const descriptor = catalog?.getSync(key);
+            if (descriptor?.isPrimaryKey)
+                return { key, descriptor };
+        }
+        const key = `${tableName}/`;
+        return { key, descriptor: catalog?.getSync(key) };
+    };
+    const hasLegacyHnswStateAtEntry = Table &&
+        origin !== 'cluster' &&
+        attributes.some((attribute) => {
+            if (attribute.indexed?.type !== 'HNSW')
+                return false;
+            const persisted = Table.dbisDB?.getSync(`${tableName}/${attribute.name || attribute.attribute || ''}`)?.indexed;
+            if (persisted?.type !== 'HNSW')
+                return false;
+            if (Object.hasOwn(persisted, 'nativePlane') && typeof persisted.nativePlane !== 'boolean')
+                return true;
+            for (const name of customIndexes_ts_1.CUSTOM_INDEXES.HNSW.numericOptions)
+                if (Object.hasOwn(persisted, name) && typeof persisted[name] !== 'number')
+                    return true;
+            return false;
+        });
+    try {
+        if (Table &&
+            hasHnswAtEntry &&
+            origin !== 'cluster' &&
+            (rootStore instanceof rocksdb_js_1.RocksDatabase || hasLegacyHnswStateAtEntry))
+            exclusiveLock();
+        const persistedAuditAtEntry = hasHnswAtEntry || (Table && Table.audit !== true)
+            ? persistedPrimaryDescriptor(Table?.dbisDB).descriptor?.audit
+            : undefined;
+        if (!auditExplicitlyDisabled && persistedAuditAtEntry === true && Table?.audit !== true)
+            Table.enableAuditing();
+        for (const attribute of attributes) {
+            if (attribute.attribute && !attribute.name) {
+                // there is some legacy code that calls the attribute's name the attribute's attribute
+                attribute.name = attribute.attribute;
+                attribute.indexed = true;
+            }
+            else
+                attribute.attribute = attribute.name;
+            if (attribute.expiresAt)
+                attribute.indexed = true;
+            if (attribute.indexed?.type === 'HNSW' && origin !== 'cluster') {
+                const existingAttribute = Table?.attributes.find((existing) => existing.name === attribute.name);
+                const persistedIndexed = Table?.dbisDB?.getSync(`${tableName}/${attribute.name || ''}`)?.indexed ?? existingAttribute?.indexed;
+                customIndexes_ts_1.CUSTOM_INDEXES.HNSW.normalizeDeclarationOptions(attribute.indexed, persistedIndexed);
+                if (attribute.indexed.nativePlane != null) {
+                    const persistedNativePlane = persistedIndexed?.nativePlane;
+                    const matchesPersistedLegacySpelling = persistedIndexed?.type === 'HNSW' &&
+                        Object.hasOwn(persistedIndexed, 'nativePlane') &&
+                        typeof persistedNativePlane !== 'boolean' &&
+                        (Object.is(persistedNativePlane, attribute.indexed.nativePlane) ||
+                            (typeof persistedNativePlane === 'string' &&
+                                typeof attribute.indexed.nativePlane === 'number' &&
+                                persistedNativePlane.trim() !== '' &&
+                                Number(persistedNativePlane) === attribute.indexed.nativePlane));
+                    if (matchesPersistedLegacySpelling) {
+                        attribute.indexed.nativePlane = persistedNativePlane;
+                    }
+                    else {
+                        attribute.indexed.nativePlane = customIndexes_ts_1.CUSTOM_INDEXES.HNSW.normalizeNativePlaneDeclaration(attribute.indexed.nativePlane);
+                    }
+                }
+            }
+        }
+        const auditEnabledAtEntry = auditExplicitlyEnabled ||
+            (!auditExplicitlyDisabled &&
+                (persistedAuditAtEntry === true || (persistedAuditAtEntry == null && Table?.audit === true)));
+        if (origin !== 'cluster' &&
+            attributes.some((attribute) => {
+                if (attribute.indexed?.type !== 'HNSW')
+                    return false;
+                if (attribute.indexed.nativePlane != null)
+                    return Boolean(attribute.indexed.nativePlane);
+                const existingAttribute = Table?.attributes.find((existing) => existing.name === attribute.name && existing.indexed?.type === 'HNSW');
+                return Boolean(existingAttribute?.indexed.nativePlane);
+            }) &&
+            !auditEnabledAtEntry) {
+            throw new hdbError_ts_1.ClientError(`Table '${databaseName}.${tableName}' must enable audit logging before using nativePlane because its transaction log is the derived-index recovery source; set nativePlane: false to use the JS index`);
         }
-        else
-            attribute.attribute = attribute.name;
-        if (attribute.expiresAt)
-            attribute.indexed = true;
     }
+    catch (error) {
+        releaseLock();
+        throw error;
+    }
+    const validateHnswOptions = (catalog, auditQualifiesDefault) => {
+        for (const attribute of attributes) {
+            const indexed = attribute.indexed;
+            if (indexed?.type !== 'HNSW')
+                continue;
+            const persistedIndexed = catalog?.getSync(`${tableName}/${attribute.name || ''}`)?.indexed;
+            if (indexed.nativePlane) {
+                customIndexes_ts_1.CUSTOM_INDEXES.HNSW.validateNativePlaneOptions(rootStore, indexed);
+                continue;
+            }
+            if (indexed.nativePlane != null)
+                continue;
+            if (persistedIndexed?.type === 'HNSW' && Object.hasOwn(persistedIndexed, 'nativePlane')) {
+                if (persistedIndexed.nativePlane)
+                    customIndexes_ts_1.CUSTOM_INDEXES.HNSW.validateNativePlaneOptions(rootStore, {
+                        ...indexed,
+                        nativePlane: persistedIndexed.nativePlane,
+                    });
+                continue;
+            }
+            if (!auditQualifiesDefault)
+                continue;
+            if (persistedIndexed?.type !== 'HNSW')
+                customIndexes_ts_1.CUSTOM_INDEXES.HNSW.canDefaultToNativePlane(rootStore, indexed);
+        }
+    };
+    if (!Table && origin !== 'cluster')
+        validateHnswOptions(undefined, auditExplicitlyEnabled);
     let hasChanges;
     let refreshRelationshipAttributes = false;
+    let refreshedLiveAttributes = false;
     let deferredPrimaryRow;
     let unpublishedPrimaryStore;
     let published = false;
-    let releaseExclusiveLock;
     const attributesToIndex = [];
     const indicesToRemove = [];
     try {
         if (Table) {
+            refreshedLiveAttributes = true;
             primaryKey = Table.primaryKey;
             if (Table.primaryStore.rootStore.status === 'closed') {
                 throw new Error(`Can not use a closed data store from ${tableName} class`);
@@ -2243,11 +2400,15 @@ function declareTable(target, tableDefinition) {
                 }
             }
             // Acquire before the first mutation of the live Table below, so a lost race leaves no
-            // attributes this worker describes but never persisted. Only the RocksDB acquire is bounded
-            // and can throw, and only it is cheap when uncontended: LMDB's exclusiveLock() opens an
-            // environment-wide write transaction that cannot time out, so it stays lazy.
+            // attributes this worker describes but never persisted. Only the RocksDB acquire is bounded;
+            // ordinary LMDB declarations stay lazy, while legacy HNSW normalization locks at entry.
             if (rootStore instanceof rocksdb_js_1.RocksDatabase)
                 exclusiveLock();
+            if (origin !== 'cluster') {
+                const lockedAttributesDbi = Table.dbisDB;
+                const persistedAuditUnderLock = persistedPrimaryDescriptor(lockedAttributesDbi).descriptor?.audit;
+                validateHnswOptions(lockedAttributesDbi, auditExplicitlyEnabled || (!auditExplicitlyDisabled && persistedAuditUnderLock === true));
+            }
             // it table already exists, get the split segments setting
             if (splitSegments == undefined)
                 splitSegments = Table.splitSegments;
@@ -2271,6 +2432,13 @@ function declareTable(target, tableDefinition) {
                 }
                 attributes = merged;
             }
+            else if (!attributes.some((attribute) => attribute.isPrimaryKey)) {
+                const existingPrimary = Table.attributes.find((attribute) => attribute.isPrimaryKey);
+                if (existingPrimary && attributes.some((attribute) => attribute.name === existingPrimary.name))
+                    throw new hdbError_ts_1.ClientError(`Cannot remove the primary key designation from '${databaseName}.${tableName}.${existingPrimary.name}'`);
+                if (existingPrimary)
+                    attributes = [existingPrimary, ...attributes];
+            }
             Table.attributes.splice(0, Table.attributes.length, ...attributes);
             // Re-assert from the live declaration so a stale value on disk (replicated event,
             // v4-era backfill) is corrected on every reload. Gated on `schemaDefinedExplicit` so
@@ -2470,7 +2638,7 @@ function declareTable(target, tableDefinition) {
             const attribute = attributes.find((attribute) => attribute.name === attribute_name);
             const removeIndex = !attribute?.indexed && value.indexed && !value.isPrimaryKey;
             // rows already present under a create are aborted state
-            const staleRow = !attribute || Boolean(deferredPrimaryRow);
+            const staleRow = (!attribute && !value.isPrimaryKey) || Boolean(deferredPrimaryRow);
             if (staleRow || removeIndex) {
                 exclusiveLock();
                 hasChanges = true;
@@ -2483,9 +2651,75 @@ function declareTable(target, tableDefinition) {
                 }
             }
         }
+        const hasHnswDeclaration = attributes.some((attribute) => attribute.indexed?.type === 'HNSW');
+        const persistedAudit = hasHnswDeclaration ? persistedPrimaryDescriptor(attributesDbi).descriptor?.audit : undefined;
+        // A cluster declaration can apply audit on a create, but deliberately cannot rewrite an existing
+        // table's primary row. Do not let an incoming audit value qualify a replicated native descriptor
+        // that this node would then persist beside its durable audit:false row.
+        const explicitAuditCanBeApplied = origin !== 'cluster' || Boolean(deferredPrimaryRow);
+        const auditEnabledForNativeDefault = (auditExplicitlyEnabled && explicitAuditCanBeApplied) || (!auditExplicitlyDisabled && persistedAudit === true);
+        const auditEnabledForNativePlane = !auditExplicitlyDisabled && (auditEnabledForNativeDefault || (persistedAudit == null && Table.audit === true));
+        for (const attribute of attributes) {
+            const indexed = attribute.indexed;
+            if (!indexed || typeof indexed !== 'object' || indexed.type !== 'HNSW')
+                continue;
+            const descriptor = attributesDbi.getSync(tableName + '/' + (attribute.name || ''));
+            const existingHnsw = descriptor?.indexed?.type === 'HNSW';
+            if (indexed.nativePlane == null) {
+                if (existingHnsw) {
+                    if (Object.hasOwn(descriptor.indexed, 'nativePlane')) {
+                        indexed.nativePlane = descriptor.indexed.nativePlane;
+                    }
+                }
+                else if (origin !== 'cluster' &&
+                    auditEnabledForNativeDefault &&
+                    customIndexes_ts_1.CUSTOM_INDEXES.HNSW.canDefaultToNativePlane(rootStore, indexed)) {
+                    indexed.nativePlane = true;
+                }
+            }
+            else if (origin === 'cluster' && !existingHnsw && indexed.nativePlane) {
+                let canRunNative = false;
+                try {
+                    canRunNative = auditEnabledForNativeDefault && customIndexes_ts_1.CUSTOM_INDEXES.HNSW.canRunNativePlane(rootStore, indexed);
+                }
+                catch { }
+                if (!canRunNative) {
+                    logger.warn(`Using the JS HNSW index for replicated attribute ${databaseName}.${tableName}.${attribute.name} because this node does not satisfy the nativePlane requirements`);
+                    indexed.nativePlane = false;
+                }
+            }
+        }
+        const nativePlaneEnabled = origin !== 'cluster' &&
+            attributes.some((attribute) => attribute.indexed?.type === 'HNSW' && attribute.indexed.nativePlane);
+        if (nativePlaneEnabled && !auditEnabledForNativePlane) {
+            throw new hdbError_ts_1.ClientError(`Table '${databaseName}.${tableName}' must enable audit logging before using nativePlane because its transaction log is the derived-index recovery source; set nativePlane: false to use the JS index`);
+        }
+        if (nativePlaneEnabled && persistedAudit !== true)
+            audit = true;
+        if (nativePlaneEnabled && persistedAudit !== true && !attributes.some((attribute) => attribute.isPrimaryKey)) {
+            exclusiveLock();
+            const primaryKey = primaryDescriptorKey();
+            const primaryDescriptor = attributesDbi.getSync(primaryKey);
+            if (primaryDescriptor && !tableIsDropping(primaryDescriptor, primaryKey)) {
+                Table.enableAuditing();
+                attributesDbi.put(primaryKey, { ...primaryDescriptor, audit: true });
+                hasChanges = true;
+            }
+        }
         // TODO: If we have attributes and the schemaDefined flag is not set, turn it on
         // iterate through the attributes to ensure that we have all the dbis created and indexed
-        for (const attribute of attributes || []) {
+        const attributesInPersistenceOrder = nativePlaneEnabled
+            ? [
+                ...attributes.filter((attribute) => attribute.isPrimaryKey),
+                ...attributes.filter((attribute) => !attribute.isPrimaryKey),
+            ]
+            : auditExplicitlyDisabled
+                ? [
+                    ...attributes.filter((attribute) => !attribute.isPrimaryKey),
+                    ...attributes.filter((attribute) => attribute.isPrimaryKey),
+                ]
+                : attributes;
+        for (const attribute of attributesInPersistenceOrder) {
             if (attribute.relationship) {
                 refreshRelationshipAttributes = true;
                 continue;
@@ -2509,7 +2743,7 @@ function declareTable(target, tableDefinition) {
                 // primary key can't change indexing, but settings can change
                 if (origin !== 'cluster' &&
                     (schemaDefinedMismatch ||
-                        (audit !== undefined && audit !== Table.audit) ||
+                        (typeof audit === 'boolean' && audit !== attributeDescriptor.audit) ||
                         (sealed !== undefined && sealed !== Table.sealed) ||
                         (replicate !== undefined && replicate !== Table.replicate) ||
                         (+expiration || undefined) !== (+attributeDescriptor.expiration || undefined) ||
@@ -2550,8 +2784,7 @@ function declareTable(target, tableDefinition) {
                 applyDurableDeclaration(attribute, attributeDescriptor);
                 const abandonedIndexBuild = attribute.indexed &&
                     (attributeDescriptor.indexingFailed ||
-                        (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
-                        attributeDescriptor.restartNumber < (worker_threads_1.workerData?.restartNumber ?? manageThreads.restartNumber));
+                        isAbandonedIndexBuild(attributeDescriptor, worker_threads_1.workerData?.restartNumber ?? manageThreads.restartNumber));
                 if (abandonedIndexBuild) {
                     // Recovery is the exception to skipping the handling below, because without it `isIndexing`
                     // stays pinned on with nothing left to clear it and every query on the attribute fails with
@@ -2640,15 +2873,13 @@ function declareTable(target, tableDefinition) {
                 if (changed ||
                     indexFormatNeedsPersist ||
                     attributeDescriptor?.indexingFailed ||
-                    (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
-                    attributeDescriptor?.restartNumber < currentRestartGeneration) {
+                    isAbandonedIndexBuild(attributeDescriptor, currentRestartGeneration)) {
                     hasChanges = true;
                     exclusiveLock();
                     attributeDescriptor = attributesDbi.getSync(dbiKey);
                     if (structurallyChanged ||
                         attributeDescriptor?.indexingFailed ||
-                        (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
-                        attributeDescriptor?.restartNumber < currentRestartGeneration) {
+                        isAbandonedIndexBuild(attributeDescriptor, currentRestartGeneration)) {
                         hasChanges = true;
                         if (attribute.indexNulls === undefined)
                             attribute.indexNulls = true;
@@ -2667,9 +2898,20 @@ function declareTable(target, tableDefinition) {
                             // representation-only option difference — preserve the checkpoint so the backfill
                             // resumes rather than restarts. Canonicalized to match structurallyChanged above.
                             const indexOptionsChanged = canonicalIndexKey(attributeDescriptor?.indexed) !== canonicalIndexKey(attribute.indexed);
-                            attribute.lastIndexedKey = indexOptionsChanged
-                                ? undefined
-                                : (attributeDescriptor?.lastIndexedKey ?? undefined);
+                            // Only a checkpoint runIndexing stamped with its own key resumes: earlier releases advanced
+                            // lastIndexedKey past failed and unflushed index writes, so any other is a full rebuild.
+                            const uncertifiedCheckpoint = attributeDescriptor?.lastIndexedKey !== undefined &&
+                                (attributeDescriptor.checkpointCertified === undefined ||
+                                    attributeDescriptor.checkpointAlgorithm !== exports.CHECKPOINT_ALGORITHM ||
+                                    (0, lmdb_1.compareKeys)(attributeDescriptor.checkpointCertified, attributeDescriptor.lastIndexedKey) !== 0);
+                            attribute.lastIndexedKey =
+                                indexOptionsChanged || uncertifiedCheckpoint
+                                    ? undefined
+                                    : (attributeDescriptor?.lastIndexedKey ?? undefined);
+                            if (attribute.lastIndexedKey !== undefined) {
+                                attribute.checkpointCertified = attribute.lastIndexedKey;
+                                attribute.checkpointAlgorithm = exports.CHECKPOINT_ALGORITHM;
+                            }
                             // Explicit reindex is the upgrade path from a legacy (un-versioned) custom-index
                             // object store to the versioned, VT-cacheable format. A full rebuild from scratch
                             // (lastIndexedKey === undefined) clears the store and rewrites every node, so the
@@ -2691,6 +2933,9 @@ function declareTable(target, tableDefinition) {
                             // the new process reuses the old PID. Cleared on clean completion; left in place
                             // on failure/crash so the next, higher-numbered restart re-triggers the backfill.
                             attribute.restartNumber = currentRestartGeneration;
+                            if (manageThreads.processIncarnation != null)
+                                attribute.indexingIncarnation = manageThreads.processIncarnation;
+                            attribute.indexingBuildId = (0, node_crypto_1.randomBytes)(8).toString('hex');
                             delete attribute.indexingFailed; // clear failure flag for the new run
                             dbi.isIndexing = true;
                             Object.defineProperty(attribute, 'dbi', { value: dbi, configurable: true, enumerable: false });
@@ -2706,6 +2951,12 @@ function declareTable(target, tableDefinition) {
                                 reindexReasons.push(`crash-recovery(pid=${attributeDescriptor.indexingPID})`);
                             if (attributeDescriptor?.restartNumber < currentRestartGeneration)
                                 reindexReasons.push('restart-number');
+                            if (uncertifiedCheckpoint)
+                                reindexReasons.push('uncertified-checkpoint');
+                            if (attributeDescriptor?.indexingPID === process.pid &&
+                                manageThreads.processIncarnation != null &&
+                                attributeDescriptor.indexingIncarnation !== manageThreads.processIncarnation)
+                                reindexReasons.push('abandoned-build(previous process incarnation)');
                             logger.info(`reindex ${databaseName}.${tableName}.${attribute.name}: reason=${reindexReasons.join(',') || 'unknown'}`);
                             // we only set indexing nulls to true if new or reindexing, we can't have partial indexing of null
                             attributesToIndex.push(attribute);
@@ -2718,12 +2969,22 @@ function declareTable(target, tableDefinition) {
                         // workers / a reload would treat the still-partial index as ready and return incomplete results.
                         attribute.indexingPID = attributeDescriptor.indexingPID;
                         attribute.lastIndexedKey = attributeDescriptor.lastIndexedKey;
+                        if (attributeDescriptor.checkpointCertified !== undefined) {
+                            attribute.checkpointCertified = attributeDescriptor.checkpointCertified;
+                            attribute.checkpointAlgorithm = attributeDescriptor.checkpointAlgorithm;
+                        }
                         // Carry the in-progress restart generation too, so persisting this metadata-only
                         // change doesn't drop it and break the crash-recovery trigger for the running backfill.
                         attribute.restartNumber = attributeDescriptor.restartNumber;
+                        attribute.indexingIncarnation = attributeDescriptor.indexingIncarnation;
+                        attribute.indexingBuildId = attributeDescriptor.indexingBuildId;
                         if (attributeDescriptor.indexingFailed)
                             attribute.indexingFailed = attributeDescriptor.indexingFailed;
                     }
+                    // The declared attribute never carries the stamp, so any rewrite of a descriptor that has
+                    // one would drop it and make a completed index look like a pre-stamp build.
+                    if (attribute.checkpointAlgorithm === undefined && attributeDescriptor?.checkpointAlgorithm !== undefined)
+                        attribute.checkpointAlgorithm = attributeDescriptor.checkpointAlgorithm;
                     attributesDbi.put(dbiKey, attribute);
                 }
                 // If a migration is in progress (indexingPID set), any newly opened dbi must also
@@ -2784,32 +3045,39 @@ function declareTable(target, tableDefinition) {
     finally {
         releaseLock();
     }
-    if (hasChanges || refreshRelationshipAttributes) {
+    if (hasChanges || refreshRelationshipAttributes)
         Table.schemaVersion++;
+    if (hasChanges || refreshRelationshipAttributes || refreshedLiveAttributes)
         Table.updatedAttributes();
-    }
     logger.trace(`${tableName} table loading, running index`);
     const branchPath = target.branch?.path;
     if (attributesToIndex.length > 0 || indicesToRemove.length > 0) {
-        Table.indexingOperation = runIndexing(Table, attributesToIndex, indicesToRemove, branchPath);
+        // captured before the backfill can rewrite the attributes
+        const buildIds = new Map(attributesToIndex.map((attribute) => [attribute, attribute.indexingBuildId]));
+        const markSettled = () => markAbandonedIndexBuild(Table, rootStore, buildIds);
+        Table.indexingOperation = runIndexing(Table, attributesToIndex, indicesToRemove, branchPath).then(markSettled, markSettled);
     }
     else if (hasChanges)
         signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName, undefined, branchPath));
+    void Table.derivedIndexRuntime?.close();
+    Table.derivedIndexRuntime = (0, hnswDerivedIndex_ts_1.attachDerivedIndexes)(Table);
     Table.origin = origin;
     // scope-private: replication and other global subscribers must not learn of a branch class
     if ((hasChanges || refreshRelationshipAttributes) && !target.branch) {
         exports.databaseEventsEmitter.emit('updateTable', Table, origin !== 'cluster');
     }
-    if (expiration || eviction || scanInterval)
+    if (expiration || eviction || scanInterval || attributes.some((attribute) => attribute.expiresAt))
         Table.setTTLExpiration({
             expiration,
             eviction,
             scanInterval,
+            fromSchema: true,
+            isolatedApplicationOwner,
         });
     logger.trace(`${tableName} table loaded`);
     return Table;
-    // dropTable() tombstones the bare table row, which is not the row a legacy catalog keeps the
-    // table's settings in, so a drop in flight has to be checked on both.
+    // A migrated catalog can retain a named primary descriptor beside a bare table tombstone, so a
+    // drop in flight has to be checked on both representations.
     function tableIsDropping(descriptor, descriptorKey) {
         if (descriptor?.dropping)
             return true;
@@ -2818,13 +3086,7 @@ function declareTable(target, tableDefinition) {
     // The catalog row initStores() reads a table's settings from: the primary key's own row when it
     // has one, and the bare table row otherwise.
     function primaryDescriptorKey() {
-        const declaredPrimaryKey = attributes?.find((attribute) => attribute.isPrimaryKey)?.name;
-        if (declaredPrimaryKey) {
-            const attributeKey = tableName + '/' + declaredPrimaryKey;
-            if (attributesDbi.getSync(attributeKey))
-                return attributeKey;
-        }
-        return tableName + '/';
+        return persistedPrimaryDescriptor(attributesDbi).key;
     }
     // The catalog of a published table stays, but a class the registration never accepted is
     // unreachable, so release what makeTable() registered process-wide instead of leaving its timers
@@ -2892,73 +3154,238 @@ function declareTable(target, tableDefinition) {
     }
 }
 /**
- * Canonical form used ONLY for the structural (reindex-triggering) comparison of index options.
- * `@indexed(...)` records options in source-argument order and as strings, while the operations API
- * and config objects can supply them reordered or as numbers; without canonicalizing, such a
- * representation-only difference flips the structural comparison and forces a needless full rebuild
- * (clearing + rebuilding the index, 503-ing the attribute throughout) for a semantically identical
- * index. Sorts object keys and coerces numeric-looking (non-zero) string scalars to numbers.
- * Conservative by design: boolean-vs-object, absent-vs-present, and string-"0"-vs-number-0
- * differences are all preserved, so a genuine change (`true` vs `{ type: 'HNSW' }`, an added/removed
- * option, a changed value) still triggers a rebuild. Persistence keys off the raw form, so the stored
- * descriptor self-heals toward this shape over time. harper#1357
+ * Stable structural form for deciding whether an index must be rebuilt. `coerceZero` extends numeric
+ * coercion to zero; a truthiness-sensitive numeric option must normalize its value before using it.
  */
-function canonicalizeIndexOptions(value) {
+function canonicalizeIndexOptions(value, coerceZero = false) {
     if (Array.isArray(value))
-        return value.map(canonicalizeIndexOptions);
+        return value.map((item) => canonicalizeIndexOptions(item, coerceZero));
     if (value && typeof value === 'object') {
         const canonical = {};
-        for (const key of Object.keys(value).sort())
-            canonical[key] = canonicalizeIndexOptions(value[key]);
+        const customIndex = value.type && customIndexes_ts_1.CUSTOM_INDEXES[value.type];
+        for (const key of Object.keys(value).sort()) {
+            if (customIndex?.truthyStructuralOptions?.has(key)) {
+                if (value[key])
+                    canonical[key] = true;
+                continue;
+            }
+            const optionValue = customIndex?.normalizeOptionValue
+                ? customIndex.normalizeOptionValue(key, value[key])
+                : value[key];
+            canonical[key] = canonicalizeIndexOptions(optionValue, coerceZero || Boolean(customIndex?.numericOptions?.has(key)));
+        }
         return canonical;
     }
-    // Coerce numeric-looking strings ("16" -> 16) so string-vs-number representations of the same
-    // option compare equal — EXCEPT zero: the string "0" is truthy while the number 0 is falsy, and
-    // index code may branch on truthiness (e.g. HNSW `if (this.optimizeRouting)` doubles maxConnections),
-    // so "0" and 0 build structurally different indexes and must still trigger a rebuild. Zero is the
-    // only finite number whose string and numeric forms diverge in truthiness, so excluding it fully
-    // closes that gap. Leave non-numeric strings, booleans, null, etc. intact.
     if (typeof value === 'string' && value.trim() !== '') {
         const numeric = Number(value);
-        if (numeric !== 0 && Number.isFinite(numeric))
+        if ((numeric !== 0 || coerceZero) && Number.isFinite(numeric))
             return numeric;
     }
     return value;
 }
+// Bumped when a change alters which keys a checkpoint may certify. A descriptor stamped by any other
+// version resumes as uncertified (full rebuild) rather than being trusted, and a completed index keeps
+// the stamp of the build that wrote it.
+exports.CHECKPOINT_ALGORITHM = 2;
 const MAX_OUTSTANDING_INDEXING = 1000;
 const MIN_OUTSTANDING_INDEXING = 10;
+const INDEXING_YIELD_INTERVAL = 100;
+// A resumable checkpoint is written only after a flush (see flushIndexStores), at most once per period
+// and never before this many more records: the flush seals every column family in the database, so a
+// slow backfill must not impose the period's flush rate on unrelated tables.
+let indexingCheckpointPeriodMs = 5000;
+let indexingCheckpointMinRecords = 10000;
+function setIndexingCheckpointPeriod(ms, minRecords = indexingCheckpointMinRecords) {
+    const previous = { ms: indexingCheckpointPeriodMs, minRecords: indexingCheckpointMinRecords };
+    indexingCheckpointPeriodMs = ms;
+    indexingCheckpointMinRecords = minRecords;
+    return previous;
+}
+const yieldEventTurn = () => new Promise((resolve) => setImmediate(resolve));
+// RocksDB index stores have no WAL (openRocksDatabase defaults disableWAL), so a flush is what makes the
+// entries a checkpoint certifies durable. A flush only covers writes issued before it started, so a caller
+// never joins one in flight: it joins the next one, which every backfill on that database asking meanwhile
+// shares — at most one in flight and one queued.
+const indexingFlushes = new WeakMap();
+function flushIndexStores(rootStore) {
+    if (!(rootStore instanceof rocksdb_js_1.RocksDatabase))
+        return;
+    let flushes = indexingFlushes.get(rootStore);
+    if (!flushes)
+        indexingFlushes.set(rootStore, (flushes = {}));
+    if (flushes.queued)
+        return flushes.queued;
+    const start = () => {
+        flushes.queued = undefined;
+        const flush = rootStore.flush().finally(() => {
+            if (flushes.inFlight === flush)
+                flushes.inFlight = undefined;
+        });
+        flushes.inFlight = flush;
+        return flush;
+    };
+    if (!flushes.inFlight)
+        return start();
+    return (flushes.queued = flushes.inFlight.then(start, start));
+}
+function resumeStartKey(attributes) {
+    let start;
+    for (const attribute of attributes) {
+        if (attribute.lastIndexedKey == undefined)
+            return undefined;
+        if (start === undefined || (0, lmdb_1.compareKeys)(attribute.lastIndexedKey, start) < 0)
+            start = attribute.lastIndexedKey;
+    }
+    return start;
+}
+/**
+ * Persists the failure marker for a build that ended without running one of runIndexing's own exit
+ * paths, so something re-triggers it. Fenced on `indexingBuildId` inside the storage engine's catalog
+ * serialization boundary, because a replacement generation (or another thread declaring different index
+ * options) can claim the attribute before an outgoing build's promise settles, and marking that would fail
+ * a live build. The fence read and write stay synchronous, and nothing here may throw because
+ * `Table.indexingOperation` reaches operations-API callers.
+ */
+async function markAbandonedIndexBuild(Table, rootStore, buildIds) {
+    for (const [attribute, buildId] of buildIds) {
+        try {
+            let marked;
+            if (buildId == null || Table.dbisDB.getSync(attribute.key)?.indexingBuildId !== buildId)
+                continue;
+            const markIfOwned = () => {
+                const descriptor = Table.dbisDB.getSync(attribute.key);
+                if (descriptor?.indexingBuildId === buildId && !descriptor.indexingFailed) {
+                    Table.dbisDB.putSync(attribute.key, { ...descriptor, indexingFailed: true });
+                    marked = true;
+                }
+            };
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                (0, Table_ts_1.acquireUpdateAttributesLock)(rootStore, `abandoned index build '${Table.tableName}.${attribute.name}'`);
+                try {
+                    markIfOwned();
+                }
+                finally {
+                    (0, Table_ts_1.releaseUpdateAttributesLock)(rootStore);
+                }
+            }
+            else {
+                rootStore.transactionSync(markIfOwned);
+            }
+            if (marked)
+                logger.warn(`Indexing of ${Table.databaseName}.${Table.tableName}.${attribute.name} ended without completing. ` +
+                    `The index stays incomplete and every query on the attribute reports it as not indexed yet; ` +
+                    `the next load of the table retries the backfill from the last checkpoint (indexingFailed=true).`);
+        }
+        catch (error) {
+            // A store closed by shutdown is the common case, and it cannot be written to at all.
+            try {
+                logger.debug(`Could not mark the abandoned index build of ${Table.tableName}.${attribute.name}`, error);
+            }
+            catch { }
+        }
+    }
+}
 async function runIndexing(Table, attributes, indicesToRemove, branchPath) {
+    let checkpointing;
+    let hadIndexingErrors = false;
+    const attributeErrorReported = {};
+    const onIndexPutRejected = (property, error) => {
+        hadIndexingErrors = true;
+        if (attributeErrorReported[property])
+            return;
+        attributeErrorReported[property] = true;
+        logger.error(`Error indexing attribute ${property}`, error);
+    };
+    const putRejectionHandlers = attributes.map((attribute) => (error) => onIndexPutRejected(attribute.name, error));
     try {
         logger.info(`Indexing ${Table.tableName} attributes`, attributes);
         await signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName, undefined, branchPath));
         let lastResolution;
+        // The checkpoint and completion barriers have to cover every mutation still in flight: any of them
+        // may reject after those barriers read hadIndexingErrors.
+        const pendingMutations = new Set();
+        let settleWaiter;
+        const track = (result, onRejected) => {
+            if (!result?.then)
+                return result;
+            const tracked = result.then(() => {
+                pendingMutations.delete(tracked);
+                settleWaiter?.();
+                return false;
+            }, (error) => {
+                pendingMutations.delete(tracked);
+                settleWaiter?.();
+                onRejected(error);
+                return true;
+            });
+            pendingMutations.add(tracked);
+            return result;
+        };
+        // The tracked promises absorb their own rejections, so one failure never abandons its siblings.
+        const drainMutations = async () => {
+            if (!pendingMutations.size)
+                return false;
+            return (await Promise.all([...pendingMutations])).some(Boolean);
+        };
+        // Waiting on a chosen entry would stall behind a slow one the others have already overtaken.
+        const nextSettlement = () => new Promise((resolve) => {
+            settleWaiter = () => {
+                settleWaiter = undefined;
+                resolve(undefined);
+            };
+        });
         for (const index of indicesToRemove) {
-            lastResolution = index.drop();
+            index.customIndex?.resetDerivedStorage?.();
+            track(index.drop(), (error) => onIndexPutRejected(index.name, error));
         }
         let interrupted;
-        let hadIndexingErrors = false;
-        const attributeErrorReported = {};
         let indexed = 0;
         const attributesLength = attributes.length;
         await new Promise((resolve) => setImmediate(resolve)); // yield event turn, indexing should consistently take at least one event turn
         if (attributesLength > 0) {
-            let start;
-            for (const attribute of attributes) {
-                // if we are resuming, we need to start from the last key we indexed by all attributes
-                if ((0, lmdb_1.compareKeys)(attribute.lastIndexedKey, start) < 0)
-                    start = attribute.lastIndexedKey;
-                if (attribute.lastIndexedKey == undefined) {
+            const start = resumeStartKey(attributes);
+            if (start === undefined) {
+                for (const attribute of attributes) {
                     // if we are starting from the beginning, clear out any previous index entries since we are rewriting
+                    attribute.dbi.customIndex?.resetDerivedStorage?.();
                     if (attribute.dbi.clearAsync) {
-                        // LMDB, note that we don't need to wait for this to complete, just gets enqueued in front of the other writes
-                        attribute.dbi.clearAsync();
+                        // LMDB enqueues this ahead of the index writes, so the scan need not wait for it — but the
+                        // barriers must, or a rejected clear certifies a checkpoint over stale entries.
+                        track(attribute.dbi.clearAsync(), (error) => onIndexPutRejected(attribute.name, error));
                     }
                     else {
                         await attribute.dbi.clear();
                     }
                 }
             }
-            let outstanding = 0;
+            // A resumed scan starts at the checkpoint, so it must only name a key whose every predecessor is
+            // durably indexed: persisted once the writes it covers have settled and flushed, frozen after any
+            // record fails so the retry re-covers it, and stamped with its own key (see the trigger in table()).
+            const persistCheckpoint = async (key) => {
+                if (hadIndexingErrors)
+                    return;
+                try {
+                    // Everything still in flight was issued for a key at or before this one: the scan has not
+                    // moved past it yet. So a failure among them is a failure this checkpoint would cover.
+                    const failed = await drainMutations();
+                    if (failed)
+                        return;
+                    await flushIndexStores(Table.primaryStore.rootStore);
+                    const puts = [];
+                    for (const attribute of attributes) {
+                        attribute.lastIndexedKey = key;
+                        attribute.checkpointCertified = key;
+                        attribute.checkpointAlgorithm = exports.CHECKPOINT_ALGORITHM;
+                        puts.push(Table.dbisDB.put(attribute.key, attribute));
+                    }
+                    await Promise.all(puts);
+                }
+                catch (error) {
+                    logger.warn(`Could not persist the indexing checkpoint for ${Table.tableName}`, error);
+                }
+            };
+            let nextCheckpointAt = performance.now() + indexingCheckpointPeriodMs;
+            let nextCheckpointRecord = indexingCheckpointMinRecords;
             // this means that a new attribute has been introduced that needs to be indexed
             for (const { key, value: record } of Table.primaryStore.getRange({
                 start,
@@ -2966,96 +3393,90 @@ async function runIndexing(Table, attributes, indicesToRemove, branchPath) {
                 versions: true,
                 snapshot: false, // don't hold a read transaction this whole time
             })) {
-                if (!record)
-                    continue; // deletion entry
+                const atInterval = ++indexed % INDEXING_YIELD_INTERVAL === 0;
                 // TODO: Do we ever need to interrupt due to a schema change that was not a restart?
                 //if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up
-                outstanding++;
-                // Custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`, so the
-                // outstanding-based yield below never fires for them. Track that this row did synchronous
+                // Custom indexes (e.g. HNSW) index synchronously and leave pendingMutations empty, so the
+                // backpressure yield below never fires for them. Track that this row did synchronous
                 // indexing work so we can still yield the event loop after it.
                 let didSynchronousIndexing = false;
                 // every index operation needs to be guarded by the version still be the same. If it has already changed before
                 // we index, that's fine because indexing is idempotent, we can just put the same values again. If it changes
                 // during the indexing, the indexing here will fail. This is also fine because it means the other thread will have
                 // performed indexing and we don't need to do anything further
-                for (let i = 0; i < attributesLength; i++) {
-                    const attribute = attributes[i];
-                    const property = attribute.name;
-                    const index = attribute.dbi;
-                    try {
-                        const resolver = attribute.resolve;
-                        const value = record && (resolver ? resolver(record) : record[property]);
-                        if (index.customIndex) {
-                            index.customIndex.index(key, value);
-                            didSynchronousIndexing = true;
-                            continue;
-                        }
-                        const values = (0, commonUtility_ts_1.getIndexedValues)(value, index.indexNulls);
-                        if (values) {
-                            for (let i = 0, l = values.length; i < l; i++) {
-                                lastResolution = index.put(values[i], key);
+                if (record) {
+                    for (let i = 0; i < attributesLength; i++) {
+                        const attribute = attributes[i];
+                        const property = attribute.name;
+                        const index = attribute.dbi;
+                        const onPutRejected = putRejectionHandlers[i];
+                        try {
+                            const resolver = attribute.resolve;
+                            const value = record && (resolver ? resolver(record) : record[property]);
+                            if (index.customIndex) {
+                                index.customIndex.index(key, value);
+                                didSynchronousIndexing = true;
+                                continue;
+                            }
+                            const values = (0, commonUtility_ts_1.getIndexedValues)(value, index.indexNulls);
+                            if (values) {
+                                for (let i = 0, l = values.length; i < l; i++) {
+                                    track(index.put(values[i], key), onPutRejected);
+                                }
                             }
                         }
-                    }
-                    catch (error) {
-                        hadIndexingErrors = true;
-                        if (!attributeErrorReported[property]) {
-                            // just report an indexing error once per attribute so we don't spam the logs.
-                            // A store closed by worker shutdown surfaces here as "Database not open"; that is
-                            // a benign interruption (the next generation re-runs the backfill), so don't log
-                            // it as an error — the outer catch returns quietly once the iterator also throws.
-                            attributeErrorReported[property] = true;
-                            if (Table.primaryStore?.rootStore?.status === 'closed')
-                                logger.debug(`Indexing attribute ${property} interrupted by store shutdown`, error);
-                            else
-                                logger.error(`Error indexing attribute ${property}`, error);
+                        catch (error) {
+                            hadIndexingErrors = true;
+                            if (!attributeErrorReported[property]) {
+                                // just report an indexing error once per attribute so we don't spam the logs.
+                                // A store closed by worker shutdown surfaces here as "Database not open"; that is
+                                // a benign interruption (the next generation re-runs the backfill), so don't log
+                                // it as an error — the outer catch returns quietly once the iterator also throws.
+                                attributeErrorReported[property] = true;
+                                if (Table.primaryStore?.rootStore?.status === 'closed')
+                                    logger.debug(`Indexing attribute ${property} interrupted by store shutdown`, error);
+                                else
+                                    logger.error(`Error indexing attribute ${property}`, error);
+                            }
                         }
                     }
                 }
-                (0, when_ts_1.when)(lastResolution, () => outstanding--, (error) => {
-                    outstanding--;
-                    hadIndexingErrors = true;
-                    logger.error(error);
-                });
                 if (worker_threads_1.workerData && worker_threads_1.workerData.restartNumber !== manageThreads.restartNumber) {
                     interrupted = true;
                 }
-                if (++indexed % 100 === 0 || interrupted) {
-                    // occasionally update our progress so if we crash, we can resume
-                    for (const attribute of attributes) {
-                        attribute.lastIndexedKey = key;
-                        Table.dbisDB.put(attribute.key, attribute);
-                    }
-                    if (interrupted)
-                        return;
+                if (interrupted) {
+                    await drainMutations();
+                    await checkpointing;
+                    await persistCheckpoint(key);
+                    return;
                 }
-                if (outstanding > MAX_OUTSTANDING_INDEXING)
-                    await lastResolution;
-                else if (outstanding > MIN_OUTSTANDING_INDEXING)
-                    await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
-                else if (didSynchronousIndexing)
-                    await new Promise((resolve) => setImmediate(resolve)); // custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`; without this yield a large backfill runs in a single event-loop turn, starving keepalive/replication and queries and never letting the isIndexing flag be observed
-            }
-        }
-        // Await the last pending put. If it rejects, that is also an indexing error.
-        // Note: the when() calls above already attach rejection handlers to each record's
-        // last-put promise; this try-catch specifically handles the case where lastResolution
-        // itself rejects (i.e. the very last put in the loop failed) which would otherwise
-        // throw past the hadIndexingErrors check to the outer catch. The broader issue of
-        // unhandled rejections from non-last puts in multi-value attributes is pre-existing
-        // and out of scope for this fix.
-        try {
-            await lastResolution;
-        }
-        catch (error) {
-            hadIndexingErrors = true;
-            logger.error(error);
+                if (atInterval && indexed >= nextCheckpointRecord && performance.now() >= nextCheckpointAt) {
+                    nextCheckpointAt = performance.now() + indexingCheckpointPeriodMs;
+                    nextCheckpointRecord = indexed + indexingCheckpointMinRecords;
+                    await checkpointing;
+                    checkpointing = persistCheckpoint(key);
+                }
+                // Checked once per record, so a record's own fan-out can overshoot before the bound applies.
+                while (pendingMutations.size > MAX_OUTSTANDING_INDEXING)
+                    await nextSettlement();
+                if (atInterval || didSynchronousIndexing || pendingMutations.size > MIN_OUTSTANDING_INDEXING)
+                    await yieldEventTurn();
+            }
+        }
+        await checkpointing;
+        // A mutation that rejects after completion is declared has no build left to park.
+        await drainMutations();
+        // the tail since the last checkpoint is not durable until flushed; announcing the index complete
+        // before that would outlive a crash that loses it
+        if (!hadIndexingErrors) {
+            try {
+                await flushIndexStores(Table.primaryStore.rootStore);
+            }
+            catch (error) {
+                hadIndexingErrors = true;
+                logger.error(`Could not flush the indexes of ${Table.tableName} before marking them complete`, error);
+            }
         }
-        // Yield one more event turn so any queued when() error callbacks (which fire as
-        // microtasks when their tracked promise settles) have a chance to set hadIndexingErrors
-        // before we decide whether to mark indexing as complete.
-        await new Promise((resolve) => setImmediate(resolve));
         if (hadIndexingErrors) {
             // Some records failed to index. Persist the failure marker in the descriptor so
             // the next call to table() (including after a restart with a fresh PID) re-triggers
@@ -3084,9 +3505,15 @@ async function runIndexing(Table, attributes, indicesToRemove, branchPath) {
             // update the attributes to indicate that we are finished
             for (const attribute of attributes) {
                 delete attribute.lastIndexedKey;
+                delete attribute.checkpointCertified;
+                // Survives completion, unlike the checkpoint fields: without it an index built here is
+                // indistinguishable from one a release that could skip a failed record declared complete.
+                attribute.checkpointAlgorithm = exports.CHECKPOINT_ALGORITHM;
                 delete attribute.indexingPID;
                 delete attribute.indexingFailed;
                 delete attribute.restartNumber;
+                delete attribute.indexingIncarnation;
+                delete attribute.indexingBuildId;
                 attribute.dbi.isIndexing = false;
                 // Also clear isIndexing on the currently-active dbi in Table.indices, which may
                 // differ from attribute.dbi if a resetDatabases() call during this migration
@@ -3103,6 +3530,7 @@ async function runIndexing(Table, attributes, indicesToRemove, branchPath) {
         }
     }
     catch (error) {
+        await checkpointing;
         // A worker shutting down closes its stores mid-backfill, so the range iterator or a
         // put throws (e.g. "Database not open" / "Iterator not initialized"). This is an
         // interruption, not a data error: the next worker generation re-runs the backfill via
@@ -3166,6 +3594,26 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableNa
                 finally {
                     columnStore.close();
                 }
+                // derived HNSW plane files live next to the store; the normal drop path removes
+                // them through the custom index, but this recovery path drops raw column stores,
+                // and a same-name recreate must never open a stale plane over a fresh CF
+                try {
+                    (0, node_fs_1.unlinkSync)((0, hnswPlaneBinding_ts_1.planeFilePathFor)(rootStore.path, columnName));
+                }
+                catch (error) {
+                    // a stale plane left behind (e.g. Windows EBUSY while still mapped) would be
+                    // opened over a fresh same-name CF, resolving another graph's node ids
+                    // against it — tombstone it so no attach ever adopts it
+                    if (error?.code !== 'ENOENT') {
+                        logger.warn(`could not delete the HNSW plane file for ${columnName}; tombstoning it as stale`, error);
+                        try {
+                            (0, node_fs_1.closeSync)((0, node_fs_1.openSync)((0, hnswPlaneBinding_ts_1.planeStalePathFor)((0, hnswPlaneBinding_ts_1.planeFilePathFor)(rootStore.path, columnName)), 'w'));
+                        }
+                        catch (tombstoneError) {
+                            logger.warn(`could not tombstone the stale HNSW plane file for ${columnName}`, tombstoneError);
+                        }
+                    }
+                }
             }
         }
     }