@harperfast/harper 5.2.9 → 5.2.12

package/dist/resources/databases.js

@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
+exports.ABANDONED_MARK_LOCK_TIMEOUT = exports.CHECKPOINT_ALGORITHM = exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
 exports.isReadOnlyMode = isReadOnlyMode;
 exports.getRocksCompression = getRocksCompression;
 exports.resetRocksCompression = resetRocksCompression;
@@ -53,6 +53,9 @@ exports.closeDatabase = closeDatabase;
 exports.closeLoadedDatabases = closeLoadedDatabases;
 exports.table = table;
 exports.canonicalizeIndexOptions = canonicalizeIndexOptions;
+exports.setIndexingCheckpointPeriod = setIndexingCheckpointPeriod;
+exports.resumeStartKey = resumeStartKey;
+exports.tryAcquireUpdateAttributesLock = tryAcquireUpdateAttributesLock;
 exports.dropTableMeta = dropTableMeta;
 exports.onUpdatedTable = onUpdatedTable;
 exports.onRemovedTable = onRemovedTable;
@@ -60,6 +63,7 @@ exports.onRemovedDB = onRemovedDB;
 exports.getDefaultCompression = getDefaultCompression;
 exports.flushDatabases = flushDatabases;
 const node_events_1 = require("node:events");
+const node_crypto_1 = require("node:crypto");
 const environmentManager_ts_1 = require("../utility/environment/environmentManager.js");
 const terms_ts_1 = require("../utility/lmdb/terms.js");
 const lmdb_1 = require("lmdb");
@@ -81,6 +85,7 @@ const harper_logger_ts_1 = __importDefault(require("../utility/logging/harper_lo
 const { forComponent } = harper_logger_ts_1.default;
 const manageThreads = __importStar(require("../server/threads/manageThreads.js"));
 const auditStore_ts_1 = require("./auditStore.js");
+const storageReclamation_ts_1 = require("../server/storageReclamation.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
 const blob_ts_1 = require("./blob.js");
 const customIndexes_ts_1 = require("./indexes/customIndexes.js");
@@ -90,7 +95,6 @@ const PrimaryRocksDatabase_ts_1 = require("./PrimaryRocksDatabase.js");
 const replayLogs_ts_1 = require("./replayLogs.js");
 const node_os_1 = require("node:os");
 const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
-const when_ts_1 = require("../utility/when.js");
 const rocksMemoryConfig_ts_1 = require("../utility/rocksMemoryConfig.js");
 const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
 const restoreMarker_ts_1 = require("../dataLayer/restoreMarker.js");
@@ -321,6 +325,23 @@ const rocksdbDatabaseEnvs = new Map();
 (0, globals_js_1._assignPackageExport)('databases', exports.databases);
 (0, globals_js_1._assignPackageExport)('tables', exports.tables);
 const NEXT_TABLE_ID = Symbol.for('next-table-id');
+/**
+ * True when a descriptor claims an index build no live operation in this process can own. The PID and
+ * worker generation cannot answer that alone: a container reuses PID 1 and starts the in-memory
+ * generation back at 1 while the persisted one is higher. A descriptor with no incarnation was written
+ * before the field existed, so it belongs to an earlier process; a thread started without one of its
+ * own cannot judge, and falls back rather than declaring a live build dead.
+ */
+function isAbandonedIndexBuild(descriptor, currentRestartGeneration) {
+    if (!descriptor)
+        return false;
+    if (descriptor.indexingPID && descriptor.indexingPID !== process.pid)
+        return true;
+    if (descriptor.restartNumber < currentRestartGeneration)
+        return true;
+    const incarnation = manageThreads.processIncarnation;
+    return !!descriptor.indexingPID && incarnation != null && descriptor.indexingIncarnation !== incarnation;
+}
 // How many times the schema load will try to finish a tombstoned drop before
 // giving up for the rest of this process's lifetime. A drop that fails once
 // almost always fails identically forever - the usual cause is a RocksDB
@@ -590,6 +611,7 @@ function readMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME, au
             rootStore = (0, lmdb_1.open)(envInit);
             lmdbDatabaseEnvs.set(path, rootStore);
         }
+        rootStore.dbisDb?.resetReadTxn();
         return initStores(path, rootStore, databaseName, defaultTable, auditPath, isLegacy);
     }
     catch (error) {
@@ -835,6 +857,8 @@ function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLe
                         indices[attribute.name] = dbi;
                         indices[attribute.name].indexNulls = attribute.indexNulls;
                     }
+                    // the only way a thread that never declares the schema reaches Table.indices
+                    indices[attribute.name].isIndexing = !!attribute.indexingPID;
                     const existingAttribute = existingAttributes.find((existingAttribute) => existingAttribute.name === attribute.name);
                     if (existingAttribute)
                         existingAttributes.splice(existingAttributes.indexOf(existingAttribute), 1, attribute);
@@ -1147,6 +1171,10 @@ async function dropDatabase(databaseName) {
         delete exports.databases[databaseName];
         exports.databaseEventsEmitter.emit('dropDatabase', databaseName);
         if (rootStore) {
+            // awaited: retirement stops the loop admitting work, the barrier is what says the pass that was
+            // already running has released the stores this is about to close and unlink
+            await rootStore.auditStore?.stopAuditCleanup?.();
+            (0, storageReclamation_ts_1.removeStorageReclamation)(rootStore.path);
             if (rootStore.status === 'open') {
                 if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
                     rootStore.close();
@@ -1164,6 +1192,8 @@ async function dropDatabase(databaseName) {
             // the drop lock now (still before any destructive step)
             if (rootStore instanceof rocksdb_js_1.RocksDatabase)
                 lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks);
+            await rootStore.auditStore?.stopAuditCleanup?.();
+            (0, storageReclamation_ts_1.removeStorageReclamation)(rootStore.path);
             if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
                 rootStore.close();
                 rootStore.destroy();
@@ -1206,10 +1236,6 @@ function closeDatabase(databaseName) {
             continue;
         if (table.primaryStore.rootStore)
             rootStores.add(table.primaryStore.rootStore);
-        for (const indexName in table.indices || {}) {
-            closeStore(table.indices[indexName], `index ${tableName}.${indexName}`);
-        }
-        closeStore(table.primaryStore, `table ${tableName}`);
     }
     // a database with no tables (an empty schema, or one whose tables were all dropped) still holds
     // an open root store, tracked only on the defined-database entry rather than any table — include
@@ -1217,7 +1243,23 @@ function closeDatabase(databaseName) {
     const definedRoot = definedDatabases?.get(databaseName)?.rootStore;
     if (definedRoot)
         rootStores.add(definedRoot);
+    // before any table store closes, so no further pass is admitted. This is synchronous, so it cannot
+    // await the drain barrier stopAuditCleanup() returns; what covers it is the in-pass status checks,
+    // plus the fact that its production callers reach it only for RocksDB databases, whose pass is one
+    // synchronous purgeLogs() call with nothing suspended mid-removal.
+    for (const rootStore of rootStores)
+        rootStore.auditStore?.stopAuditCleanup?.();
+    for (const tableName in dbTables) {
+        const table = dbTables[tableName];
+        if (!table?.primaryStore)
+            continue;
+        for (const indexName in table.indices || {}) {
+            closeStore(table.indices[indexName], `index ${tableName}.${indexName}`);
+        }
+        closeStore(table.primaryStore, `table ${tableName}`);
+    }
     for (const rootStore of rootStores) {
+        (0, storageReclamation_ts_1.removeStorageReclamation)(rootStore.path);
         closeStore(rootStore.dbisDb, 'attributes store');
         closeStore(rootStore, 'root store');
         lmdbDatabaseEnvs.delete(rootStore.path);
@@ -1778,15 +1820,13 @@ function table(tableDefinition) {
                 if (changed ||
                     indexFormatNeedsPersist ||
                     attributeDescriptor?.indexingFailed ||
-                    (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
-                    attributeDescriptor?.restartNumber < currentRestartGeneration) {
+                    isAbandonedIndexBuild(attributeDescriptor, currentRestartGeneration)) {
                     hasChanges = true;
                     exclusiveLock();
                     attributeDescriptor = attributesDbi.getSync(dbiKey);
                     if (structurallyChanged ||
                         attributeDescriptor?.indexingFailed ||
-                        (attributeDescriptor?.indexingPID && attributeDescriptor?.indexingPID !== process.pid) ||
-                        attributeDescriptor?.restartNumber < currentRestartGeneration) {
+                        isAbandonedIndexBuild(attributeDescriptor, currentRestartGeneration)) {
                         hasChanges = true;
                         if (attribute.indexNulls === undefined)
                             attribute.indexNulls = true;
@@ -1805,9 +1845,20 @@ function table(tableDefinition) {
                             // representation-only option difference — preserve the checkpoint so the backfill
                             // resumes rather than restarts. Canonicalized to match structurallyChanged above.
                             const indexOptionsChanged = canonicalIndexKey(attributeDescriptor?.indexed) !== canonicalIndexKey(attribute.indexed);
-                            attribute.lastIndexedKey = indexOptionsChanged
-                                ? undefined
-                                : (attributeDescriptor?.lastIndexedKey ?? undefined);
+                            // Only a checkpoint runIndexing stamped with its own key resumes: earlier releases advanced
+                            // lastIndexedKey past failed and unflushed index writes, so any other is a full rebuild.
+                            const uncertifiedCheckpoint = attributeDescriptor?.lastIndexedKey !== undefined &&
+                                (attributeDescriptor.checkpointCertified === undefined ||
+                                    attributeDescriptor.checkpointAlgorithm !== exports.CHECKPOINT_ALGORITHM ||
+                                    (0, lmdb_1.compareKeys)(attributeDescriptor.checkpointCertified, attributeDescriptor.lastIndexedKey) !== 0);
+                            attribute.lastIndexedKey =
+                                indexOptionsChanged || uncertifiedCheckpoint
+                                    ? undefined
+                                    : (attributeDescriptor?.lastIndexedKey ?? undefined);
+                            if (attribute.lastIndexedKey !== undefined) {
+                                attribute.checkpointCertified = attribute.lastIndexedKey;
+                                attribute.checkpointAlgorithm = exports.CHECKPOINT_ALGORITHM;
+                            }
                             // Explicit reindex is the upgrade path from a legacy (un-versioned) custom-index
                             // object store to the versioned, VT-cacheable format. A full rebuild from scratch
                             // (lastIndexedKey === undefined) clears the store and rewrites every node, so the
@@ -1829,6 +1880,9 @@ function table(tableDefinition) {
                             // the new process reuses the old PID. Cleared on clean completion; left in place
                             // on failure/crash so the next, higher-numbered restart re-triggers the backfill.
                             attribute.restartNumber = currentRestartGeneration;
+                            if (manageThreads.processIncarnation != null)
+                                attribute.indexingIncarnation = manageThreads.processIncarnation;
+                            attribute.indexingBuildId = (0, node_crypto_1.randomBytes)(8).toString('hex');
                             delete attribute.indexingFailed; // clear failure flag for the new run
                             dbi.isIndexing = true;
                             Object.defineProperty(attribute, 'dbi', { value: dbi, configurable: true, enumerable: false });
@@ -1844,6 +1898,12 @@ function table(tableDefinition) {
                                 reindexReasons.push(`crash-recovery(pid=${attributeDescriptor.indexingPID})`);
                             if (attributeDescriptor?.restartNumber < currentRestartGeneration)
                                 reindexReasons.push('restart-number');
+                            if (uncertifiedCheckpoint)
+                                reindexReasons.push('uncertified-checkpoint');
+                            if (attributeDescriptor?.indexingPID === process.pid &&
+                                manageThreads.processIncarnation != null &&
+                                attributeDescriptor.indexingIncarnation !== manageThreads.processIncarnation)
+                                reindexReasons.push('abandoned-build(previous process incarnation)');
                             logger.info(`reindex ${databaseName}.${tableName}.${attribute.name}: reason=${reindexReasons.join(',') || 'unknown'}`);
                             // we only set indexing nulls to true if new or reindexing, we can't have partial indexing of null
                             attributesToIndex.push(attribute);
@@ -1856,12 +1916,22 @@ function table(tableDefinition) {
                         // workers / a reload would treat the still-partial index as ready and return incomplete results.
                         attribute.indexingPID = attributeDescriptor.indexingPID;
                         attribute.lastIndexedKey = attributeDescriptor.lastIndexedKey;
+                        if (attributeDescriptor.checkpointCertified !== undefined) {
+                            attribute.checkpointCertified = attributeDescriptor.checkpointCertified;
+                            attribute.checkpointAlgorithm = attributeDescriptor.checkpointAlgorithm;
+                        }
                         // Carry the in-progress restart generation too, so persisting this metadata-only
                         // change doesn't drop it and break the crash-recovery trigger for the running backfill.
                         attribute.restartNumber = attributeDescriptor.restartNumber;
+                        attribute.indexingIncarnation = attributeDescriptor.indexingIncarnation;
+                        attribute.indexingBuildId = attributeDescriptor.indexingBuildId;
                         if (attributeDescriptor.indexingFailed)
                             attribute.indexingFailed = attributeDescriptor.indexingFailed;
                     }
+                    // The declared attribute never carries the stamp, so any rewrite of a descriptor that has
+                    // one would drop it and make a completed index look like a pre-stamp build.
+                    if (attribute.checkpointAlgorithm === undefined && attributeDescriptor?.checkpointAlgorithm !== undefined)
+                        attribute.checkpointAlgorithm = attributeDescriptor.checkpointAlgorithm;
                     attributesDbi.put(dbiKey, attribute);
                 }
                 // If a migration is in progress (indexingPID set), any newly opened dbi must also
@@ -1892,7 +1962,10 @@ function table(tableDefinition) {
     }
     logger.trace(`${tableName} table loading, running index`);
     if (attributesToIndex.length > 0 || indicesToRemove.length > 0) {
-        Table.indexingOperation = runIndexing(Table, attributesToIndex, indicesToRemove);
+        // captured before the backfill can rewrite the attributes
+        const buildIds = new Map(attributesToIndex.map((attribute) => [attribute, attribute.indexingBuildId]));
+        const markSettled = () => markAbandonedIndexBuild(Table, rootStore, buildIds);
+        Table.indexingOperation = runIndexing(Table, attributesToIndex, indicesToRemove).then(markSettled, markSettled);
     }
     else if (hasChanges)
         signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName));
@@ -1964,40 +2037,238 @@ function canonicalizeIndexOptions(value) {
     }
     return value;
 }
+// Bumped when a change alters which keys a checkpoint may certify. A descriptor stamped by any other
+// version resumes as uncertified (full rebuild) rather than being trusted, and a completed index keeps
+// the stamp of the build that wrote it.
+exports.CHECKPOINT_ALGORITHM = 2;
 const MAX_OUTSTANDING_INDEXING = 1000;
 const MIN_OUTSTANDING_INDEXING = 10;
+const INDEXING_YIELD_INTERVAL = 100;
+// A resumable checkpoint is written only after a flush (see flushIndexStores), at most once per period
+// and never before this many more records: the flush seals every column family in the database, so a
+// slow backfill must not impose the period's flush rate on unrelated tables.
+let indexingCheckpointPeriodMs = 5000;
+let indexingCheckpointMinRecords = 10000;
+function setIndexingCheckpointPeriod(ms, minRecords = indexingCheckpointMinRecords) {
+    const previous = { ms: indexingCheckpointPeriodMs, minRecords: indexingCheckpointMinRecords };
+    indexingCheckpointPeriodMs = ms;
+    indexingCheckpointMinRecords = minRecords;
+    return previous;
+}
+const yieldEventTurn = () => new Promise((resolve) => setImmediate(resolve));
+// RocksDB index stores have no WAL (openRocksDatabase defaults disableWAL), so a flush is what makes the
+// entries a checkpoint certifies durable. A flush only covers writes issued before it started, so a caller
+// never joins one in flight: it joins the next one, which every backfill on that database asking meanwhile
+// shares — at most one in flight and one queued.
+const indexingFlushes = new WeakMap();
+function flushIndexStores(rootStore) {
+    if (!(rootStore instanceof rocksdb_js_1.RocksDatabase))
+        return;
+    let flushes = indexingFlushes.get(rootStore);
+    if (!flushes)
+        indexingFlushes.set(rootStore, (flushes = {}));
+    if (flushes.queued)
+        return flushes.queued;
+    const start = () => {
+        flushes.queued = undefined;
+        const flush = rootStore.flush().finally(() => {
+            if (flushes.inFlight === flush)
+                flushes.inFlight = undefined;
+        });
+        flushes.inFlight = flush;
+        return flush;
+    };
+    if (!flushes.inFlight)
+        return start();
+    return (flushes.queued = flushes.inFlight.then(start, start));
+}
+function resumeStartKey(attributes) {
+    let start;
+    for (const attribute of attributes) {
+        if (attribute.lastIndexedKey == undefined)
+            return undefined;
+        if (start === undefined || (0, lmdb_1.compareKeys)(attribute.lastIndexedKey, start) < 0)
+            start = attribute.lastIndexedKey;
+    }
+    return start;
+}
+// Bounded, unlike the exclusiveLock() spin above: that one runs on the declaring path, where the
+// caller is waiting on the result and there is nothing useful to do without the lock. This one runs
+// after a backfill has already settled, so a holder that never releases would wedge the worker's
+// event loop for nothing. Giving up costs only the marker, and the next load of the table
+// re-triggers the build regardless.
+exports.ABANDONED_MARK_LOCK_TIMEOUT = 10000;
+const abandonedMarkLockWait = new Int32Array(new SharedArrayBuffer(4));
+// `timeout` is the test seam; production callers take the default.
+function tryAcquireUpdateAttributesLock(rootStore, timeout = exports.ABANDONED_MARK_LOCK_TIMEOUT) {
+    if (rootStore.tryLock('update-attributes'))
+        return true;
+    const startTime = performance.now();
+    let waitTime = 1;
+    while (!rootStore.tryLock('update-attributes')) {
+        const elapsed = performance.now() - startTime;
+        if (elapsed >= timeout)
+            return false;
+        // Atomics.wait rather than a busy spin: the section this guards is synchronous, so the wait
+        // blocks this thread either way, but sleeping does not burn a core while it does.
+        if (elapsed >= 2) {
+            Atomics.wait(abandonedMarkLockWait, 0, 0, Math.min(waitTime, timeout - elapsed));
+            if (waitTime < 16)
+                waitTime *= 2;
+        }
+    }
+    return true;
+}
+/**
+ * Persists the failure marker for a build that ended without running one of runIndexing's own exit
+ * paths, so something re-triggers it. Fenced on `indexingBuildId` inside the storage engine's catalog
+ * serialization boundary, because a replacement generation (or another thread declaring different index
+ * options) can claim the attribute before an outgoing build's promise settles, and marking that would fail
+ * a live build. The fence read and write stay synchronous, and nothing here may throw because
+ * `Table.indexingOperation` reaches operations-API callers.
+ */
+async function markAbandonedIndexBuild(Table, rootStore, buildIds) {
+    for (const [attribute, buildId] of buildIds) {
+        try {
+            let marked;
+            if (buildId == null || Table.dbisDB.getSync(attribute.key)?.indexingBuildId !== buildId)
+                continue;
+            const markIfOwned = () => {
+                const descriptor = Table.dbisDB.getSync(attribute.key);
+                if (descriptor?.indexingBuildId === buildId && !descriptor.indexingFailed) {
+                    Table.dbisDB.putSync(attribute.key, { ...descriptor, indexingFailed: true });
+                    marked = true;
+                }
+            };
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                if (!tryAcquireUpdateAttributesLock(rootStore)) {
+                    logger.warn(`Could not mark the abandoned index build of ${Table.databaseName}.${Table.tableName}.${attribute.name}: ` +
+                        `timed out after ${exports.ABANDONED_MARK_LOCK_TIMEOUT}ms waiting for the exclusive 'update-attributes' lock. ` +
+                        `The index stays incomplete and the next load of the table re-triggers the backfill.`);
+                    continue;
+                }
+                try {
+                    markIfOwned();
+                }
+                finally {
+                    rootStore.unlock('update-attributes');
+                }
+            }
+            else {
+                rootStore.transactionSync(markIfOwned);
+            }
+            if (marked)
+                logger.warn(`Indexing of ${Table.databaseName}.${Table.tableName}.${attribute.name} ended without completing. ` +
+                    `The index stays incomplete and every query on the attribute reports it as not indexed yet; ` +
+                    `the next load of the table retries the backfill from the last checkpoint (indexingFailed=true).`);
+        }
+        catch (error) {
+            // A store closed by shutdown is the common case, and it cannot be written to at all.
+            try {
+                logger.debug(`Could not mark the abandoned index build of ${Table.tableName}.${attribute.name}`, error);
+            }
+            catch { }
+        }
+    }
+}
 async function runIndexing(Table, attributes, indicesToRemove) {
+    let checkpointing;
+    let hadIndexingErrors = false;
+    const attributeErrorReported = {};
+    const onIndexPutRejected = (property, error) => {
+        hadIndexingErrors = true;
+        if (attributeErrorReported[property])
+            return;
+        attributeErrorReported[property] = true;
+        logger.error(`Error indexing attribute ${property}`, error);
+    };
+    const putRejectionHandlers = attributes.map((attribute) => (error) => onIndexPutRejected(attribute.name, error));
     try {
         logger.info(`Indexing ${Table.tableName} attributes`, attributes);
         await signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName));
         let lastResolution;
+        // The checkpoint and completion barriers have to cover every mutation still in flight: any of them
+        // may reject after those barriers read hadIndexingErrors.
+        const pendingMutations = new Set();
+        let settleWaiter;
+        const track = (result, onRejected) => {
+            if (!result?.then)
+                return result;
+            const tracked = result.then(() => {
+                pendingMutations.delete(tracked);
+                settleWaiter?.();
+                return false;
+            }, (error) => {
+                pendingMutations.delete(tracked);
+                settleWaiter?.();
+                onRejected(error);
+                return true;
+            });
+            pendingMutations.add(tracked);
+            return result;
+        };
+        // The tracked promises absorb their own rejections, so one failure never abandons its siblings.
+        const drainMutations = async () => {
+            if (!pendingMutations.size)
+                return false;
+            return (await Promise.all([...pendingMutations])).some(Boolean);
+        };
+        // Waiting on a chosen entry would stall behind a slow one the others have already overtaken.
+        const nextSettlement = () => new Promise((resolve) => {
+            settleWaiter = () => {
+                settleWaiter = undefined;
+                resolve(undefined);
+            };
+        });
         for (const index of indicesToRemove) {
-            lastResolution = index.drop();
+            track(index.drop(), (error) => onIndexPutRejected(index.name, error));
         }
         let interrupted;
-        let hadIndexingErrors = false;
-        const attributeErrorReported = {};
         let indexed = 0;
         const attributesLength = attributes.length;
         await new Promise((resolve) => setImmediate(resolve)); // yield event turn, indexing should consistently take at least one event turn
         if (attributesLength > 0) {
-            let start;
-            for (const attribute of attributes) {
-                // if we are resuming, we need to start from the last key we indexed by all attributes
-                if ((0, lmdb_1.compareKeys)(attribute.lastIndexedKey, start) < 0)
-                    start = attribute.lastIndexedKey;
-                if (attribute.lastIndexedKey == undefined) {
-                    // if we are starting from the beginning, clear out any previous index entries since we are rewriting
+            const start = resumeStartKey(attributes);
+            if (start === undefined) {
+                for (const attribute of attributes) {
                     if (attribute.dbi.clearAsync) {
-                        // LMDB, note that we don't need to wait for this to complete, just gets enqueued in front of the other writes
-                        attribute.dbi.clearAsync();
+                        // LMDB enqueues this ahead of the index writes, so the scan need not wait for it — but the
+                        // barriers must, or a rejected clear certifies a checkpoint over stale entries.
+                        track(attribute.dbi.clearAsync(), (error) => onIndexPutRejected(attribute.name, error));
                     }
                     else {
                         await attribute.dbi.clear();
                     }
                 }
             }
-            let outstanding = 0;
+            // A resumed scan starts at the checkpoint, so it must only name a key whose every predecessor is
+            // durably indexed: persisted once the writes it covers have settled and flushed, frozen after any
+            // record fails so the retry re-covers it, and stamped with its own key (see the trigger in table()).
+            const persistCheckpoint = async (key) => {
+                if (hadIndexingErrors)
+                    return;
+                try {
+                    // Everything still in flight was issued for a key at or before this one: the scan has not
+                    // moved past it yet. So a failure among them is a failure this checkpoint would cover.
+                    const failed = await drainMutations();
+                    if (failed)
+                        return;
+                    await flushIndexStores(Table.primaryStore.rootStore);
+                    const puts = [];
+                    for (const attribute of attributes) {
+                        attribute.lastIndexedKey = key;
+                        attribute.checkpointCertified = key;
+                        attribute.checkpointAlgorithm = exports.CHECKPOINT_ALGORITHM;
+                        puts.push(Table.dbisDB.put(attribute.key, attribute));
+                    }
+                    await Promise.all(puts);
+                }
+                catch (error) {
+                    logger.warn(`Could not persist the indexing checkpoint for ${Table.tableName}`, error);
+                }
+            };
+            let nextCheckpointAt = performance.now() + indexingCheckpointPeriodMs;
+            let nextCheckpointRecord = indexingCheckpointMinRecords;
             // this means that a new attribute has been introduced that needs to be indexed
             for (const { key, value: record } of Table.primaryStore.getRange({
                 start,
@@ -2005,96 +2276,90 @@ async function runIndexing(Table, attributes, indicesToRemove) {
                 versions: true,
                 snapshot: false, // don't hold a read transaction this whole time
             })) {
-                if (!record)
-                    continue; // deletion entry
+                const atInterval = ++indexed % INDEXING_YIELD_INTERVAL === 0;
                 // TODO: Do we ever need to interrupt due to a schema change that was not a restart?
                 //if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up
-                outstanding++;
-                // Custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`, so the
-                // outstanding-based yield below never fires for them. Track that this row did synchronous
+                // Custom indexes (e.g. HNSW) index synchronously and leave pendingMutations empty, so the
+                // backpressure yield below never fires for them. Track that this row did synchronous
                 // indexing work so we can still yield the event loop after it.
                 let didSynchronousIndexing = false;
                 // every index operation needs to be guarded by the version still be the same. If it has already changed before
                 // we index, that's fine because indexing is idempotent, we can just put the same values again. If it changes
                 // during the indexing, the indexing here will fail. This is also fine because it means the other thread will have
                 // performed indexing and we don't need to do anything further
-                for (let i = 0; i < attributesLength; i++) {
-                    const attribute = attributes[i];
-                    const property = attribute.name;
-                    const index = attribute.dbi;
-                    try {
-                        const resolver = attribute.resolve;
-                        const value = record && (resolver ? resolver(record) : record[property]);
-                        if (index.customIndex) {
-                            index.customIndex.index(key, value);
-                            didSynchronousIndexing = true;
-                            continue;
-                        }
-                        const values = (0, commonUtility_ts_1.getIndexedValues)(value, index.indexNulls);
-                        if (values) {
-                            for (let i = 0, l = values.length; i < l; i++) {
-                                lastResolution = index.put(values[i], key);
+                if (record) {
+                    for (let i = 0; i < attributesLength; i++) {
+                        const attribute = attributes[i];
+                        const property = attribute.name;
+                        const index = attribute.dbi;
+                        const onPutRejected = putRejectionHandlers[i];
+                        try {
+                            const resolver = attribute.resolve;
+                            const value = record && (resolver ? resolver(record) : record[property]);
+                            if (index.customIndex) {
+                                index.customIndex.index(key, value);
+                                didSynchronousIndexing = true;
+                                continue;
+                            }
+                            const values = (0, commonUtility_ts_1.getIndexedValues)(value, index.indexNulls);
+                            if (values) {
+                                for (let i = 0, l = values.length; i < l; i++) {
+                                    track(index.put(values[i], key), onPutRejected);
+                                }
                             }
                         }
-                    }
-                    catch (error) {
-                        hadIndexingErrors = true;
-                        if (!attributeErrorReported[property]) {
-                            // just report an indexing error once per attribute so we don't spam the logs.
-                            // A store closed by worker shutdown surfaces here as "Database not open"; that is
-                            // a benign interruption (the next generation re-runs the backfill), so don't log
-                            // it as an error — the outer catch returns quietly once the iterator also throws.
-                            attributeErrorReported[property] = true;
-                            if (Table.primaryStore?.rootStore?.status === 'closed')
-                                logger.debug(`Indexing attribute ${property} interrupted by store shutdown`, error);
-                            else
-                                logger.error(`Error indexing attribute ${property}`, error);
+                        catch (error) {
+                            hadIndexingErrors = true;
+                            if (!attributeErrorReported[property]) {
+                                // just report an indexing error once per attribute so we don't spam the logs.
+                                // A store closed by worker shutdown surfaces here as "Database not open"; that is
+                                // a benign interruption (the next generation re-runs the backfill), so don't log
+                                // it as an error — the outer catch returns quietly once the iterator also throws.
+                                attributeErrorReported[property] = true;
+                                if (Table.primaryStore?.rootStore?.status === 'closed')
+                                    logger.debug(`Indexing attribute ${property} interrupted by store shutdown`, error);
+                                else
+                                    logger.error(`Error indexing attribute ${property}`, error);
+                            }
                         }
                     }
                 }
-                (0, when_ts_1.when)(lastResolution, () => outstanding--, (error) => {
-                    outstanding--;
-                    hadIndexingErrors = true;
-                    logger.error(error);
-                });
                 if (worker_threads_1.workerData && worker_threads_1.workerData.restartNumber !== manageThreads.restartNumber) {
                     interrupted = true;
                 }
-                if (++indexed % 100 === 0 || interrupted) {
-                    // occasionally update our progress so if we crash, we can resume
-                    for (const attribute of attributes) {
-                        attribute.lastIndexedKey = key;
-                        Table.dbisDB.put(attribute.key, attribute);
-                    }
-                    if (interrupted)
-                        return;
+                if (interrupted) {
+                    await drainMutations();
+                    await checkpointing;
+                    await persistCheckpoint(key);
+                    return;
+                }
+                if (atInterval && indexed >= nextCheckpointRecord && performance.now() >= nextCheckpointAt) {
+                    nextCheckpointAt = performance.now() + indexingCheckpointPeriodMs;
+                    nextCheckpointRecord = indexed + indexingCheckpointMinRecords;
+                    await checkpointing;
+                    checkpointing = persistCheckpoint(key);
                 }
-                if (outstanding > MAX_OUTSTANDING_INDEXING)
-                    await lastResolution;
-                else if (outstanding > MIN_OUTSTANDING_INDEXING)
-                    await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
-                else if (didSynchronousIndexing)
-                    await new Promise((resolve) => setImmediate(resolve)); // custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`; without this yield a large backfill runs in a single event-loop turn, starving keepalive/replication and queries and never letting the isIndexing flag be observed
+                // Checked once per record, so a record's own fan-out can overshoot before the bound applies.
+                while (pendingMutations.size > MAX_OUTSTANDING_INDEXING)
+                    await nextSettlement();
+                if (atInterval || didSynchronousIndexing || pendingMutations.size > MIN_OUTSTANDING_INDEXING)
+                    await yieldEventTurn();
             }
         }
-        // Await the last pending put. If it rejects, that is also an indexing error.
-        // Note: the when() calls above already attach rejection handlers to each record's
-        // last-put promise; this try-catch specifically handles the case where lastResolution
-        // itself rejects (i.e. the very last put in the loop failed) which would otherwise
-        // throw past the hadIndexingErrors check to the outer catch. The broader issue of
-        // unhandled rejections from non-last puts in multi-value attributes is pre-existing
-        // and out of scope for this fix.
-        try {
-            await lastResolution;
-        }
-        catch (error) {
-            hadIndexingErrors = true;
-            logger.error(error);
+        await checkpointing;
+        // A mutation that rejects after completion is declared has no build left to park.
+        await drainMutations();
+        // the tail since the last checkpoint is not durable until flushed; announcing the index complete
+        // before that would outlive a crash that loses it
+        if (!hadIndexingErrors) {
+            try {
+                await flushIndexStores(Table.primaryStore.rootStore);
+            }
+            catch (error) {
+                hadIndexingErrors = true;
+                logger.error(`Could not flush the indexes of ${Table.tableName} before marking them complete`, error);
+            }
         }
-        // Yield one more event turn so any queued when() error callbacks (which fire as
-        // microtasks when their tracked promise settles) have a chance to set hadIndexingErrors
-        // before we decide whether to mark indexing as complete.
-        await new Promise((resolve) => setImmediate(resolve));
         if (hadIndexingErrors) {
             // Some records failed to index. Persist the failure marker in the descriptor so
             // the next call to table() (including after a restart with a fresh PID) re-triggers
@@ -2123,9 +2388,15 @@ async function runIndexing(Table, attributes, indicesToRemove) {
             // update the attributes to indicate that we are finished
             for (const attribute of attributes) {
                 delete attribute.lastIndexedKey;
+                delete attribute.checkpointCertified;
+                // Survives completion, unlike the checkpoint fields: without it an index built here is
+                // indistinguishable from one a release that could skip a failed record declared complete.
+                attribute.checkpointAlgorithm = exports.CHECKPOINT_ALGORITHM;
                 delete attribute.indexingPID;
                 delete attribute.indexingFailed;
                 delete attribute.restartNumber;
+                delete attribute.indexingIncarnation;
+                delete attribute.indexingBuildId;
                 attribute.dbi.isIndexing = false;
                 // Also clear isIndexing on the currently-active dbi in Table.indices, which may
                 // differ from attribute.dbi if a resetDatabases() call during this migration
@@ -2142,6 +2413,7 @@ async function runIndexing(Table, attributes, indicesToRemove) {
         }
     }
     catch (error) {
+        await checkpointing;
         // A worker shutting down closes its stores mid-backfill, so the range iterator or a
         // put throws (e.g. "Database not open" / "Iterator not initialized"). This is an
         // interruption, not a data error: the next worker generation re-runs the backfill via