@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

package/dist/resources/Table.js

@@ -63,12 +63,14 @@ const Resource_ts_1 = require("./Resource.js");
 const when_ts_1 = require("../utility/when.js");
 const DatabaseTransaction_ts_1 = require("./DatabaseTransaction.js");
 const recordLock_ts_1 = require("./recordLock.js");
+const nodeName_ts_1 = require("../server/nodeName.js");
 const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
 const transactionBroadcast_ts_1 = require("./transactionBroadcast.js");
 const hdbError_ts_1 = require("../utility/errors/hdbError.js");
 const signalling = __importStar(require("../utility/signalling.js"));
 const itc_js_1 = require("../server/threads/itc.js");
 const databases_ts_1 = require("./databases.js");
+const replicatedApplyFailure_ts_1 = require("./replicatedApplyFailure.js");
 const search_ts_1 = require("./search.js");
 const logger_ts_1 = require("../utility/logging/logger.js");
 const staticResourceDispatch_ts_1 = require("./staticResourceDispatch.js");
@@ -77,6 +79,8 @@ const transaction_ts_1 = require("./transaction.js");
 const ordered_binary_1 = require("ordered-binary");
 const manageThreads_js_1 = require("../server/threads/manageThreads.js");
 const auditStore_ts_1 = require("./auditStore.js");
+const derivedIndexRegistry_ts_1 = require("./derivedIndexRegistry.js");
+const recordLockCoordinator_ts_1 = require("./recordLockCoordinator.js");
 const embedHook_ts_1 = require("./models/embedHook.js");
 const common_utils_ts_1 = require("../utility/common_utils.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
@@ -97,6 +101,9 @@ const { sortBy } = lodash_1.default;
 const { validateAttribute } = lmdbProcessRows_js_1.default;
 const NULL_WITH_TIMESTAMP = new Uint8Array(9);
 NULL_WITH_TIMESTAMP[8] = 0xc0; // null
+const sourceWriteTypes = new Set(['put', 'patch', 'delete', 'publish', 'message', 'invalidate', 'relocate']);
+const isSourceWriteType = (type) => sourceWriteTypes.has(type);
+const SOURCE_APPLY_POSITION = Symbol('sourceApplyPosition');
 const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
 const MAX_DATE_TIMESTAMP = 8.64e15;
 const RECORD_PRUNING_INTERVAL = 60000; // one minute
@@ -124,6 +131,20 @@ const MAX_COUNT_PAGE = 10_000;
 // How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask
 // check). Keeps a large scan from monopolizing the event loop without adding a yield per row.
 const COUNT_YIELD_INTERVAL = 2_048;
+// Smallest forward sample `getRecordCount` will extrapolate a record rate from; below it the scan runs
+// to completion and reports an exact count.
+const MIN_ESTIMATOR_SAMPLE = 1_000;
+// Budget intervals the forward scan may spend before it must estimate rather than keep scanning.
+const MAX_ESTIMATE_CHECKPOINTS = 20;
+// A store estimate's `count`, or 0 when the store answered with a shape that cannot be trusted --
+// DESIGN.md's invariant for this API family is that such an answer degrades rather than poisons.
+function usableCount(estimate) {
+    const { count, confidence } = estimate ?? {};
+    // `confidence` needs its own finiteness check, not just the range: `null >= 0 && null <= 1` is true
+    if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+        return 0;
+    return count;
+}
 envMngr.initSync();
 const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
 const LOCK_TIMEOUT = 10000;
@@ -416,6 +437,17 @@ function contextArgument(context) {
     const resolved = context.getContext?.() || context;
     return resolved instanceof DatabaseTransaction_ts_1.DatabaseTransaction ? { transaction: resolved } : resolved;
 }
+/** The cluster round never ran for a node-scoped handle, so no peer ever deferred to it. */
+function scopeViolation(handle, resolved, databaseName) {
+    if (resolved.scope !== 'cluster' || handle.clusterTsR !== undefined)
+        return undefined;
+    // The same predicate lock() fails closed on, not the transport alone: a coalesced caller re-checks
+    // this after its wait, and a transport unregistered during that wait leaves the database still
+    // clustered while the lookup answers undefined. Only the implicit Phase 0 case falls through.
+    if (!resolved.scopeRequested && !(0, recordLockCoordinator_ts_1.isClusterLockRequired)(databaseName) && !(0, recordLockCoordinator_ts_1.getClusterLockTransport)(databaseName))
+        return undefined;
+    return new hdbError_ts_1.ClientError('This transaction already holds a node-scoped lock on this record, so a cluster-scoped lock cannot be taken on top of it', 409);
+}
 /** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
 function isPlainOptions(value) {
     return (typeof value === 'object' &&
@@ -424,10 +456,23 @@ function isPlainOptions(value) {
         !(value instanceof URLSearchParams) &&
         value.id === undefined);
 }
+// Lets a transport push a received control entry straight to the right coordinator without
+// importing Table (which would be a cycle through databases.ts).
+(0, recordLockCoordinator_ts_1.setLockCoordinatorResolver)((database, tableName) => databases_ts_1.databases[database]?.[tableName]?.lockCoordinator, (database, tableName) => databases_ts_1.databases[database]?.[tableName]?.admittingCoordinator, (database, tableName) => {
+    const Table = databases_ts_1.databases[database]?.[tableName];
+    if (typeof Table?.writeLockControlEntry !== 'function')
+        return undefined;
+    return (entry) => Table.writeLockControlEntry(entry);
+});
 function makeTable(options) {
     var _a;
     const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, } = options;
     let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
+    // Set when the TTL exists only on this thread: either application code configured it at runtime, or
+    // an isolated application's schema was declared here. Hydrating persisted metadata does not set it:
+    // dedicated workers open unrelated shared tables too, whose scan remains owned by the pool.
+    let ttlConfiguredByApplication = false;
+    let ttlFromLoad = false; // true only around the creation-time call below
     evictionMs ??= 0;
     // Eviction without explicit expiration means expiration:0. Apply at construction so
     // describe_all sees it on every worker, not just ones that ran setTTLExpiration.
@@ -439,6 +484,9 @@ function makeTable(options) {
     if (!properties)
         properties = (0, jsonSchemaTypes_ts_1.projectAttributesToProperties)(attributes);
     const updateRecord = (0, RecordEncoder_ts_1.recordUpdater)(primaryStore, tableId, auditStore);
+    // Created on first cluster-scoped lock() or first arriving control entry, and only while a
+    // transport is registered for this database.
+    let lockCoordinator;
     let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
     let warnedFutureSourceVersion = false; // likewise (see getFromSource)
     let sourceLoad; // if a source has a load function (replicator), record it here
@@ -665,6 +713,28 @@ function makeTable(options) {
         }
         return { txnLogKey: version, nodeId };
     }
+    // Canonical-source applies (sourceApply), replay and replication notifications are never shed;
+    // dropping one would advance the source cursor past a write that never landed.
+    function assertDerivedIndexAdmission(options, transaction) {
+        if (options?.isNotification || transaction?.sourceApply || transaction?.isReplay)
+            return;
+        const reason = (0, derivedIndexRegistry_ts_1.derivedIndexWriteRejection)(auditStore, tableId);
+        if (reason)
+            throw new hdbError_ts_1.DerivedIndexLagError(reason);
+    }
+    function stageDerivedIndexEviction(transaction, id, version) {
+        if (!(0, derivedIndexRegistry_ts_1.hasDerivedIndexRegistration)(auditStore, tableId))
+            return;
+        const nodeId = (0, nodeIdMapping_ts_1.getThisNodeId)(auditStore) ?? 0;
+        auditStore.put(null, {
+            type: 'evict',
+            tableId,
+            recordId: id,
+            version,
+            nodeId,
+            extendedType: auditStore_ts_1.LOCAL_ONLY,
+        }, { transaction, nodeId });
+    }
     class TableResource extends Resource_ts_1.Resource {
         #record; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
         #changes; // the changes to the record that have been made (should not be modified directly)
@@ -673,13 +743,27 @@ function makeTable(options) {
         #savingOperation; // operation for the record is currently being saved
         #lockHandle; // the record lock acquired by lock() — scoped or hold
         #lockWritable; // set by #reloadLocked to let save() stage lock-writable updates
+        #writeGeneration;
+        [tracked_ts_1.ASSERT_TRACKED_WRITABLE](generation = this.#writeGeneration) {
+            if (!generation)
+                return;
+            if (generation.internalWrites > 0)
+                return;
+            if (generation !== this.#writeGeneration || generation.closed)
+                throw new hdbError_ts_1.ClientError('Can not modify an update instance after it has been saved; call update() again', 409);
+        }
+        [tracked_ts_1.GET_TRACKED_WRITE_GENERATION]() {
+            return (this.#writeGeneration ??= { closed: false, internalWrites: 0 });
+        }
         /**
          * Shared guard: if this instance is lock-writable but the handle is gone (expired or
          * released), throw 409 before staging any write. Covers update/invalidate/relocate/delete
          * in addition to the save() path. Every lock-writable instance carries its own handle in
          * #lockHandle (scoped and hold alike), so we never need to search the registry here.
          */
-        #assertLiveHandle(id) {
+        #assertLiveHandle(id, allowClosed = false) {
+            if (!allowClosed && this.#writeGeneration?.closed && (0, DatabaseTransaction_ts_1.writeKeyId)(id) === (0, DatabaseTransaction_ts_1.writeKeyId)(this.getId()))
+                this[tracked_ts_1.ASSERT_TRACKED_WRITABLE]();
             if (!this.#lockWritable)
                 return;
             const handle = this.#lockHandle;
@@ -687,7 +771,7 @@ function makeTable(options) {
             // exact key the lock was acquired for.
             if (handle.keyId !== (0, DatabaseTransaction_ts_1.writeKeyId)(id))
                 return;
-            if (handle.expired || handle.released) {
+            if (handle.isExpired()) {
                 throw (0, recordLock_ts_1.lockNotHeldError)(handle);
             }
         }
@@ -699,6 +783,7 @@ function makeTable(options) {
         static tableName = tableName;
         static tableId = tableId;
         static indices = indices;
+        static derivedIndexRuntime;
         static audit = audit;
         static databasePath = databasePath;
         static databaseName = databaseName;
@@ -784,8 +869,60 @@ function makeTable(options) {
             (async () => {
                 let userRoleUpdate = false;
                 let lastSequenceId;
+                let pendingApplyFailures;
+                const reportDroppedWrite = (event, context, error) => {
+                    const position = event === context ? context[SOURCE_APPLY_POSITION] : (event.timestamp ?? context[SOURCE_APPLY_POSITION]);
+                    const notification = (0, replicatedApplyFailure_ts_1.notifyReplicatedApplyFailure)(databaseName, {
+                        nodeId: event.nodeId ?? context.nodeId,
+                        table: event.table ?? context.table,
+                        localTime: event.localTime ?? context.localTime,
+                    }, position, error, tableName);
+                    pendingApplyFailures = pendingApplyFailures
+                        ? Promise.all([pendingApplyFailures, notification]).then(noop)
+                        : notification;
+                    return notification;
+                };
+                /** Cluster lock coordination entries (harper#483 Phase 1) describe no record. */
+                const applyLockControlEvent = (event, context) => {
+                    const entry = (0, recordLockCoordinator_ts_1.decodeLockControlPayload)(event.type, event.value);
+                    if (!entry) {
+                        logger_ts_1.logger.warn?.('discarding a malformed record lock control entry from', event.nodeId, event.type);
+                        return reportDroppedWrite(event, context, new Error('Malformed record lock control entry'));
+                    }
+                    const target = event.table ? databases_ts_1.databases[databaseName]?.[event.table] : _a;
+                    try {
+                        // The audit header's nodeId is the origin, translated on receive and preserved across
+                        // relays. The payload's own names are peer-supplied and prove nothing. Rebuild the id
+                        // map on a miss rather than waiting out the negative-cache window: a dropped release
+                        // leaves the key's home holding its grant until the delegation's own deadline, and
+                        // control entries are far too rare to drive the store.
+                        //
+                        // Inside the guard, not before it: that rebuild reads the audit store, and a throw
+                        // there would escape this sink and stall the apply loop for every later entry — the §8
+                        // rule that a receive boundary settles its callers and keeps admission closed.
+                        const author = (0, nodeIdMapping_ts_1.getNodeNameForId)(auditStore, event.nodeId, true);
+                        if (!author) {
+                            logger_ts_1.logger.warn?.('discarding a record lock control entry whose origin node could not be resolved');
+                            return reportDroppedWrite(event, context, new Error('Record lock control origin could not be resolved'));
+                        }
+                        // The coordinator getter fails closed on an unusable node identity. That is right for
+                        // an acquire and wrong here: rejecting out of this sink stalls the apply loop for
+                        // every later entry rather than dropping one.
+                        // `admittingCoordinator`, because `lockCoordinator` answers undefined while a transport
+                        // is momentarily unregistered — and this sink runs off the replication stream, not off
+                        // that transport. Dropping a peer's clean-handoff release there leaves the home holding
+                        // its grant for the delegation's whole deadline.
+                        target?.admittingCoordinator?.applyEntry(entry, author, event.timestamp);
+                    }
+                    catch (error) {
+                        logger_ts_1.logger.warn?.('dropping a record lock control entry: the coordinator is unavailable', error);
+                        return reportDroppedWrite(event, context, error);
+                    }
+                };
                 // perform the write of an individual write event
                 const writeUpdate = async (event, context) => {
+                    if ((0, auditStore_ts_1.isLockControlType)(event.type))
+                        return applyLockControlEvent(event, context);
                     const value = event.value;
                     const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
                     if (databaseName === hdbTerms_ts_1.SYSTEM_SCHEMA_NAME &&
@@ -816,6 +953,15 @@ function makeTable(options) {
                         async: true,
                     };
                     const id = event.id;
+                    if (!isSourceWriteType(event.type)) {
+                        logger_ts_1.logger.error?.('Unknown operation', event.type, event.id);
+                        const notification = reportDroppedWrite(event, context, new Error('Unknown source operation'));
+                        if (event.finished)
+                            await event.finished;
+                        return notification;
+                    }
+                    if (Table && event.type === 'put' && value == null && !shouldRevalidateEvents)
+                        await reportDroppedWrite(event, context, new Error('Source-applied put has no record content'));
                     const resource = await Table.getResource(id, context, options);
                     if (event.finished)
                         await event.finished;
@@ -837,12 +983,15 @@ function makeTable(options) {
                             return resource._writeInvalidate(id, value, options);
                         case 'relocate':
                             return resource._writeRelocate(id, options);
-                        default:
-                            logger_ts_1.logger.error?.('Unknown operation', event.type, event.id);
                     }
                 };
                 /** Keeps the writes to any one key in arrival order; see DESIGN.md (harper#2211). */
                 const stageWrite = (event, context) => {
+                    // A grant must not queue behind whatever the key it names is doing.
+                    if ((0, auditStore_ts_1.isLockControlType)(event.type) ||
+                        !isSourceWriteType(event.type) ||
+                        (event.type === 'put' && event.value == null && !shouldRevalidateEvents))
+                        return writeUpdate(event, context);
                     let chainKey;
                     try {
                         const Table = event.table ? databases_ts_1.databases[databaseName][event.table] : _a;
@@ -886,14 +1035,18 @@ function makeTable(options) {
                         omitCurrent: true,
                     };
                     const subscribeOnThisThread = source.subscribeOnThisThread
-                        ? source.subscribeOnThisThread((0, manageThreads_js_1.getWorkerIndex)(), subscriptionOptions)
-                        : (0, manageThreads_js_1.getWorkerIndex)() === 0;
+                        ? source.subscribeOnThisThread((0, manageThreads_js_1.applicationWorkerIndex)(), subscriptionOptions)
+                        : (0, manageThreads_js_1.runsApplicationCodeSingletons)(); // set up by the defining application's code, so it runs where that code does
                     const subscription = hasSubscribe && subscribeOnThisThread && (await source.subscribe?.(subscriptionOptions));
                     if (subscription) {
                         let txnInProgress;
                         // we listen for events by iterating through the async iterator provided by the subscription
                         for await (const event of subscription) {
+                            let failureEvent = event;
+                            let failurePosition;
+                            let applied = false;
                             try {
+                                failurePosition = event?.timestamp;
                                 if (!event || typeof event !== 'object') {
                                     logger_ts_1.logger.error?.('Bad subscription event', event);
                                     continue;
@@ -901,6 +1054,7 @@ function makeTable(options) {
                                 const firstWrite = event.type === 'transaction' ? event.writes[0] : event;
                                 if (!firstWrite) {
                                     logger_ts_1.logger.error?.('Bad subscription event', event);
+                                    await (0, replicatedApplyFailure_ts_1.notifyReplicatedApplyFailure)(databaseName, event, failurePosition, new Error('Subscription transaction has no writes'), tableName);
                                     continue;
                                 }
                                 event.source = source;
@@ -909,11 +1063,16 @@ function makeTable(options) {
                                 // there is no re-subscribe / sequence-id-resume path to recover it. Mark the context so the
                                 // commit retries such conflicts without a cap (see DatabaseTransaction commit).
                                 event.sourceApply = true;
+                                event[SOURCE_APPLY_POSITION] = failurePosition;
                                 if (event.type === 'end_txn') {
                                     // Capture the in-progress transaction in a stable local: the loop variable is reset
                                     // once this transaction completes (below), but the seq-id closure and the commit await
                                     // still need to reference it afterward.
                                     const committingTxn = txnInProgress;
+                                    if (committingTxn) {
+                                        failureEvent = committingTxn;
+                                        failurePosition = committingTxn[SOURCE_APPLY_POSITION];
+                                    }
                                     committingTxn?.resolve();
                                     let updateRecordedSequenceId;
                                     if (event.localTime && lastSequenceId !== event.localTime) {
@@ -996,6 +1155,7 @@ function makeTable(options) {
                                     let committed;
                                     try {
                                         committed = committingTxn ? await committingTxn.committed : undefined;
+                                        applied = true;
                                         if (event.onCommit) {
                                             // the onCommit callback can be async and carry associated work (e.g. blob
                                             // transfer); wait for it too before recording the sequence id. Pass the commit
@@ -1032,6 +1192,7 @@ function makeTable(options) {
                                             // than rethrow) so the current beginTxn still starts a fresh transaction with
                                             // correct boundaries instead of having its writes applied as standalone ones.
                                             logger_ts_1.logger.error?.('source-applied transaction commit failed during apply', error);
+                                            await (0, replicatedApplyFailure_ts_1.notifyReplicatedApplyFailure)(databaseName, txnInProgress, txnInProgress[SOURCE_APPLY_POSITION], error, tableName);
                                         }
                                         finally {
                                             // Clear it regardless of outcome so a rejected commit isn't re-awaited on the
@@ -1121,6 +1282,7 @@ function makeTable(options) {
                                         // standalone write: backpressure on the commit before pulling the next event,
                                         // and pass the commit resolution through to the callback.
                                         const committed = commitResolution ? await commitResolution : undefined;
+                                        applied = true;
                                         await event.onCommit(committed);
                                     }
                                 }
@@ -1131,6 +1293,15 @@ function makeTable(options) {
                             }
                             catch (error) {
                                 logger_ts_1.logger.error?.('error in subscription handler', error);
+                                if (!applied)
+                                    await (0, replicatedApplyFailure_ts_1.notifyReplicatedApplyFailure)(databaseName, failureEvent, failurePosition, error, tableName);
+                            }
+                            finally {
+                                while (pendingApplyFailures) {
+                                    const notification = pendingApplyFailures;
+                                    pendingApplyFailures = undefined;
+                                    await notification;
+                                }
                             }
                         }
                     }
@@ -1394,15 +1565,24 @@ function makeTable(options) {
          * This also informs the scheduling for record eviction.
          * @param opts Time in seconds until records expire, or an options object with `expiration`, `eviction`,
          * and `scanInterval` (all in seconds, all optional). Number form preserves any previously configured
-         * eviction/scanInterval; object form replaces all three.
+         * eviction/scanInterval; object form replaces all three. An internal schema ownership-only call with
+         * none of those values preserves the settings already loaded from the catalog.
          */
         static setTTLExpiration(opts) {
             if (opts == null || (typeof opts !== 'number' && typeof opts !== 'object'))
                 throw new Error('Invalid expiration value type');
+            const declaredHere = typeof opts === 'object' && opts.fromSchema;
+            const isolatedApplicationOwner = declaredHere && opts.isolatedApplicationOwner;
+            const preserveLoadedConfiguration = declaredHere && opts.expiration === undefined && opts.eviction === undefined && opts.scanInterval === undefined;
+            if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlConfiguredByApplication) {
+                ttlConfiguredByApplication = true;
+                // the scan owner may have changed with this: re-evaluate even if the interval did not
+                lastCleanupInterval = undefined;
+            }
             if (typeof opts === 'number') {
                 expirationMs = opts * 1000;
             }
-            else {
+            else if (!preserveLoadedConfiguration) {
                 // `??` so an explicit 0 is treated as the user's chosen value, not as "missing"
                 expirationMs = (opts.expiration ?? 0) * 1000;
                 evictionMs = (opts.eviction ?? 0) * 1000;
@@ -1410,10 +1590,19 @@ function makeTable(options) {
             }
             if (expirationMs < 0)
                 throw new Error('Expiration can not be negative');
-            // default to one quarter of the total expiration+eviction window
-            cleanupInterval = cleanupInterval || (expirationMs + evictionMs) / 4;
-            expirationScanScheduled = true;
-            scheduleCleanup();
+            if (!preserveLoadedConfiguration) {
+                // default to one quarter of the total expiration+eviction window
+                cleanupInterval = cleanupInterval || (expirationMs + evictionMs) / 4;
+                expirationScanScheduled = true;
+            }
+            // Re-evaluate an existing table-level scan after an ownership-only declaration, but do not
+            // create the default daily cleanup timer for a table that has only an @expiresAt field.
+            if (!preserveLoadedConfiguration || expirationScanScheduled || evictionMs)
+                scheduleCleanup();
+            // @expiresAt has its own interval rather than the cleanup timer above. Arm it whenever a live
+            // declaration introduces the attribute, including after this application already claimed TTL.
+            if (expiresAtProperty && !recordExpirationInterval)
+                runRecordExpirationEviction();
         }
         static getResidencyRecord(id) {
             // getSync (not get): callers consume the result synchronously (e.g. residency.includes(...) in a
@@ -1521,6 +1710,39 @@ function makeTable(options) {
         static async dropTable() {
             _a.assertSchemaMutable('drop a table');
             const rootStore = primaryStore.rootStore;
+            if (databaseName === databasePath &&
+                rootStore instanceof rocksdb_js_1.RocksDatabase &&
+                dbisDb.put !== dbisDb.putSync)
+                throw new Error(`Cannot drop ${databaseName}.${_a.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`);
+            // Release post-commit derived-index delivery before any destructive work: the runner's
+            // backend must have quiesced before its stores and native file are destroyed, and a
+            // same-name recreate must not race an owner still applying to the old generation.
+            const derivedIndexRuntime = _a.derivedIndexRuntime;
+            const restoreDerivedIndexesAfterFailedDrop = () => {
+                try {
+                    _a.derivedIndexRuntime = derivedIndexRuntime?.restoreAfterFailedDrop?.();
+                }
+                catch (restoreError) {
+                    _a.derivedIndexRuntime = undefined;
+                    logger_ts_1.logger.error?.(`Could not restore derived indexes after failed drop of ${databaseName}.${_a.tableName}`, restoreError);
+                }
+            };
+            try {
+                await derivedIndexRuntime?.close(true);
+            }
+            catch (error) {
+                restoreDerivedIndexesAfterFailedDrop();
+                throw error;
+            }
+            const abortStaleDrop = () => {
+                derivedIndexRuntime?.completeDrop?.(false);
+                _a.derivedIndexRuntime = undefined;
+                _a.cleanup();
+                if (databases_ts_1.databases[databaseName]?.[tableName] === _a)
+                    delete databases_ts_1.databases[databaseName][tableName];
+            };
+            let dropIdentityConfirmed = databaseName !== databasePath;
+            let primaryCatalogKey = _a.tableName + '/';
             if (databaseName === databasePath) {
                 // Persist a drop tombstone on the primary catalog entry BEFORE any
                 // destructive work. If the process dies or a column family drop fails
@@ -1528,11 +1750,21 @@ function makeTable(options) {
                 // the next startup (or a same-name create) completes the drop via
                 // completeInterruptedDrop in databases.ts instead of resurrecting
                 // the table.
-                const primaryCatalogKey = _a.tableName + '/';
+                let tombstoneWrite;
                 const writeTombstone = () => {
-                    const primaryMeta = dbisDb.getSync(primaryCatalogKey);
-                    if (!primaryMeta || primaryMeta.dropping)
-                        return;
+                    let primaryMeta = dbisDb.getSync(primaryCatalogKey);
+                    if (!primaryMeta && primaryKey) {
+                        const legacyPrimaryKey = `${_a.tableName}/${primaryKey}`;
+                        const legacyPrimaryMeta = dbisDb.getSync(legacyPrimaryKey);
+                        if (legacyPrimaryMeta?.isPrimaryKey) {
+                            primaryCatalogKey = legacyPrimaryKey;
+                            primaryMeta = legacyPrimaryMeta;
+                        }
+                    }
+                    if (!primaryMeta || (primaryMeta.tableId != null && primaryMeta.tableId !== tableId))
+                        return false;
+                    if (primaryMeta.dropping)
+                        return true;
                     primaryMeta.dropping = true;
                     // Stamps this drop's identity so the interrupted-drop retry budget in
                     // databases.ts can be scoped to THIS drop rather than the table name: a
@@ -1542,26 +1774,33 @@ function makeTable(options) {
                     // the budget by generation instead makes the new drop's tombstone carry
                     // its own fresh key regardless of what any worker last observed.
                     primaryMeta.dropGeneration = (0, node_crypto_1.randomUUID)();
-                    return dbisDb.put(primaryCatalogKey, primaryMeta);
+                    tombstoneWrite = dbisDb.put(primaryCatalogKey, primaryMeta);
+                    return true;
                 };
-                if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
-                    // withUpdateAttributesLock's locked section cannot be held across an await, so a durable
-                    // tombstone depends on put being rebound to putSync for RocksDB primary stores (see
-                    // createOpenDBIObject). Check that BEFORE writing anything: a tombstone left behind by a
-                    // refused drop would delete the table on the next load.
-                    if (dbisDb.put !== dbisDb.putSync)
-                        throw new Error(`Cannot drop ${databaseName}.${_a.tableName}: the catalog store's put is asynchronous, so the drop tombstone cannot be made durable before the column families are dropped`);
-                    withUpdateAttributesLock(rootStore, `drop table '${databaseName}.${_a.tableName}'`, writeTombstone);
+                try {
+                    if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                        // withUpdateAttributesLock's locked section cannot be held across an await, so a durable
+                        // tombstone depends on put being rebound to putSync for RocksDB primary stores.
+                        dropIdentityConfirmed = withUpdateAttributesLock(rootStore, `drop table '${databaseName}.${_a.tableName}'`, writeTombstone);
+                    }
+                    else {
+                        rootStore.transactionSync(() => {
+                            dropIdentityConfirmed = writeTombstone();
+                        });
+                        if (typeof tombstoneWrite?.then === 'function')
+                            await tombstoneWrite;
+                    }
                 }
-                else {
-                    let tombstoneWrite;
-                    rootStore.transactionSync(() => {
-                        tombstoneWrite = writeTombstone();
-                    });
-                    if (typeof tombstoneWrite?.then === 'function')
-                        await tombstoneWrite;
+                catch (error) {
+                    restoreDerivedIndexesAfterFailedDrop();
+                    throw error;
                 }
             }
+            if (!dropIdentityConfirmed) {
+                abortStaleDrop();
+                return;
+            }
+            _a.derivedIndexRuntime = undefined;
             // A get() against a sourcedFrom table resolves to its caller before the resolved
             // record's cache write has committed (see getFromSource) - the write lands "in the
             // background" for latency reasons. Flip this BEFORE removing the table from the
@@ -1574,7 +1813,8 @@ function makeTable(options) {
             // family drops below. If a drop fails past this point the table stays
             // invisible, and the tombstone guarantees the drop completes on the
             // next startup (or on a same-name create).
-            delete databases_ts_1.databases[databaseName][tableName];
+            if (databases_ts_1.databases[databaseName]?.[tableName] === _a)
+                delete databases_ts_1.databases[databaseName][tableName];
             // The above stops new source-fill writes from starting, but a write from a get()
             // that already returned to its caller may still be in flight. Dropping the column
             // families out from under that write is a genuine invariant violation, not just a
@@ -1605,14 +1845,21 @@ function makeTable(options) {
                 ]);
                 clearTimeout(timer);
                 if (result === timedOut) {
+                    derivedIndexRuntime?.completeDrop?.();
                     throw new Error(`dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.length} in-flight source-populated cache write(s) on ${tableName} to settle; refusing to drop the column families out from under a write that may still be staged. The drop tombstone is durable, so this will be retried on the next load.`);
                 }
             }
-            for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) {
-                if (entry.metadataFlags & auditStore_ts_1.HAS_BLOBS && entry.value) {
-                    (0, blob_ts_1.deleteBlobsInObject)(entry.value);
+            try {
+                for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) {
+                    if (entry.metadataFlags & auditStore_ts_1.HAS_BLOBS && entry.value) {
+                        (0, blob_ts_1.deleteBlobsInObject)(entry.value);
+                    }
                 }
             }
+            catch (error) {
+                derivedIndexRuntime?.completeDrop?.();
+                throw error;
+            }
             if (databaseName === databasePath) {
                 // part of a database.
                 // Drop the column families, then remove the catalog metadata - never
@@ -1629,13 +1876,13 @@ function makeTable(options) {
                 // same-name create completes the interrupted drop and writes fresh
                 // catalog rows, and clobbering those would orphan the new table.
                 const removeTombstonedCatalog = () => {
-                    const currentPrimary = dbisDb.getSync(_a.tableName + '/');
-                    if (!currentPrimary?.dropping)
+                    const currentPrimary = dbisDb.getSync(primaryCatalogKey);
+                    if (!currentPrimary?.dropping || (currentPrimary.tableId != null && currentPrimary.tableId !== tableId))
                         return false;
                     for (const attribute of attributes) {
                         dbisDb.remove(_a.tableName + '/' + attribute.name);
                     }
-                    dbisDb.remove(_a.tableName + '/');
+                    dbisDb.remove(primaryCatalogKey);
                     return true;
                 };
                 if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
@@ -1647,42 +1894,76 @@ function makeTable(options) {
                     // completeInterruptedDrop does), never an awaited drop(), or a
                     // concurrent create's wait would be stuck on a drop that the blocked
                     // event loop can never resolve, burning its full deadline before failing.
-                    const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
-                        for (const attribute of attributes) {
-                            const index = indices[attribute.name];
-                            if (index)
-                                try {
-                                    index.dropSync();
-                                }
-                                catch (error) {
-                                    ignoreAlreadyDropped(error);
-                                }
-                        }
-                        try {
-                            primaryStore.dropSync();
-                        }
-                        catch (error) {
-                            ignoreAlreadyDropped(error);
-                        }
-                        return removeTombstonedCatalog();
-                    });
-                    if (removed)
-                        await dbisDb.committed;
+                    let removed;
+                    try {
+                        removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
+                            const currentPrimary = dbisDb.getSync(primaryCatalogKey);
+                            if (!currentPrimary?.dropping || (currentPrimary.tableId != null && currentPrimary.tableId !== tableId))
+                                return false;
+                            for (const attribute of attributes) {
+                                const index = indices[attribute.name];
+                                if (index)
+                                    try {
+                                        index.customIndex?.resetDerivedStorage?.();
+                                        index.dropSync();
+                                    }
+                                    catch (error) {
+                                        ignoreAlreadyDropped(error);
+                                    }
+                            }
+                            try {
+                                primaryStore.dropSync();
+                            }
+                            catch (error) {
+                                ignoreAlreadyDropped(error);
+                            }
+                            return removeTombstonedCatalog();
+                        });
+                        if (removed)
+                            await dbisDb.committed;
+                    }
+                    catch (error) {
+                        derivedIndexRuntime?.completeDrop?.();
+                        throw error;
+                    }
+                    if (!removed) {
+                        abortStaleDrop();
+                        return;
+                    }
                 }
                 else {
                     // LMDB: no shared column-family double-drop, and its engine lock is
                     // transactional rather than this spin lock, so keep the awaited drop
                     // plus the same tombstone-guarded catalog removal.
-                    const drops = [];
-                    for (const attribute of attributes) {
-                        const index = indices[attribute.name];
-                        if (index)
-                            drops.push(index.drop().catch(ignoreAlreadyDropped));
+                    let removed;
+                    try {
+                        const currentPrimary = dbisDb.getSync(primaryCatalogKey);
+                        if (!currentPrimary?.dropping || (currentPrimary.tableId != null && currentPrimary.tableId !== tableId)) {
+                            abortStaleDrop();
+                            return;
+                        }
+                        const drops = [];
+                        for (const attribute of attributes) {
+                            const index = indices[attribute.name];
+                            if (index) {
+                                index.customIndex?.resetDerivedStorage?.();
+                                drops.push(index.drop().catch(ignoreAlreadyDropped));
+                            }
+                        }
+                        drops.push(primaryStore.drop().catch(ignoreAlreadyDropped));
+                        await Promise.all(drops);
+                        removed = removeTombstonedCatalog();
+                        if (removed)
+                            await dbisDb.committed;
+                    }
+                    catch (error) {
+                        derivedIndexRuntime?.completeDrop?.();
+                        throw error;
+                    }
+                    if (!removed) {
+                        abortStaleDrop();
+                        throw new Error(`Could not complete drop of ${databaseName}.${tableName}: a replacement table became current while the LMDB stores were being dropped`);
                     }
-                    drops.push(primaryStore.drop().catch(ignoreAlreadyDropped));
-                    await Promise.all(drops);
-                    if (removeTombstonedCatalog())
-                        await dbisDb.committed;
                 }
             }
             else {
@@ -1690,11 +1971,18 @@ function makeTable(options) {
                 // assigns `primaryStore.auditStore` — openAuditStore() assigns `rootStore.auditStore`, and
                 // this is the reference makeTable() was handed. Awaited so a pass suspended mid-removal has
                 // released the primary DBI before it is closed and unlinked.
-                await auditStore?.stopAuditCleanup?.();
-                (0, storageReclamation_ts_1.removeStorageReclamation)(primaryStore.path);
-                await primaryStore.close();
-                node_fs_1.default.unlinkSync(primaryStore.path);
+                try {
+                    await auditStore?.stopAuditCleanup?.();
+                    (0, storageReclamation_ts_1.removeStorageReclamation)(primaryStore.path);
+                    await primaryStore.close();
+                    node_fs_1.default.unlinkSync(primaryStore.path);
+                }
+                catch (error) {
+                    derivedIndexRuntime?.completeDrop?.();
+                    throw error;
+                }
             }
+            derivedIndexRuntime?.completeDrop?.();
             signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, hdbTerms_ts_1.OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName));
         }
         // #section: read-path
@@ -1716,7 +2004,7 @@ function makeTable(options) {
                     records: './', // an href to the records themselves
                     name: tableName,
                     database: databaseName,
-                    auditSize: auditStore instanceof rocksdb_js_1.RocksDatabase ? auditStore.getKeysCount() : auditStore?.getStats().entryCount,
+                    auditSize: auditStore?.getStats().entryCount,
                     attributes,
                     recordCount: undefined,
                     estimatedRecordRange: undefined,
@@ -1947,12 +2235,18 @@ function makeTable(options) {
             else {
                 id = requestTargetToId(target);
             }
+            if (this.#writeGeneration?.closed) {
+                this.#changes = undefined;
+                this.#writeGeneration = undefined;
+            }
+            this.#assertLiveHandle(id, true);
             const context = this.getContext();
             const envTxn = txnForContext(context);
             if (!envTxn)
                 throw new Error('Can not update a table resource outside of a transaction');
             // record in the list of updating records so it can be written to the database when we commit
-            if (updates === false) {
+            // `false` is the patch-cancel sentinel, not a record root — but only incrementally.
+            if (updates === false && !fullUpdate) {
                 // TODO: Remove from transaction
                 return this;
             }
@@ -2000,14 +2294,22 @@ function makeTable(options) {
                     });
                 }
             }
-            return (0, when_ts_1.when)(this._writeUpdate(id, this.#changes, fullUpdate), () => this);
+            // Keep absent changes distinguishable from an explicit empty patch: framework-created
+            // post/publish updates do not necessarily mutate or save the instance.
+            // A supplied root must reach validation as itself, not as the staged changes (harper#1298).
+            const recordRoot = updates === undefined ? this.#changes : updates;
+            return (0, when_ts_1.when)(this._writeUpdate(id, recordRoot, fullUpdate), () => this);
         }
         /**
          * Save any changes into this instance to the current transaction
          */
         save() {
-            this.#assertLiveHandle(this.getId()); // a write through a released or expired lock never lands
             const operation = this.#savingOperation;
+            if (!this.#lockWritable &&
+                this.#writeGeneration?.closed &&
+                (!operation || operation.writeGeneration === this.#writeGeneration))
+                return;
+            this.#assertLiveHandle(operation?.key ?? this.getId()); // a write through a released or expired lock never lands
             if ((!operation || operation.dropped) && this.#lockWritable && this.#lockHandle?.hold) {
                 // A held lock's record stages its update here rather than at lock() time: it is often
                 // written after the acquiring transaction has already completed, which would have
@@ -2019,7 +2321,7 @@ function makeTable(options) {
                 // released between lock acquisition and this save(), throw 409 rather than silently
                 // committing stale data. Every lock-writable instance carries its own handle.
                 const saveHandle = this.#lockHandle;
-                if (saveHandle.expired || saveHandle.released) {
+                if (saveHandle.isExpired()) {
                     throw (0, recordLock_ts_1.lockNotHeldError)(saveHandle);
                 }
                 const changes = this.#changes;
@@ -2067,12 +2369,23 @@ function makeTable(options) {
                 // resolve before that native commit actually settles. Chain on innerCommit (as the
                 // lock-writable hold branch above already does) so callers awaiting save() see the
                 // write durably land, not just the outer (possibly premature) resolution.
-                const result = this.#saveOperation(operation);
+                let result;
+                try {
+                    result = this.#saveOperation(operation);
+                }
+                catch (error) {
+                    if (!operation.saved)
+                        this.#savingOperation = operation;
+                    throw error;
+                }
                 const innerCommit = operation.innerCommit;
                 return innerCommit ? (0, when_ts_1.when)(innerCommit, () => result) : result;
             }
         }
         #saveOperation(operation) {
+            // LMDB validates staged writes at transaction commit, so bind a lazy update to the
+            // generation selected by save() before another update can replace its changes.
+            operation.captureChanges?.();
             const transaction = txnForContext(this.getContext());
             const holder = operation.stagedIn;
             // never-drop-on-conflict lives on the transaction and would not travel with the write, so an
@@ -2090,13 +2403,26 @@ function makeTable(options) {
                 // merge and index diff would be relative to a record that may never land.
                 operation.priorWrite = undefined;
                 operation.deferSave = false;
-                return (0, when_ts_1.when)(transaction.addWrite(operation), () => operation.promise ?? operation.result);
+                const result = (0, when_ts_1.when)(transaction.addWrite(operation), () => operation.promise ?? operation.result);
+                this.#closeWriteChain(operation);
+                return result;
             }
             const owner = holder ?? transaction;
-            if (owner.save)
-                return owner.save(operation) || operation.promise || operation.result;
+            if (owner.save) {
+                const result = owner.save(operation) || operation.promise || operation.result;
+                this.#closeWriteChain(operation);
+                return result;
+            }
+        }
+        #closeWriteChain(operation) {
+            const owner = operation.stagedIn;
+            for (let write = operation; write && !write.instanceClosed; write = write.priorWrite) {
+                if (write === operation || owner?.ownedWrites?.has(write))
+                    (0, DatabaseTransaction_ts_1.closeWriteInstance)(write);
+            }
         }
         addTo(property, value) {
+            this[tracked_ts_1.ASSERT_TRACKED_WRITABLE]();
             if (typeof value === 'number' || typeof value === 'bigint') {
                 if (this.#savingOperation?.fullUpdate)
                     this.set(property, (+this.getProperty(property) || 0) + value);
@@ -2152,6 +2478,7 @@ function makeTable(options) {
             const context = this.getContext();
             checkValidId(id);
             const transaction = txnForContext(this.getContext());
+            assertDerivedIndexAdmission(options, transaction);
             const write = {
                 key: id,
                 store: primaryStore,
@@ -2159,6 +2486,7 @@ function makeTable(options) {
                 entry: this.#entry,
                 recordVersion: options?.version,
                 lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
+                reloadCommitBase: true,
                 commit: (txnTime, existingEntry, _retry, transaction) => {
                     const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
                     write.skipped = false; // reset on each retry; cleanup happens after commit if still true
@@ -2201,6 +2529,7 @@ function makeTable(options) {
             const context = this.getContext();
             checkValidId(id);
             const transaction = txnForContext(this.getContext());
+            assertDerivedIndexAdmission(options, transaction);
             const write = {
                 key: id,
                 store: primaryStore,
@@ -2208,6 +2537,7 @@ function makeTable(options) {
                 entry: this.#entry,
                 recordVersion: options?.version,
                 lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
+                reloadCommitBase: true,
                 before: this.constructor.source?.relocate && !context?.source
                     ? this.constructor.source.relocate.bind(this.constructor.source, id, undefined, context)
                     : undefined,
@@ -2301,9 +2631,8 @@ function makeTable(options) {
                     if (primaryStore.hasLock(id, entry.version))
                         return;
                 }
-                // evictions never go in the audit log, so we can not record a deletion entry for the eviction
-                // as there is no corresponding audit entry and it would never get cleaned up. So we must simply
-                // removed the entry entirely, but first cleanup indices
+                // Eviction is not a canonical delete. Indexed caching tables add a local-only control entry so
+                // their derived indexes can remove the resident projection without exposing a delete event.
                 let lmdbCompletion;
                 if (primaryStore.ifVersion) {
                     // lmdb: the index cleanup and the record removal are both version-guarded optimistic writes.
@@ -2317,6 +2646,7 @@ function makeTable(options) {
                 }
                 else {
                     updateIndices(id, existingRecord, null, options);
+                    stageDerivedIndexEviction(transaction, id, existingVersion);
                     (0, RecordEncoder_ts_1.removeEntry)(primaryStore, entry ?? primaryStore.getEntry(id), options);
                 }
                 committed = true;
@@ -2412,13 +2742,24 @@ function makeTable(options) {
             }
             const id = target != null ? requestTargetToId(target) : this.getId();
             checkValidId(id);
+            this.#assertLiveHandle(id);
             const resolved = (0, recordLock_ts_1.resolveLockOptions)(options);
             const context = this.getContext();
             const link = txnForContext(context);
             const keyId = (0, DatabaseTransaction_ts_1.writeKeyId)(id);
+            // Before the re-entrant paths, not after: a transaction that already holds this key
+            // node-scoped would otherwise be handed that handle back for an explicit cluster request,
+            // while the same request on a fresh key fails closed.
+            if (resolved.scope === 'cluster' &&
+                (resolved.scopeRequested || (0, recordLockCoordinator_ts_1.isClusterLockRequired)(databaseName)) &&
+                !(0, recordLockCoordinator_ts_1.getClusterLockTransport)(databaseName))
+                return Promise.reject(new hdbError_ts_1.LockUnavailableError(`Cluster-scoped record locks are not available on ${databaseName}: no record lock transport is registered`));
             const held = this.#lockHandle;
-            if (held && !held.released && !held.expired && held.keyId === keyId) {
+            if (held && !held.isExpired() && held.keyId === keyId) {
                 // Re-entrant: upgrade to hold if requested, then preserve staged changes.
+                const violation = scopeViolation(held, resolved, databaseName);
+                if (violation)
+                    return Promise.reject(violation);
                 if (resolved.hold && !held.hold) {
                     held.upgradeToHold(resolved.lease);
                     // The scoped phase eagerly staged a TransactionWrite (see #reloadLocked); hold
@@ -2432,7 +2773,10 @@ function makeTable(options) {
                 return Promise.resolve(this.#reloadLocked(id, undefined, true));
             }
             const scoped = link.recordLockFor(primaryStore, keyId);
-            if (scoped && !scoped.released && !scoped.expired) {
+            if (scoped && !scoped.isExpired()) {
+                const violation = scopeViolation(scoped, resolved, databaseName);
+                if (violation)
+                    return Promise.reject(violation);
                 if (resolved.hold && !scoped.hold) {
                     // Upgrade scoped → hold: flip the existing handle object to hold mode so every
                     // instance that already references this handle stays valid.  Retiring and creating a
@@ -2445,6 +2789,18 @@ function makeTable(options) {
                 // Already held with the same type: re-entrant return. Preserve any staged changes.
                 return Promise.resolve(this.#reloadLocked(id, scoped, true));
             }
+            // Cluster scope needs a registered transport. An EXPLICIT { scope: 'cluster' } without one is
+            // a caller asking for a guarantee this node cannot make, so it fails closed rather than
+            // silently returning the node-local lock; the default keeps Phase 0 behavior, which is what
+            // a build with no replication has anyway.
+            // The getter fails closed on an unusable node identity, and lock() answers with a promise.
+            let coordinator;
+            try {
+                coordinator = resolved.scope === 'node' ? undefined : _a.lockCoordinator;
+            }
+            catch (error) {
+                return Promise.reject(error);
+            }
             const key = (0, recordLock_ts_1.lockAttemptKey)(tableId, id);
             // Coalesce concurrent lock() calls for the same key inside one link so they don't
             // self-block: Promise.all([T.lock(id), T.lock(id)]) would otherwise have both calls
@@ -2457,6 +2813,10 @@ function makeTable(options) {
                 // The follower waits on the leader's acquisition, but only for its own timeout.
                 let followerTimer;
                 const followerTimedOut = Symbol('follower timeout');
+                // Why the leader failed, so the follower can report that instead of inventing contention
+                // when its own budget runs out. A leader 503 means the guarantee could not be established
+                // at all; retrying is still right (the condition may clear) but 423 at the end is not.
+                let leaderFailure;
                 const followerStart = Date.now();
                 const followerDeadline = new Promise((_, reject) => {
                     followerTimer = setTimeout(() => reject(followerTimedOut), resolved.timeout).unref();
@@ -2471,13 +2831,24 @@ function makeTable(options) {
                         throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
                     const remaining = resolved.timeout - (Date.now() - followerStart);
                     if (remaining <= 0)
-                        throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
-                    return this.lock(target, { ...resolved, timeout: remaining });
+                        throw leaderFailure ?? new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
+                    // Carry the scope only if the caller named it: spreading the resolved options would turn
+                    // a defaulted 'cluster' into an explicit one, which is fail-closed when no transport is
+                    // registered.
+                    return this.lock(target, {
+                        lease: resolved.lease,
+                        timeout: remaining,
+                        hold: resolved.hold,
+                        scope: resolved.scopeRequested ? resolved.scope : undefined,
+                    });
                 };
                 return Promise.race([pending, followerDeadline]).then(() => {
                     clearTimeout(followerTimer);
                     const acquired = link.recordLockFor(primaryStore, keyId);
-                    if (acquired && !acquired.released && !acquired.expired) {
+                    if (acquired && !acquired.isExpired()) {
+                        const violation = scopeViolation(acquired, resolved, databaseName);
+                        if (violation)
+                            throw violation;
                         if (resolved.hold && !acquired.hold) {
                             detachScopedUpgradeWrite(link, keyId, acquired);
                             acquired.upgradeToHold(resolved.lease);
@@ -2487,20 +2858,89 @@ function makeTable(options) {
                     return retryOnRemainingBudget();
                 }, (error) => {
                     clearTimeout(followerTimer);
+                    // A follower that simply ran out of its own wait was waiting on another caller in this
+                    // process, which is the contention 423 describes. But if the LEADER failed for a reason
+                    // that is not contention, that reason is the true one — keep it and report it if the
+                    // retries below also run out, rather than ending on a 423 for a key nobody held.
                     if (error === followerTimedOut)
                         throw new hdbError_ts_1.ClientError(`Record is locked and was not released in time`, 423);
+                    if (error instanceof hdbError_ts_1.LockUnavailableError)
+                        leaderFailure = error;
                     return retryOnRemainingBudget();
                 });
             }
             const pendingPromise = (0, recordLock_ts_1.acquireRecordKey)(link, primaryStore, key, keyId, resolved.timeout, resolved.lease, resolved.hold);
-            link.registerPendingLock(primaryStore, keyId, pendingPromise);
-            return pendingPromise.then((handle) => {
-                link.unregisterPendingLock(primaryStore, keyId);
-                if (link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits) {
+            const clusterStart = node_perf_hooks_1.performance.now();
+            // What a follower waits on must span the cluster round and registration, not just the native
+            // acquire. Waking it at the native hand-off leaves it in a window where the key is held but no
+            // handle is registered, so it retries and parks on the leader's own lock for its full timeout
+            // — inside a transaction that cannot finish until it gives up.
+            const acquisition = pendingPromise.then(async (handle) => {
+                const closedWhileWaiting = () => link.open === DatabaseTransaction_ts_1.TRANSACTION_STATE.CLOSED && !link.saveCommits;
+                if (closedWhileWaiting()) {
                     // The transaction was aborted while this call waited; nothing would ever release the handle.
                     handle.release();
                     throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
                 }
+                // Anything that fails from here must give the native key back, or it becomes a lock this
+                // caller does not know it owns.
+                // Re-resolved, not the snapshot taken before `acquireRecordKey`: that wait can run the
+                // caller's whole timeout, long enough for harper-pro to register the transport on this
+                // worker. Using the snapshot would take the native key alone and hand back a node-scoped
+                // handle while a peer that already had the transport is granted the same key.
+                try {
+                    if (resolved.scope !== 'node')
+                        coordinator = _a.lockCoordinator ?? coordinator;
+                }
+                catch (error) {
+                    // The getter fails closed on an unusable node identity, and that has to reach the caller
+                    // the same way it does before the wait. Swallowing it let an implicit cluster lock fall
+                    // through to node-local authority — the one outcome failing closed exists to prevent —
+                    // because `coordinator` is still whatever it was, including undefined.
+                    handle.release();
+                    throw error;
+                }
+                if (coordinator) {
+                    try {
+                        // Not a 423 when the budget is gone, and not a skip either: the native wait can consume
+                        // the whole timeout, and `acquire` with no wait left still admits from a live delegation
+                        // or a local grant without sending anything. Only if it cannot does the caller learn the
+                        // guarantee was unavailable — which is not the same as the key being held.
+                        const remaining = Math.max(0, resolved.timeout - (node_perf_hooks_1.performance.now() - clusterStart));
+                        const round = await coordinator.acquire(id, resolved.lease, remaining);
+                        // Resolved through the getter rather than captured, so a transport swap between
+                        // acquisition and release reaches the coordinator that now owns the delegation.
+                        if (!handle.joinClusterRound(round.tsR, resolved.lease, round.mintedMono, () => _a.admittingCoordinator?.release(id, round.admissionId))) {
+                            // The round completed inside its lease but the lease elapsed before the handle
+                            // could take it. The coordinator still holds it, and only this call knows the
+                            // hold was never handed out.
+                            // The getter, not the captured coordinator: after a transport swap the captured one no
+                            // longer owns this admission, so releasing through it would be a silent no-op.
+                            // `.then`, not `Promise.resolve(release())`: the call can throw synchronously, and that
+                            // throw would escape the catch and replace the 423 below with an internal error.
+                            Promise.resolve()
+                                .then(() => _a.admittingCoordinator?.release(id, round.admissionId))
+                                .catch(noop);
+                            // 503, not 423: the home granted this key to US and the lease elapsed before the handle
+                            // could take it, so nobody ever held it. The coordinator classifies the same thing the
+                            // same way — see its `timeout` denial.
+                            throw new hdbError_ts_1.LockUnavailableError(`A cluster record lock on ${databaseName}.${tableName} was granted after its lease had elapsed`);
+                        }
+                        // A recall must be able to fence a write this handle staged and then unlocked, so
+                        // the coordinator needs a way to revoke it — see LockCoordinator.registerAdmission.
+                        // The getter again: a swap during the acquisition moved this admission to the
+                        // successor, and registering on the predecessor would revoke a handle that is fine.
+                        _a.admittingCoordinator?.registerAdmission(round.admissionId, () => handle.revokeLease());
+                    }
+                    catch (error) {
+                        handle.release();
+                        throw error;
+                    }
+                    if (closedWhileWaiting()) {
+                        handle.release();
+                        throw new hdbError_ts_1.ServerError('Transaction was closed while waiting for a record lock', 500);
+                    }
+                }
                 link.registerRecordLock(handle);
                 if (link.saveCommits && context?.timestamp)
                     handle.noteCandidateFloor(context.timestamp);
@@ -2530,10 +2970,15 @@ function makeTable(options) {
                 }
                 // ImmediateTransaction: no clock pinning in lock(); save() stamps each write
                 // from the committed handle floor for both scoped and hold handles.
+                return handle;
+            });
+            link.registerPendingLock(primaryStore, keyId, acquisition);
+            return acquisition.then((handle) => {
+                link.unregisterPendingLock(primaryStore, keyId);
                 return this.#reloadLocked(id, handle);
-            }, (err) => {
+            }, (error) => {
                 link.unregisterPendingLock(primaryStore, keyId);
-                throw err;
+                throw error;
             });
         }
         #reloadLocked(id, holdHandle, preserveChanges = false) {
@@ -2733,6 +3178,7 @@ function makeTable(options) {
             const context = this.getContext();
             const transaction = txnForContext(context);
             const replaying = transaction.isReplay === true;
+            assertDerivedIndexAdmission(options, transaction);
             checkValidId(id);
             if (fullUpdate && recordUpdate == null && options?.isNotification) {
                 // A source/replication-applied put must carry the record; these applies skip record
@@ -2747,6 +3193,16 @@ function makeTable(options) {
                 }
                 return;
             }
+            let captureChanges;
+            if (recordUpdate === undefined) {
+                let captured = false;
+                captureChanges = () => {
+                    if (!captured) {
+                        captured = true;
+                        recordUpdate = this.#changes;
+                    }
+                };
+            }
             const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
             const writeToSource = () => {
                 if (!this.constructor.source || context?.source)
@@ -2768,12 +3224,19 @@ function makeTable(options) {
                     }
                 }
             };
+            const receiverId = this.getId();
+            const closesReceiver = !this.isCollection &&
+                !isSearchTarget(receiverId) &&
+                (id === receiverId || (0, DatabaseTransaction_ts_1.writeKeyId)(id) === (0, DatabaseTransaction_ts_1.writeKeyId)(receiverId));
             const write = {
                 key: id,
                 store: primaryStore,
                 entry,
                 nodeName: context?.nodeName,
                 fullUpdate,
+                chainsStagedState: true,
+                // copy-apply rows keep their pre-read base: one read per row, healed by the post-copy replay
+                reloadCommitBase: options?.isCopyApply !== true,
                 deferSave: true,
                 // the origin's record version on an applied write; absent for a locally-originated one
                 recordVersion: options?.version,
@@ -2782,9 +3245,10 @@ function makeTable(options) {
                 // Only attach the hold handle when it covers exactly this key; off-key writes
                 // are ordinary and must not carry an unrelated hold's handle.
                 lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
+                writeGeneration: !this.#lockWritable && closesReceiver ? this[tracked_ts_1.GET_TRACKED_WRITE_GENERATION]() : undefined,
+                captureChanges,
                 validate: (txnTime, committedBy = transaction) => {
-                    if (!recordUpdate)
-                        recordUpdate = this.#changes;
+                    write.captureChanges?.();
                     if (fullUpdate || (recordUpdate && (0, tracked_ts_1.hasChanges)(this.#changes === recordUpdate ? this : recordUpdate))) {
                         if (!context?.source) {
                             committedBy.checkOverloaded();
@@ -2834,10 +3298,13 @@ function makeTable(options) {
                                             : txnTime;
                             }
                             if (createdTimeProperty) {
-                                if (entry?.value) {
+                                // the reloaded commit base, not the pre-read one: a full PUT racing a create
+                                // would otherwise stamp a fresh created time over the real one
+                                const base = write.entry;
+                                if (base?.value) {
                                     if (fullUpdate || recordUpdate[createdTimeProperty.name]) {
                                         // make sure to retain original created time
-                                        recordUpdate[createdTimeProperty.name] = entry?.value[createdTimeProperty.name];
+                                        recordUpdate[createdTimeProperty.name] = base.value[createdTimeProperty.name];
                                     }
                                 }
                                 else {
@@ -2954,6 +3421,8 @@ function makeTable(options) {
                         // existing timestamp, which means that we received updates out of order, and must resequence the application
                         // of the updates to the record to ensure consistency across the cluster
                         // TODO: can the previous version be older, but even more previous version be newer?
+                        let belowAuditFloor = false;
+                        let dedupVersionCouldBeRetained;
                         if (audit) {
                             // A re-delivered out-of-order write (full-copy audit-replay re-delivers writes) must not have
                             // its commutative ops re-folded. additionalAuditRefs is the record's own list of folded
@@ -2990,7 +3459,7 @@ function makeTable(options) {
                             // Resolve the oldest retained entry once, for the same log the dedup reads.
                             let oldestRetainedAuditTime;
                             let oldestRetainedAuditTimeResolved = false;
-                            const dedupVersionCouldBeRetained = (version) => {
+                            dedupVersionCouldBeRetained = (version) => {
                                 if (!isRocksDB)
                                     return true; // LMDB keeps its exact, unbounded lookup (keyed by local audit time)
                                 if (!oldestRetainedAuditTimeResolved) {
@@ -3032,6 +3501,19 @@ function makeTable(options) {
                                     return; // duplicate already applied; avoid the resequencing walk
                                 }
                             }
+                            // The walk terminates at this write only by reaching an audit entry whose log key is at or below
+                            // txnTime (the loop condition below), so below the floor it cannot: it runs the whole retained
+                            // chain to an outcome the floor already determines (harper#2642). txnTime is the coordinate
+                            // because it is the one that loop compares; txnLogKey addresses this write's own entry, a
+                            // different question that dedupVersionCouldBeRetained already answers. An unknown floor is
+                            // Infinity, which fails closed for a cursor check but has to fail OPEN here — walk rather than
+                            // discard. RocksDB only: LMDB keeps its exact, unbounded reconciliation.
+                            if (isRocksDB && precedesExisting < 0) {
+                                const auditFloor = (0, auditStore_ts_1.getAuditFloor)(auditStore);
+                                belowAuditFloor = Number.isFinite(auditFloor) && txnTime < auditFloor;
+                            }
+                        }
+                        if (audit && !belowAuditFloor) {
                             // incremental CRDT updates are only available with audit logging on
                             const initialAuditHead = isRocksDB
                                 ? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
@@ -3039,7 +3521,12 @@ function makeTable(options) {
                             let localTime = initialAuditHead.txnLogKey;
                             let auditedVersion = existingEntry.version;
                             logger_ts_1.logger.debug?.('Applying CRDT update to record with id: ', id, 'txn time', new Date(txnTime), 'applying later update from:', new Date(auditedVersion), 'local recorded time', new Date(localTime));
-                            let nodeId = initialAuditHead.nodeId;
+                            // Normalized here and not at the lookup, because the two sources of an undefined nodeId do not
+                            // mean the same thing: a record's own nodeId is resolved by the same expression as the id its
+                            // audit entry is logged under, which applies `?? 0` (RecordEncoder), so absent means log 0;
+                            // `previousNodeId` is never encoded, so absent there means the log is unknown and only the
+                            // aggregate lookup can resolve a cross-origin predecessor.
+                            let nodeId = initialAuditHead.nodeId ?? 0;
                             const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
                             const auditRefsToVisit = existingEntry.additionalAuditRefs
                                 ? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
@@ -3276,6 +3763,32 @@ function makeTable(options) {
                                 return;
                             }
                         }
+                        else if (belowAuditFloor) {
+                            // The walk cannot reach this write, so it may contribute only what is order-independent: its
+                            // commutative ops. Whether a plain field survives depends on what newer writes did to that key,
+                            // which is what the purged history no longer answers, so applying one would resurrect state a
+                            // newer put may have erased. Everything else loses to the strictly newer head: a full update, a
+                            // head carrying no record (a delete or a residency-omitted record, which an op must not
+                            // resurrect), and an op-less patch. Bare return, no writeCommit, so no audit record references
+                            // the losing update's pre-saved blobs.
+                            incrementalUpdateToApply = fullUpdate || existingRecord == null ? null : (0, crdt_ts_1.commutativeOpsOf)(recordUpdate);
+                            if (!incrementalUpdateToApply) {
+                                write.skipped = true;
+                                return;
+                            }
+                            // The surviving head's addressable log-key pointer lives in these, wherever the record and log
+                            // clocks differ.
+                            if (existingEntry.additionalAuditRefs) {
+                                for (const ref of existingEntry.additionalAuditRefs) {
+                                    additionalAuditRefs.push(ref);
+                                }
+                            }
+                            // Once the walk no longer runs, this ref is what the read-your-writes check above matches a
+                            // re-delivery of these ops on. Best-effort, like every other guard here: the encoder bounds
+                            // the persisted list, so an identity can age out of it (harper#1148's full-copy convergence
+                            // is the backstop).
+                            additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
+                        }
                         else if (fullUpdate) {
                             // if no audit, we can't accurately do incremental updates, so we just assume the last update
                             // was the same type. Assuming a full update this record update loses and there are no changes —
@@ -3305,8 +3818,8 @@ function makeTable(options) {
                     if (recordToStore && recordToStore.getRecord)
                         throw new Error('Can not assign a record to a record, check for circular references');
                     if (residencyId == undefined) {
-                        if (entry?.residencyId)
-                            context.previousResidency = _a.getResidencyRecord(entry.residencyId);
+                        if (existingEntry?.residencyId)
+                            context.previousResidency = _a.getResidencyRecord(existingEntry.residencyId);
                         const residency = residencyFromFunction(_a.getResidency(recordToStore, context));
                         if (residency) {
                             if (!residency.includes(server.hostname)) {
@@ -3518,6 +4031,7 @@ function makeTable(options) {
             this.#assertLiveHandle(id);
             const context = this.getContext();
             const transaction = txnForContext(context);
+            assertDerivedIndexAdmission(options, transaction);
             checkValidId(id);
             const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
             const write = {
@@ -3525,6 +4039,7 @@ function makeTable(options) {
                 store: primaryStore,
                 entry,
                 chainsStagedState: true,
+                reloadCommitBase: true,
                 nodeName: context?.nodeName,
                 recordVersion: options?.version,
                 lockHandle: this.#lockHandle && this.#lockHandle.keyId === (0, DatabaseTransaction_ts_1.writeKeyId)(id) ? this.#lockHandle : undefined,
@@ -3736,6 +4251,7 @@ function makeTable(options) {
             // objects. Entries are small and shallow; the clone is cheap next to the query.
             conditions = cloneConditions(conditions);
             let orderAlignedCondition;
+            let syntheticOrderCondition;
             const filtered = {};
             function prepareConditions(conditions, operator) {
                 // some validation:
@@ -3875,14 +4391,19 @@ function makeTable(options) {
                             // if it is indexed, we add a pseudo-condition to align with the natural sort order of the index.
                             // the primary key has no secondary index, but the primary store is itself keyed in
                             // primary-key order, so scanning it is already aligned with the sort
-                            orderAlignedCondition = { ...sort, comparator: 'sort' };
+                            orderAlignedCondition = syntheticOrderCondition = { ...sort, comparator: 'sort' };
                             conditions.push(orderAlignedCondition);
                         }
                         else if (conditions.length === 0 && !target.allowFullScan)
                             throw (0, hdbError_ts_1.handleHDBError)(new Error(), `${Array.isArray(attribute_name) ? attribute_name.join('.') : attribute_name} is not indexed and not combined with any other conditions`, 404);
                     }
-                    if (orderAlignedCondition)
+                    if (orderAlignedCondition) {
                         orderAlignedCondition.descending = Boolean(sort.descending);
+                        if (orderAlignedCondition.maxIndexLagMilliseconds === undefined)
+                            orderAlignedCondition.maxIndexLagMilliseconds = sort.maxIndexLagMilliseconds;
+                        if (orderAlignedCondition.waitForIndexMilliseconds === undefined)
+                            orderAlignedCondition.waitForIndexMilliseconds = sort.waitForIndexMilliseconds;
+                    }
                 }
             }
             conditions = orderConditions(conditions, operator);
@@ -3902,9 +4423,11 @@ function makeTable(options) {
                     }
                 }
                 else {
-                    // if we had to add an aligned condition that isn't first, we remove it and do ordering later
-                    if (orderAlignedCondition)
-                        conditions.splice(conditions.indexOf(orderAlignedCondition), 1);
+                    // if we had to add an aligned condition that isn't first, we remove it and do ordering later —
+                    // only the one we added; a caller's own condition on the sort attribute is still a filter
+                    const syntheticIndex = syntheticOrderCondition ? conditions.indexOf(syntheticOrderCondition) : -1;
+                    if (syntheticIndex >= 0)
+                        conditions.splice(syntheticIndex, 1);
                     postOrdering = sort;
                 }
             }
@@ -3949,150 +4472,156 @@ function makeTable(options) {
             const recordAccess = boundRowFilter || typeof target.vectorFilter === 'function'
                 ? { rowFilter: boundRowFilter, vectorFilter: target.vectorFilter }
                 : undefined;
-            const entries = (0, search_ts_1.executeConditions)(conditions, operator, _a, readTxn, target, context, (results, filters) => transformToEntries(results, select, context, readTxn, filters), filtered, recordAccess);
-            const ensure_loaded = target.ensureLoaded !== false;
-            // The guards inside executeConditions evaluate the
-            // LOCAL record, but on a caching table transformEntryForSelect may then revalidate an
-            // expired/invalidated row from source and return a DIFFERENT record. The explicit row filter
-            // must hold on the record actually returned, so it is re-checked
-            // there, after materialization (the earlier evaluation stays as a prune that also bounds HNSW
-            // traversal). vectorFilter and condition filters intentionally keep the local-record
-            // semantics all query filters have on caching tables.
-            //
-            // A row that is past its TTL but not yet swept by the background eviction
-            // scan is still physically present. A write that is about to overwrite it
-            // anyway (e.g. the SQL engine locating UPDATE/DELETE targets) needs to see
-            // it as a match — the same leniency a direct by-id put/patch already gets,
-            // since those never run the ensureLoaded-gated freshness check this transform
-            // otherwise applies unconditionally to every read.
-            const includeExpired = target.includeExpired === true;
-            const transformToRecord = _a.transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, true, boundRowFilter, includeExpired, postOrdering);
-            let results = _a.transformToOrderedSelect(entries, select, postOrdering, context, readTxn, transformToRecord);
-            const offset = target.offset || 0;
-            const end = target.limit !== undefined ? offset + target.limit : undefined;
-            // `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
-            // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
-            // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
-            // table estimate. Opt-in only — the default streaming path below is untouched.
-            //
-            // Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
-            // offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
-            // window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
-            // oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
-            // limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
-            // normal streaming path with no count. This bounds the offset too: without it a huge offset would
-            // postpone the exact guardrail (which only engages past the page) until that offset was scanned.
-            const pageLimit = target.limit;
-            if (target.count &&
-                Number.isInteger(pageLimit) &&
-                pageLimit >= 0 &&
-                pageLimit <= MAX_COUNT_PAGE &&
-                Number.isInteger(offset) &&
-                offset >= 0 &&
-                offset + pageLimit <= MAX_EXACT_COUNT_SCAN) {
-                const wantExact = target.count === 'exact';
-                const pageEnd = offset + pageLimit;
-                const countStart = node_perf_hooks_1.performance.now();
-                // A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
-                // chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
-                // the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
-                // different `count=exact` totals. Any query whose execution touches a custom index is affected: a
-                // custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
-                // filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
-                // filter. Report the total as unavailable for those rather than advertising it as count=exact
-                // (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
-                // sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
-                const touchesCustomIndex = (conds) => conds.some((c) => {
-                    if (!c)
-                        return false;
-                    if (c.conditions)
-                        return touchesCustomIndex(c.conditions);
-                    const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
-                    return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
-                });
-                const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
-                return (async () => {
-                    const page = [];
-                    let scanned = 0;
-                    let exact = true;
-                    try {
-                        for await (const record of results) {
-                            if (scanned >= offset && scanned < pageEnd)
-                                page.push(record);
-                            scanned++;
-                            // A store whose async iterator settles synchronously (the common indexed-scan case) would
-                            // otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
-                            // for the whole count. Yield to the macrotask queue periodically so concurrent requests and
-                            // I/O still make progress during a large exact scan.
-                            if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0)
-                                await new Promise((resolve) => setImmediate(resolve));
-                            // The page window [offset, pageEnd) is always collected in full first — the guardrail
-                            // only ever abandons the running TOTAL, never truncates the page body.
-                            if (scanned >= pageEnd) {
-                                // `estimated` needs nothing past the page; an approximate (vector) exact total is going to
-                                // be reported unavailable anyway, so don't drain its tail for a number we won't publish.
-                                if (!wantExact || approximateResultSet)
-                                    break;
-                                // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
-                                // large match set can't turn a bounded page fetch into an unbounded scan.
-                                if (scanned > MAX_EXACT_COUNT_SCAN || node_perf_hooks_1.performance.now() - countStart > MAX_EXACT_COUNT_MS) {
-                                    exact = false;
-                                    break;
+            try {
+                const entries = (0, search_ts_1.executeConditions)(conditions, operator, _a, readTxn, target, context, (results, filters) => transformToEntries(results, select, context, readTxn, filters), filtered, recordAccess);
+                const ensure_loaded = target.ensureLoaded !== false;
+                // The guards inside executeConditions evaluate the
+                // LOCAL record, but on a caching table transformEntryForSelect may then revalidate an
+                // expired/invalidated row from source and return a DIFFERENT record. The explicit row filter
+                // must hold on the record actually returned, so it is re-checked
+                // there, after materialization (the earlier evaluation stays as a prune that also bounds HNSW
+                // traversal). vectorFilter and condition filters intentionally keep the local-record
+                // semantics all query filters have on caching tables.
+                //
+                // A row that is past its TTL but not yet swept by the background eviction
+                // scan is still physically present. A write that is about to overwrite it
+                // anyway (e.g. the SQL engine locating UPDATE/DELETE targets) needs to see
+                // it as a match — the same leniency a direct by-id put/patch already gets,
+                // since those never run the ensureLoaded-gated freshness check this transform
+                // otherwise applies unconditionally to every read.
+                const includeExpired = target.includeExpired === true;
+                const transformToRecord = _a.transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded, true, boundRowFilter, includeExpired, postOrdering);
+                let results = _a.transformToOrderedSelect(entries, select, postOrdering, context, readTxn, transformToRecord);
+                const offset = target.offset || 0;
+                const end = target.limit !== undefined ? offset + target.limit : undefined;
+                // `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
+                // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
+                // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
+                // table estimate. Opt-in only — the default streaming path below is untouched.
+                //
+                // Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
+                // offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
+                // window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
+                // oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
+                // limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
+                // normal streaming path with no count. This bounds the offset too: without it a huge offset would
+                // postpone the exact guardrail (which only engages past the page) until that offset was scanned.
+                const pageLimit = target.limit;
+                if (target.count &&
+                    Number.isInteger(pageLimit) &&
+                    pageLimit >= 0 &&
+                    pageLimit <= MAX_COUNT_PAGE &&
+                    Number.isInteger(offset) &&
+                    offset >= 0 &&
+                    offset + pageLimit <= MAX_EXACT_COUNT_SCAN) {
+                    const wantExact = target.count === 'exact';
+                    const pageEnd = offset + pageLimit;
+                    const countStart = node_perf_hooks_1.performance.now();
+                    // A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is
+                    // chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not
+                    // the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two
+                    // different `count=exact` totals. Any query whose execution touches a custom index is affected: a
+                    // custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold
+                    // filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector
+                    // filter. Report the total as unavailable for those rather than advertising it as count=exact
+                    // (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector
+                    // sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact.
+                    const touchesCustomIndex = (conds) => conds.some((c) => {
+                        if (!c)
+                            return false;
+                        if (c.conditions)
+                            return touchesCustomIndex(c.conditions);
+                        const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
+                        return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
+                    });
+                    const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
+                    return (async () => {
+                        const page = [];
+                        let scanned = 0;
+                        let exact = true;
+                        try {
+                            for await (const record of results) {
+                                if (scanned >= offset && scanned < pageEnd)
+                                    page.push(record);
+                                scanned++;
+                                // A store whose async iterator settles synchronously (the common indexed-scan case) would
+                                // otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop
+                                // for the whole count. Yield to the macrotask queue periodically so concurrent requests and
+                                // I/O still make progress during a large exact scan.
+                                if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0)
+                                    await new Promise((resolve) => setImmediate(resolve));
+                                // The page window [offset, pageEnd) is always collected in full first — the guardrail
+                                // only ever abandons the running TOTAL, never truncates the page body.
+                                if (scanned >= pageEnd) {
+                                    // `estimated` needs nothing past the page; an approximate (vector) exact total is going to
+                                    // be reported unavailable anyway, so don't drain its tail for a number we won't publish.
+                                    if (!wantExact || approximateResultSet)
+                                        break;
+                                    // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
+                                    // large match set can't turn a bounded page fetch into an unbounded scan.
+                                    if (scanned > MAX_EXACT_COUNT_SCAN || node_perf_hooks_1.performance.now() - countStart > MAX_EXACT_COUNT_MS) {
+                                        exact = false;
+                                        break;
+                                    }
                                 }
                             }
                         }
-                    }
-                    finally {
-                        // We own the iteration here (no results.onDone consumer), so release the read
-                        // transaction unconditionally — including when the drain throws — or the snapshot leaks.
-                        txn.doneReadTxn();
-                    }
-                    let total;
-                    if (wantExact) {
-                        // `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
-                        // an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
-                        total = exact && !approximateResultSet ? scanned : null;
-                    }
-                    else if (boundRowFilter || typeof target.vectorFilter === 'function') {
-                        // An opaque row/vector filter shapes the result but isn't reflected in the index/condition
-                        // estimate; guessing would both mislead and disclose cardinality the filter hides.
-                        total = null;
-                    }
-                    else if (!hasUserConditions) {
-                        total = (0, search_ts_1.estimatedEntryCount)(primaryStore);
-                    }
-                    else {
-                        // Estimate from the real conditions only — drop the planner's synthetic `sort`
-                        // pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
-                        const est = (0, search_ts_1.estimateCondition)(_a)({
-                            conditions: conditions.filter((c) => c.comparator !== 'sort'),
-                            operator: operator ? String(operator).toLowerCase() : 'and',
-                        });
-                        total = isFinite(est) ? Math.round(est) : null;
-                    }
-                    // For an estimate, never report a total below the last row actually returned — keeps the
-                    // Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
-                    // Exact totals are authoritative (and an empty page past the end must not be clamped up).
-                    if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
-                        total = offset + page.length;
-                    }
-                    page.recordCount = total;
-                    page.recordCountExact = wantExact && exact && !approximateResultSet;
-                    page.selectApplied = true;
-                    page.getColumns = getColumns;
-                    return page;
-                })();
-            }
-            // apply any offset/limit after all the sorting and filtering
-            if (target.offset || target.limit !== undefined)
-                results = results.slice(offset, end);
-            results.onDone = () => {
-                results.onDone = null; // ensure that it isn't called twice
+                        finally {
+                            // We own the iteration here (no results.onDone consumer), so release the read
+                            // transaction unconditionally — including when the drain throws — or the snapshot leaks.
+                            txn.doneReadTxn();
+                        }
+                        let total;
+                        if (wantExact) {
+                            // `scanned` is only an authoritative total when the iteration was exhaustive and deterministic;
+                            // an approximate (vector/HNSW) result set is neither, so report the total as unavailable.
+                            total = exact && !approximateResultSet ? scanned : null;
+                        }
+                        else if (boundRowFilter || typeof target.vectorFilter === 'function') {
+                            // An opaque row/vector filter shapes the result but isn't reflected in the index/condition
+                            // estimate; guessing would both mislead and disclose cardinality the filter hides.
+                            total = null;
+                        }
+                        else if (!hasUserConditions) {
+                            total = (0, search_ts_1.estimatedEntryCount)(primaryStore);
+                        }
+                        else {
+                            // Estimate from the real conditions only — drop the planner's synthetic `sort`
+                            // pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
+                            const est = (0, search_ts_1.estimateCondition)(_a)({
+                                conditions: conditions.filter((c) => c.comparator !== 'sort'),
+                                operator: operator ? String(operator).toLowerCase() : 'and',
+                            });
+                            total = isFinite(est) ? Math.round(est) : null;
+                        }
+                        // For an estimate, never report a total below the last row actually returned — keeps the
+                        // Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
+                        // Exact totals are authoritative (and an empty page past the end must not be clamped up).
+                        if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
+                            total = offset + page.length;
+                        }
+                        page.recordCount = total;
+                        page.recordCountExact = wantExact && exact && !approximateResultSet;
+                        page.selectApplied = true;
+                        page.getColumns = getColumns;
+                        return page;
+                    })();
+                }
+                // apply any offset/limit after all the sorting and filtering
+                if (target.offset || target.limit !== undefined)
+                    results = results.slice(offset, end);
+                results.onDone = () => {
+                    results.onDone = null; // ensure that it isn't called twice
+                    txn.doneReadTxn();
+                };
+                results.selectApplied = true;
+                results.getColumns = getColumns;
+                return results;
+            }
+            catch (error) {
                 txn.doneReadTxn();
-            };
-            results.selectApplied = true;
-            results.getColumns = getColumns;
-            return results;
+                throw error;
+            }
         }
         /**
          * This is responsible for ordering and select()ing the attributes/properties from returned entries
@@ -4108,10 +4637,18 @@ function makeTable(options) {
             if (sort) {
                 // there might be some situations where we don't need to transform to entries for sorting, not sure
                 entries = transformToEntries(entries, select, context, readTxn, null);
-                let ordered;
+                // Sort keys are resolved as entries are collected, so comparison never dereferences a record: a
+                // cached entry holds its record only weakly, and a re-read per comparison is what this avoids.
+                const clauses = [];
+                for (let order = sort; order; order = order.next)
+                    clauses.push(order);
+                const clauseCount = clauses.length;
                 // if we are doing post-ordering, we need to get records first, then sort them
                 results.iterate = function (options) {
-                    let sortedArrayIterator;
+                    let ordered;
+                    let orderedKeys;
+                    let sortedPositions;
+                    let sortedIndex;
                     const dbIterator = options?.async && entries[Symbol.asyncIterator]
                         ? entries[Symbol.asyncIterator]()
                         : entries[Symbol.iterator]();
@@ -4120,26 +4657,34 @@ function makeTable(options) {
                     let enqueuedEntryForNextGroup;
                     let lastGroupingValue;
                     let firstEntry = true;
-                    function createComparator(order) {
-                        const nextComparator = order.next && createComparator(order.next);
-                        const descending = order.descending;
-                        return (entryA, entryB) => {
-                            const a = getAttributeValue(entryA, order.attribute, context, order);
-                            const b = getAttributeValue(entryB, order.attribute, context, order);
-                            const diff = descending
-                                ? (0, ordered_binary_1.compareKeys)(convertToComparableKeys(b), convertToComparableKeys(a))
-                                : (0, ordered_binary_1.compareKeys)(convertToComparableKeys(a), convertToComparableKeys(b));
-                            if (diff === 0)
-                                return nextComparator?.(entryA, entryB) || 0;
-                            return diff;
-                        };
+                    function collect(entry) {
+                        ordered.push(entry);
+                        for (let i = 0; i < clauseCount; i++) {
+                            const clause = clauses[i];
+                            orderedKeys[i].push(convertToComparableKeys(getAttributeValue(entry, clause.attribute, context, clause)));
+                        }
+                    }
+                    function comparePositions(positionA, positionB) {
+                        for (let i = 0; i < clauseCount; i++) {
+                            const keys = orderedKeys[i];
+                            const diff = clauses[i].descending
+                                ? (0, ordered_binary_1.compareKeys)(keys[positionB], keys[positionA])
+                                : (0, ordered_binary_1.compareKeys)(keys[positionA], keys[positionB]);
+                            if (diff !== 0)
+                                return diff;
+                        }
+                        return 0;
+                    }
+                    function nextSorted() {
+                        if (sortedIndex < sortedPositions.length)
+                            return { done: false, value: ordered[sortedPositions[sortedIndex++]] };
+                        return { done: true, value: undefined };
                     }
-                    const comparator = createComparator(sort);
                     return {
                         async next() {
                             let iteration;
-                            if (sortedArrayIterator) {
-                                iteration = sortedArrayIterator.next();
+                            if (sortedPositions) {
+                                iteration = nextSorted();
                                 if (iteration.done) {
                                     if (dbDone) {
                                         if (results.onDone)
@@ -4153,8 +4698,11 @@ function makeTable(options) {
                                     };
                             }
                             ordered = [];
+                            orderedKeys = [];
+                            for (let i = 0; i < clauseCount; i++)
+                                orderedKeys.push([]);
                             if (enqueuedEntryForNextGroup)
-                                ordered.push(enqueuedEntryForNextGroup);
+                                collect(enqueuedEntryForNextGroup);
                             // need to load all the entries into ordered
                             do {
                                 iteration = await dbIterator.next();
@@ -4186,17 +4734,18 @@ function makeTable(options) {
                                             break;
                                         }
                                     }
-                                    // we store the value we will sort on, for fast sorting, and the entry so the records can be GC'ed if necessary
-                                    // before the sorting is completed
-                                    ordered.push(entry);
+                                    collect(entry);
                                 }
                             } while (true);
                             if (sort.isGrouped) {
                                 // TODO: Return grouped results
                             }
-                            ordered.sort(comparator);
-                            sortedArrayIterator = ordered[Symbol.iterator]();
-                            iteration = sortedArrayIterator.next();
+                            sortedPositions = [];
+                            for (let i = 0; i < ordered.length; i++)
+                                sortedPositions.push(i);
+                            sortedPositions.sort(comparePositions);
+                            sortedIndex = 0;
+                            iteration = nextSorted();
                             if (!iteration.done)
                                 return {
                                     value: await transformToRecord.call(this, iteration.value),
@@ -4601,6 +5150,9 @@ function makeTable(options) {
                     return;
                 try {
                     let type = auditRecord.type;
+                    // Ahead of the rawEvents branch, which forwards every type verbatim.
+                    if ((0, auditStore_ts_1.isLockControlType)(type))
+                        return;
                     let value;
                     if (type === 'message' || request.rawEvents) {
                         // we only send the full message, this are individual messages that can be sent out of order
@@ -4699,7 +5251,9 @@ function makeTable(options) {
                                     if (!isActive())
                                         return;
                                 }
-                                if (auditRecord.tableId !== tableId)
+                                if (auditRecord.tableId !== tableId || auditRecord.type === 'evict')
+                                    continue;
+                                if ((0, auditStore_ts_1.isLockControlType)(auditRecord.type))
                                     continue;
                                 const id = auditRecord.recordId;
                                 if (thisId == null || isDescendantId(thisId, id)) {
@@ -4739,7 +5293,9 @@ function makeTable(options) {
                                     return;
                             }
                             try {
-                                if (auditRecord.tableId !== tableId)
+                                if (auditRecord.tableId !== tableId || auditRecord.type === 'evict')
+                                    continue;
+                                if ((0, auditStore_ts_1.isLockControlType)(auditRecord.type))
                                     continue;
                                 const id = auditRecord.recordId;
                                 if (thisId == null || isDescendantId(thisId, id)) {
@@ -5171,6 +5727,108 @@ function makeTable(options) {
                 });
             });
         }
+        /**
+         * Write one cluster record-lock control entry (harper#483 Phase 1). Not local-only: replicating
+         * it IS the send.
+         *
+         * `recordId` must stay null. An entry carrying the locked key would share
+         * `(version, tableId, recordId, nodeId)` with the holder's own first write, which is stamped at
+         * exactly `ts_R`, and `RocksTransactionLogStore.getSync` answers with the FIRST entry at a
+         * timestamp and key — so `_writeUpdate`'s keyed dedup would find this one and drop that write.
+         * The payload goes in as bytes rather than through `recordUpdater`, which would run it through
+         * schema projection and the table's shared structure dictionary.
+         */
+        static writeLockControlEntry(entry) {
+            const encodedRecord = (0, recordLockCoordinator_ts_1.encodeLockControlPayload)(entry);
+            const nodeId = (0, nodeIdMapping_ts_1.getThisNodeId)(auditStore) ?? 0;
+            let position;
+            // No entry pins its clock, the request included. `ts_R` is minted before the write, so pinning
+            // to it can land the entry behind a peer's replication cursor if any write to this table
+            // commits in between — the same hazard that rules it out for grants and releases, which are
+            // written later still. The protocol reads `ts_R` from the payload, so the entry's own log key
+            // never has to equal it.
+            const context = {};
+            return Promise.resolve((0, transaction_ts_1.transaction)(context, (txn) => {
+                const tableTxn = txnForContext({ transaction: txn });
+                tableTxn.addWrite({
+                    key: null,
+                    store: primaryStore,
+                    skipReplicationConfirmation: true,
+                    commit: (txnTime, _existingEntry, _retry, nativeTransaction) => {
+                        position = txnTime;
+                        return auditStore[isRocksDB ? 'putSync' : 'put'](null, {
+                            version: txnTime,
+                            tableId,
+                            recordId: null,
+                            nodeId,
+                            type: entry.type,
+                            encodedRecord,
+                            extendedType: 0,
+                            // Zero, not the table's count: these bytes were packed by the private control `Packr`
+                            // and carry none of the table's structures. `RocksTransactionLogStore` raises the
+                            // per-(log, table) structure watermark from this field and flags the entry that does
+                            // it, so claiming the table's version would let a release take `HAS_STRUCTURE_UPDATE`
+                            // and leave the next real write at that version unflagged — a receiver that learns
+                            // structures only from flagged entries then decodes later records against a stale set
+                            // (harper#1348's class). A payload with no table structures cannot advance them.
+                            structureVersion: 0,
+                        }, { instructedWrite: true, transaction: nativeTransaction, nodeId, viaNodeId: nodeId });
+                    },
+                });
+            })).then(() => position);
+        }
+        /**
+         * The coordinator that holds this node's admissions, transport or not. Releasing and registering
+         * go here rather than through `lockCoordinator`, which answers undefined while a transport is
+         * momentarily unregistered — and a release dropped on that answer leaves the key's home holding
+         * its grant until the delegation's own deadline.
+         */
+        static get admittingCoordinator() {
+            return lockCoordinator;
+        }
+        /**
+         * This table's cluster lock coordinator, created on first use and only while a transport is
+         * registered for the database. Nothing is allocated on the Phase 0 path.
+         */
+        static get lockCoordinator() {
+            const transport = (0, recordLockCoordinator_ts_1.getClusterLockTransport)(databaseName);
+            if (!transport) {
+                // Deliberately NOT closed. harper-pro unregisters without a standalone claim during a
+                // reconnect, and closing here would drop this node's record of the delegations it has
+                // issued as a home — so the next registration would start empty and could grant a key
+                // whose delegate is still admitting. The coordinator keeps ticking, its grants expire on
+                // their own deadlines, and `isClusterLockRequired` is what fails an acquire closed in the
+                // meantime. A genuine standalone claim clears the requirement and the coordinator with it.
+                if (!(0, recordLockCoordinator_ts_1.isClusterLockRequired)(databaseName)) {
+                    lockCoordinator?.close();
+                    lockCoordinator = undefined;
+                }
+                return undefined;
+            }
+            if (lockCoordinator?.transport !== transport) {
+                // The transport object changed, but this node's delegations and the handles they admitted
+                // did not. The successor adopts that live authority in its constructor; the predecessor
+                // is closed afterwards so nothing is dropped in between. See LockCoordinatorOptions.adopt.
+                const predecessor = lockCoordinator;
+                lockCoordinator = new recordLockCoordinator_ts_1.LockCoordinator({
+                    database: databaseName,
+                    table: tableName,
+                    nodeId: (0, nodeName_ts_1.getThisNodeName)(),
+                    transport,
+                    adopt: predecessor,
+                    // Writing to the local transaction log IS the send, so a transport that only computes
+                    // the participant set gets core's writer.
+                    writeControl: transport.writeControl
+                        ? (entry) => transport.writeControl(tableName, entry)
+                        : (entry) => _a.writeLockControlEntry(entry),
+                    keyIdOf: DatabaseTransaction_ts_1.writeKeyId,
+                    nextTimestamp: () => primaryStore.getMonotonicTimestamp(),
+                    grantableAfterMono: transport.grantableAfterMono,
+                });
+                predecessor?.close();
+            }
+            return lockCoordinator;
+        }
         // #section: validation
         validate(record, patch) {
             // Accumulate structured per-field issues so the 400 carries `{ path, code,
@@ -5383,85 +6041,182 @@ function makeTable(options) {
             const exactCount = options?.exactCount;
             const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms
             const start = node_perf_hooks_1.performance.now();
-            // `entryCount` (the exact key count) is only needed once the scan blows the time budget --
-            // to decide whether to estimate and as the extrapolation base. On RocksDB it is a full
-            // key-only scan, so we defer it: tables that finish within budget (the common case) and
-            // `exact_count` requests never pay for it. `halfway`/`entryCount` stay 0 until first computed.
             let entryCount = 0;
-            let halfway = 0;
-            let counted = false;
-            let completeForExact = false;
+            let remainderPhysical = 0;
+            let estimator;
+            // feature-detected per DESIGN.md's invariant for this API family; LMDB stores do not implement it
+            const canEstimate = typeof primaryStore.createCountEstimator === 'function' && typeof primaryStore.estimateCount === 'function';
+            let estimatorFailed = false;
+            let warnedNoBase = false;
+            let checkpoints = 0;
+            let checkpointedEntries = 0;
             let recordCount = 0;
             let entriesScanned = 0;
+            let lastKey;
             let limit;
-            for (const { value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) {
+            let nextCheckAt = start + TIME_LIMIT;
+            for (const { key, value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) {
                 if (value != null)
                     recordCount++;
                 entriesScanned++;
+                lastKey = key;
                 await rest();
-                if (!exactCount && !completeForExact && node_perf_hooks_1.performance.now() - start > TIME_LIMIT) {
-                    if (!counted) {
-                        counted = true;
-                        entryCount = isRocksDB
-                            ? primaryStore.getKeysCount({ start: undefined })
-                            : primaryStore.getStats().entryCount;
-                        halfway = Math.floor(entryCount / 2);
-                    }
-                    if (entriesScanned < halfway) {
-                        // it is taking too long, so we will just take this sample and a sample from the end to estimate
-                        limit = entriesScanned;
-                        break;
+                // a table too small to reach the floor is small enough to finish exactly
+                if (exactCount || entriesScanned < MIN_ESTIMATOR_SAMPLE)
+                    continue;
+                const now = node_perf_hooks_1.performance.now();
+                if (now <= nextCheckAt)
+                    continue;
+                nextCheckAt = now + TIME_LIMIT;
+                checkpoints++;
+                if (canEstimate && !estimatorFailed) {
+                    try {
+                        estimator ??= primaryStore.createCountEstimator({ start: true });
+                        estimator.advance(lastKey, entriesScanned - checkpointedEntries);
+                        checkpointedEntries = entriesScanned;
+                        entryCount = usableCount(estimator.estimate());
+                    }
+                    catch (error) {
+                        // a store closing concurrently -- drop_table can, while this scan is parked in a yield
+                        logger_ts_1.logger.debug?.('Count estimator unavailable, falling back to an exact scan', error);
+                        estimatorFailed = true;
+                        estimator = undefined;
+                        entryCount = 0;
                     }
-                    // Past the halfway point already: finishing the scan for an exact count is cheaper
-                    // than estimating. Set the flag so we stop re-evaluating the budget on each remaining iteration.
-                    completeForExact = true;
+                }
+                else if (!canEstimate) {
+                    // `canEstimate` false is not the same as "LMDB": a RocksDB store whose native module predates
+                    // the estimator API lands here too, and `RocksDatabase.getStats()` carries no `entryCount`.
+                    try {
+                        const stats = primaryStore.getStats?.();
+                        entryCount = Number.isFinite(stats?.entryCount) && stats.entryCount > 0 ? stats.entryCount : 0;
+                    }
+                    catch {
+                        entryCount = 0;
+                    }
+                }
+                if (!entryCount && canEstimate && !estimatorFailed) {
+                    // Range estimates are block-granular and can report 0 for a store whose entries are still
+                    // in the memtable. Without a base the escape cannot fire at all, so fall back to the
+                    // whole-store property rather than silently walking the table.
+                    try {
+                        const wholeStore = primaryStore.getEstimatedKeyCount();
+                        entryCount = Number.isFinite(wholeStore) && wholeStore > 0 ? wholeStore : 0;
+                    }
+                    catch {
+                        entryCount = 0;
+                    }
+                    if (!entryCount && !warnedNoBase) {
+                        warnedNoBase = true;
+                        logger_ts_1.logger.debug?.(`No usable key-count estimate for ${tableName}; counting records by full scan`);
+                    }
+                }
+                // Zero is "no usable base": degrade to the exact scan. The checkpoint ceiling is what stops a
+                // base that keeps undershooting from holding the halfway test false forever and walking the
+                // whole table; the reverse sample is bounded by `limit` in turn.
+                if (entryCount > 0 &&
+                    (checkpoints >= MAX_ESTIMATE_CHECKPOINTS || entriesScanned < Math.floor(entryCount / 2))) {
+                    if (canEstimate) {
+                        try {
+                            const remaining = primaryStore.estimateCount({ start: lastKey, exclusiveStart: true });
+                            // widened by its own reported untrustworthiness: block-granular, so it can land below
+                            // the live count it is meant to bound
+                            const remainingCount = usableCount(remaining);
+                            remainderPhysical = remainingCount > 0 ? remainingCount * (2 - remaining.confidence) : 0;
+                        }
+                        catch {
+                            remainderPhysical = 0;
+                        }
+                        // A zero or unusable remainder is valid -- entries still in the memtable read as none
+                        // through range statistics -- but it would leave `baseMax` resting on the sampled ends
+                        // alone. The whole-store property is a separate, non-range source, so fall back to it.
+                        if (!remainderPhysical) {
+                            try {
+                                const wholeStore = primaryStore.getEstimatedKeyCount();
+                                if (Number.isFinite(wholeStore))
+                                    remainderPhysical = Math.max(wholeStore - entriesScanned, 0);
+                            }
+                            catch {
+                                remainderPhysical = 0;
+                            }
+                        }
+                    }
+                    limit = entriesScanned;
+                    break;
                 }
             }
             if (limit) {
                 // in this case we are going to make an estimate of the table count using the first thousand
                 // entries and last thousand entries
                 const firstRecordCount = recordCount;
+                const firstKey = lastKey;
                 recordCount = 0;
                 // Bound the reverse scan explicitly. The getRange `limit` option is honored by lmdb-js but
                 // ignored by rocksdb-js; without this break the scan reads the whole table, so `recordRate`
                 // blows up to ~entryCount/(2*limit) and the estimate scales with entryCount^2 -- the source
                 // of the wildly inflated `record_count` (e.g. 20,000,000 for ~105k rows) on large RocksDB
-                // tables. The early-exit above guarantees limit < entryCount/2, so the two samples stay disjoint.
+                // tables.
                 let reverseScanned = 0;
-                for (const { value } of primaryStore.getRange({
+                // Sized independently of the forward scan. `entriesScanned` is whatever the forward pass
+                // covered before it escaped, and the checkpoint ceiling lets that run twenty budget
+                // intervals when the base keeps undershooting; matching it here would read that same count
+                // again and double the wall clock of the call this path exists to bound.
+                const reverseLimit = Math.min(limit, MIN_ESTIMATOR_SAMPLE);
+                // Disjointness is enforced against the forward scan's own last key rather than inferred from
+                // the base, which is an estimate that can overshoot by more than 2x.
+                let sampledWholeTable = false;
+                for (const { key, value } of primaryStore.getRange({
                     start: '\uffff',
                     reverse: true,
                     lazy: true,
-                    limit,
+                    limit: reverseLimit,
                     snapshot: false,
                 })) {
+                    if ((0, ordered_binary_1.compareKeys)(key, firstKey) <= 0) {
+                        sampledWholeTable = true;
+                        break;
+                    }
                     if (value != null)
                         recordCount++;
                     reverseScanned++;
                     await rest();
-                    if (reverseScanned >= limit)
+                    if (reverseScanned >= reverseLimit)
                         break;
                 }
+                // the samples met, so between them they covered every entry
+                if (sampledWholeTable)
+                    return { recordCount: recordCount + firstRecordCount };
                 // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit`
                 // (concurrent deletions under snapshot:false, or an overestimated entryCount), and counting
                 // those un-scanned slots would inflate the denominator and underestimate the rate.
-                const sampleSize = limit + reverseScanned;
-                const recordRate = (recordCount + firstRecordCount) / sampleSize;
-                const variance = Math.pow((recordCount - firstRecordCount + 1) / limit / 2, 2) + // variance between samples
-                    (recordRate * (1 - recordRate)) / sampleSize;
-                const sd = Math.max(Math.sqrt(variance) * entryCount, 1);
-                const estimatedRecordCount = Math.round(recordRate * entryCount);
-                // TODO: This uses a normal/Wald interval, but a binomial confidence interval is probably better calculated using
-                // Wilson score interval or Agresti-Coull interval (I think the latter is a little easier to calculate/implement).
-                const lowerCiLimit = Math.max(estimatedRecordCount - 1.96 * sd, recordCount + firstRecordCount);
-                const upperCiLimit = Math.min(estimatedRecordCount + 1.96 * sd, entryCount);
-                let significantUnit = Math.pow(10, Math.round(Math.log10(sd)));
-                if (significantUnit > estimatedRecordCount)
-                    significantUnit = significantUnit / 10;
-                recordCount = Math.round(estimatedRecordCount / significantUnit) * significantUnit;
+                const sampledRecords = recordCount + firstRecordCount;
+                const recordRate = sampledRecords / (limit + reverseScanned);
+                // Endpoints for the extrapolation base, spanning both ways an estimated base can be wrong:
+                // every remaining entry superseded (only what was sampled is live) through every remaining
+                // entry live (the uncalibrated physical count). Churn concentrated outside the sampled ends
+                // calibrates to nothing, so an interval derived from the estimator's confidence would sit
+                // narrowly around the wrong number. Both endpoints are themselves estimates on RocksDB, so
+                // this is a widened heuristic interval, not a guaranteed bound on the live count.
+                const baseMin = entriesScanned + reverseScanned;
+                const baseMax = Math.max(entriesScanned + remainderPhysical, entryCount, baseMin);
+                const estimatedRecordCount = Math.round(recordRate * Math.max(entryCount, baseMin));
+                // The samples counted these directly, and the entries between them can only add; everything
+                // outside the samples could be live. A statistical interval inside those endpoints would be
+                // narrowest exactly where the ends are least representative of the middle -- sampled ends
+                // that are all deletion entries give a rate of 0, collapsing an upper end to ~0 with live
+                // rows in between -- so the endpoints are the evidence itself.
+                const lower = sampledRecords;
+                const upper = Math.round(baseMax);
+                // Report only the precision the interval supports, but never so coarse a unit that the
+                // estimate rounds away: `baseMax` is physical and can exceed a calibrated estimate by
+                // orders of magnitude, which a single division cannot walk back.
+                let significantUnit = Math.pow(10, Math.round(Math.log10(Math.max((upper - lower) / 2, 1))));
+                while (significantUnit > estimatedRecordCount && significantUnit > 1)
+                    significantUnit /= 10;
+                recordCount = Math.min(Math.max(Math.round(estimatedRecordCount / significantUnit) * significantUnit, lower), upper);
                 return {
                     recordCount,
-                    estimatedRange: [Math.round(lowerCiLimit), Math.round(upperCiLimit)],
+                    estimatedRange: [lower, upper],
                 };
             }
             return {
@@ -5475,6 +6230,7 @@ function makeTable(options) {
             // Refresh on every call: schema reload mutates `attributes` in place, so the
             // class-construction snapshot would otherwise go stale.
             this.embedAttributes = this.attributes.filter((a) => a?.embed);
+            expiresAtProperty = this.attributes.find((attribute) => attribute.expiresAt);
             // Drop registry entries for attributes that are no longer `@embed`, so a dropped
             // directive doesn't leave a stale embedder or block a default refresh on re-add.
             const embedNames = new Set(this.embedAttributes.map((a) => a.name));
@@ -5541,7 +6297,7 @@ function makeTable(options) {
                                 const id = object[relationship.from ? relationship.from : primaryKey];
                                 const relatedTable = attribute.elements.definition.tableClass;
                                 if (returnEntry) {
-                                    return (0, search_ts_1.searchByIndex)({ attribute: relationship.to, value: id }, txnForContext(context).getReadTxn(), false, relatedTable, false).map((entry) => {
+                                    return (0, search_ts_1.searchByIndex)({ attribute: relationship.to, value: id }, txnForContext(context).getReadTxn(), false, relatedTable, { allowFullScan: false }).map((entry) => {
                                         if (entry && entry.key !== undefined)
                                             return entry;
                                         return relatedTable.primaryStore.getEntry(entry, {
@@ -5788,10 +6544,24 @@ function makeTable(options) {
             }
             const drainRemovals = () => Promise.all(inFlightRemovals);
             let entriesDeleted = 0;
+            // LMDB only: RocksTransactionLogStore.remove() is a no-op, so a RocksDB deleteHistory removes
+            // nothing and must not claim it did.
+            // A bound above everything reachable must not be recorded as the floor: the floor only rises
+            // and a store with a record is never re-stamped, so it would never come down, for every table in
+            // this database. `boundedAuditPruneEnd` clamps the cutoff to just above the newest key in the
+            // log, and the scan below uses that same value as its range end, so the prune cannot remove an
+            // entry the floor does not cover.
+            let pruneEnd = endTime;
+            if (!isRocksDB) {
+                pruneEnd = (0, auditStore_ts_1.boundedAuditPruneEnd)(auditStore, endTime);
+                (0, auditStore_ts_1.raiseAuditFloor)(auditStore, pruneEnd);
+            }
             try {
                 for (const auditRecord of auditStore.getRange({
-                    start: 1, // must not be zero; see getHistory below for why
-                    end: endTime,
+                    // must not be zero: 0 encodes to all zero bytes and so overlaps the symbol keys, as in
+                    // getHistory below
+                    start: 1,
+                    end: pruneEnd,
                 })) {
                     await rest(); // yield to other async operations
                     if (auditRecord.tableId !== tableId)
@@ -5816,7 +6586,7 @@ function makeTable(options) {
                         const auditTime = isRocksDB && version != null
                             ? resolveAuditHead(key, version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
                             : localTime;
-                        if (value === null && version != null && auditTime < endTime) {
+                        if (value === null && version != null && auditTime < pruneEnd) {
                             const backpressure = queueRemoval(() => primaryStore.remove(key, version), 'Error removing deleted record during deleteHistory');
                             if (backpressure)
                                 await backpressure;
@@ -5840,7 +6610,7 @@ function makeTable(options) {
                 end: endTime,
             })) {
                 await rest(); // yield to other async operations
-                if (auditRecord.tableId !== tableId)
+                if (auditRecord.tableId !== tableId || auditRecord.type === 'evict' || (0, auditStore_ts_1.isLockControlType)(auditRecord.type))
                     continue;
                 yield {
                     id: auditRecord.recordId,
@@ -5874,7 +6644,10 @@ function makeTable(options) {
                 let highestPreviousVersion = 0;
                 const start = nextVersion - auditWindow;
                 for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) {
-                    if (auditRecord.tableId === tableId && (0, ordered_binary_1.compareKeys)(auditRecord.recordId, id) === 0) {
+                    if (auditRecord.tableId === tableId &&
+                        auditRecord.type !== 'evict' &&
+                        !(0, auditStore_ts_1.isLockControlType)(auditRecord.type) &&
+                        (0, ordered_binary_1.compareKeys)(auditRecord.recordId, id) === 0) {
                         history.splice(insertionPoint, 0, {
                             id: auditRecord.recordId,
                             localTime: isRocksDB ? auditRecord.txnLogKey : auditRecord.version,
@@ -5905,6 +6678,7 @@ function makeTable(options) {
             const promises = [primaryStore.clear()];
             for (const key in indices) {
                 const index = indices[key];
+                index.customIndex?.resetDerivedStorage?.();
                 promises.push(index.clearAsync ? index.clearAsync() : index.clear());
             }
             return Promise.all(promises);
@@ -5912,6 +6686,7 @@ function makeTable(options) {
         /** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
         static cleanup() {
             disposed = true;
+            void _a.derivedIndexRuntime?.close();
             clearTimeout(cleanupTimer);
             settlePendingCleanup();
             clearInterval(recordExpirationInterval);
@@ -5938,9 +6713,16 @@ function makeTable(options) {
     });
     try {
         TableResource.updatedAttributes(); // on creation, update accessors as well
-        if (expirationMs)
-            TableResource.setTTLExpiration(expirationMs / 1000);
-        if (expiresAtProperty)
+        if (expirationMs) {
+            ttlFromLoad = true;
+            try {
+                TableResource.setTTLExpiration(expirationMs / 1000);
+            }
+            finally {
+                ttlFromLoad = false;
+            }
+        }
+        if (expiresAtProperty && !recordExpirationInterval)
             runRecordExpirationEviction();
     }
     catch (error) {
@@ -6729,25 +7511,35 @@ function makeTable(options) {
                     resolve(resolvedEntry);
                 }
                 catch (error) {
-                    error.message += ` while resolving record ${id} for ${tableName}`;
-                    if (existingRecord &&
-                        (((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
-                            !context?.mustRevalidate) ||
-                            (context?.staleIfError &&
-                                (error.statusCode === 500 ||
-                                    error.statusCode === 502 ||
-                                    error.statusCode === 503 ||
-                                    error.statusCode === 504)))) {
-                        // these are conditions under which we can use stale data after an error
-                        resolve({
-                            key: id,
-                            version: existingVersion,
-                            value: existingRecord,
-                        });
-                        logger_ts_1.logger.trace?.(error.message, '(returned stale record)');
+                    // A source may reject with anything at all, so deciding how to settle is itself
+                    // fallible: `message` is not assignable on every error (a DOMException from
+                    // AbortSignal.timeout), and a nullish rejection makes the reads below throw.
+                    // Leaving this promise unsettled hangs the caller forever, so every path here
+                    // has to end in resolve() or reject().
+                    try {
+                        (0, hdbError_ts_1.appendErrorContext)(error, ` while resolving record ${id} for ${tableName}`);
+                        if (existingRecord &&
+                            (((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
+                                !context?.mustRevalidate) ||
+                                (context?.staleIfError &&
+                                    (error.statusCode === 500 ||
+                                        error.statusCode === 502 ||
+                                        error.statusCode === 503 ||
+                                        error.statusCode === 504)))) {
+                            // these are conditions under which we can use stale data after an error
+                            resolve({
+                                key: id,
+                                version: existingVersion,
+                                value: existingRecord,
+                            });
+                            logger_ts_1.logger.trace?.(error?.message, '(returned stale record)');
+                        }
+                        else
+                            reject(error);
+                    }
+                    catch (settlingError) {
+                        reject(error ?? settlingError);
                     }
-                    else
-                        reject(error);
                     const resolveDuration = node_perf_hooks_1.performance.now() - start;
                     (0, write_ts_1.recordAction)(resolveDuration, 'cache-resolution', tableName, null, 'fail');
                     if (responseHeaders)
@@ -6957,6 +7749,7 @@ function makeTable(options) {
                     if (hasSourceGet && primaryStore.hasLock(item.key, entry.version))
                         continue; // resolution in progress
                     updateIndices(item.key, entry.value, null, options);
+                    stageDerivedIndexEviction(transaction, item.key, entry.version);
                 }
                 (0, RecordEncoder_ts_1.removeEntry)(primaryStore, entry, options);
                 staged++;
@@ -7054,7 +7847,7 @@ function makeTable(options) {
         if (cleanupInterval === lastCleanupInterval && !runImmediately)
             return;
         lastCleanupInterval = cleanupInterval;
-        if ((0, manageThreads_js_1.getWorkerIndex)() === (0, manageThreads_js_1.getWorkerCount)() - 1) {
+        if ((0, manageThreads_js_1.ownsStoreMaintenance)(primaryStore.path) || (ttlConfiguredByApplication && (0, manageThreads_js_1.isDedicatedWorker)())) {
             // run on the last thread so we aren't overloading lower-numbered threads
             if (cleanupTimer)
                 clearTimeout(cleanupTimer);
@@ -7196,12 +7989,14 @@ function makeTable(options) {
     }
     function runRecordExpirationEviction() {
         // Periodically evict expired records, searching for records who expiresAt timestamp is before now
-        if ((0, manageThreads_js_1.getWorkerIndex)() === 0) {
+        if ((0, manageThreads_js_1.ownsStoreExpiration)(primaryStore.path) || (ttlConfiguredByApplication && (0, manageThreads_js_1.isDedicatedWorker)())) {
             // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
             recordExpirationInterval = setInterval(async () => {
                 // go through each database and table and then search for expired entries
                 // find any entries that are set to expire before now
-                if (disposed || runningRecordExpiration)
+                // updatedAttributes() clears expiresAtProperty when a live redeclaration drops the directive,
+                // and there is nothing left for this interval to scan by
+                if (disposed || runningRecordExpiration || !expiresAtProperty)
                     return;
                 runningRecordExpiration = true;
                 try {